{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nset whether the Thread device requires the full network data or only requires the stable network data", "response": "def setNetworkDataRequirement(self, eDataRequirement):\n        \"\"\"set whether the Thread device requires the full network data\n           or only requires the stable network data\n\n        Args:\n            eDataRequirement: is true if requiring the full network data\n\n        Returns:\n            True: successful to set the network requirement\n        \"\"\"\n        print '%s call setNetworkDataRequirement' % self.port\n        print eDataRequirement\n\n        if eDataRequirement == Device_Data_Requirement.ALL_DATA:\n            self.networkDataRequirement = 'n'\n        return True"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nconfigures the border router with a given external route prefix entry", "response": "def configExternalRouter(self, P_Prefix, P_stable, R_Preference=0):\n        \"\"\"configure border router with a given external route prefix entry\n\n        Args:\n            P_Prefix: IPv6 prefix for the route\n            P_Stable: is true if the external route prefix is stable network data\n            R_Preference: a two-bit signed integer indicating Router preference\n                          1: high\n                          0: medium\n                         -1: low\n\n        Returns:\n            True: successful to configure the border router with a given external route prefix\n            False: fail to configure the border router with a given external route prefix\n        \"\"\"\n        print '%s call configExternalRouter' % self.port\n        print P_Prefix\n        stable = ''\n        prefix = self.__convertIp6PrefixStringToIp6Address(str(P_Prefix))\n        try:\n            if R_Preference == 1:\n                prf = 'high'\n            elif R_Preference == 0:\n                prf = 'med'\n            elif R_Preference == -1:\n                prf = 'low'\n            else:\n                pass\n\n            if P_stable:\n                stable += 's'\n                cmd = 'route add %s/64 %s %s' % (prefix, stable, prf)\n            else:\n                cmd = 'route add %s/64 %s' % (prefix, prf)\n            print cmd\n\n            if self.__sendCommand(cmd)[0] == 'Done':\n                # send server data ntf to leader\n                return self.__sendCommand('netdataregister')[0] == 'Done'\n        except Exception, e:\n            ModuleHelper.WriteIntoDebugLogger(\"configExternalRouter() Error: \" + str(e))"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nget neighboring routers information", "response": "def getNeighbouringRouters(self):\n        \"\"\"get neighboring routers information\n\n        Returns:\n            neighboring routers' extended address\n        \"\"\"\n        print '%s call getNeighbouringRouters' % self.port\n        try:\n            routerInfo = []\n            routerList = []\n            routerList = self.__sendCommand('router list')[0].split()\n            print routerList\n\n            if 'Done' in routerList:\n                print 'no neighbouring routers'\n                return None\n\n            for index in routerList:\n                router = []\n                cmd = 'router %s' % index\n                router = self.__sendCommand(cmd)\n\n                for line in router:\n                    if 'Done' in line:\n                        break\n                    #elif 'Rloc' in line:\n                    #    rloc16 = line.split()[1]\n                    elif 'Ext Addr' in line:\n                        eui = line.split()[2]\n                        routerInfo.append(int(eui, 16))\n                    #elif 'LQI In' in line:\n                    #    lqi_in = line.split()[1]\n                    #elif 'LQI Out' in line:\n                    #    lqi_out = line.split()[1]\n                    else:\n                        pass\n\n            print routerInfo\n            return routerInfo\n        except Exception, e:\n            ModuleHelper.WriteIntoDebugLogger(\"getNeighbouringDevice() Error: \" + str(e))"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nget all children information", "response": "def getChildrenInfo(self):\n        \"\"\"get all children information\n\n        Returns:\n            children's extended address\n        \"\"\"\n        print '%s call getChildrenInfo' % self.port\n        try:\n            childrenInfoAll = []\n            childrenInfo = {'EUI': 0, 'Rloc16': 0, 'MLEID': ''}\n            childrenList = self.__sendCommand('child list')[0].split()\n            print childrenList\n\n            if 'Done' in childrenList:\n                print 'no children'\n                return None\n\n            for index in childrenList:\n                cmd = 'child %s' % index\n                child = []\n                child = self.__sendCommand(cmd)\n\n                for line in child:\n                    if 'Done' in line:\n                        break\n                    elif 'Rloc' in line:\n                        rloc16 = line.split()[1]\n                    elif 'Ext Addr' in line:\n                        eui = line.split()[2]\n                    #elif 'Child ID' in line:\n                    #    child_id = line.split()[2]\n                    #elif 'Mode' in line:\n                    #    mode = line.split()[1]\n                    else:\n                        pass\n\n                childrenInfo['EUI'] = int(eui, 16)\n                childrenInfo['Rloc16'] = int(rloc16, 16)\n                #children_info['MLEID'] = self.getMLEID()\n\n                childrenInfoAll.append(childrenInfo['EUI'])\n                #childrenInfoAll.append(childrenInfo)\n\n            print childrenInfoAll\n            return childrenInfoAll\n        except Exception, e:\n            ModuleHelper.WriteIntoDebugLogger(\"getChildrenInfo() Error: \" + str(e))"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ngets the neighboring devices extended address to compute the DUT extended address automatically", "response": "def getNeighbouringDevices(self):\n        \"\"\"gets the neighboring devices' extended address to compute the DUT\n           extended address automatically\n\n        Returns:\n            A list including extended address of neighboring routers, parent\n            as well as children\n        \"\"\"\n        print '%s call getNeighbouringDevices' % self.port\n        neighbourList = []\n\n        # get parent info\n        parentAddr = self.getParentAddress()\n        if parentAddr != 0:\n            neighbourList.append(parentAddr)\n\n        # get ED/SED children info\n        childNeighbours = self.getChildrenInfo()\n        if childNeighbours != None and len(childNeighbours) > 0:\n            for entry in childNeighbours:\n                neighbourList.append(entry)\n\n        # get neighboring routers info\n        routerNeighbours = self.getNeighbouringRouters()\n        if routerNeighbours != None and len(routerNeighbours) > 0:\n            for entry in routerNeighbours:\n                neighbourList.append(entry)\n\n        print neighbourList\n        return neighbourList"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ngets the global unicast IPv6 address of Thread device", "response": "def getGUA(self, filterByPrefix=None):\n        \"\"\"get expected global unicast IPv6 address of Thread device\n\n        Args:\n            filterByPrefix: a given expected global IPv6 prefix to be matched\n\n        Returns:\n            a global IPv6 address\n        \"\"\"\n        print '%s call getGUA' % self.port\n        print filterByPrefix\n        globalAddrs = []\n        try:\n            # get global addrs set if multiple\n            globalAddrs = self.getGlobal()\n\n            if filterByPrefix is None:\n                return globalAddrs[0]\n            else:\n                for line in globalAddrs:\n                    fullIp = ModuleHelper.GetFullIpv6Address(line)\n                    if fullIp.startswith(filterByPrefix):\n                        return fullIp\n                print 'no global address matched'\n                return str(globalAddrs[0])\n        except Exception, e:\n            ModuleHelper.WriteIntoDebugLogger(\"getGUA() Error: \" + str(e))"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nset mesh local prefix", "response": "def setMLPrefix(self, sMeshLocalPrefix):\n        \"\"\"set mesh local prefix\"\"\"\n        print '%s call setMLPrefix' % self.port\n        try:\n            cmd = 'dataset meshlocalprefix %s' % sMeshLocalPrefix\n            self.hasActiveDatasetToCommit = True\n            return self.__sendCommand(cmd)[0] == 'Done'\n        except Exception, e:\n            ModuleHelper.WriteIntoDebugLogger(\"setMLPrefix() Error: \" + str(e))"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nforcing to set a slaac address to Thread interface", "response": "def forceSetSlaac(self, slaacAddress):\n        \"\"\"force to set a slaac IPv6 address to Thread interface\n\n        Args:\n            slaacAddress: a slaac IPv6 address to be set\n\n        Returns:\n            True: successful to set slaac address to Thread interface\n            False: fail to set slaac address to Thread interface\n        \"\"\"\n        print '%s call forceSetSlaac' % self.port\n        print slaacAddress\n        try:\n            cmd = 'ipaddr add %s' % str(slaacAddress)\n            print cmd\n            return self.__sendCommand(cmd)[0] == 'Done'\n        except Exception, e:\n            ModuleHelper.WriteIntoDebugLogger(\"forceSetSlaac() Error: \" + str(e))"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nstarting Collapsed Commissioner Returns: True: successful to start Commissioner False: fail to start Commissioner", "response": "def startCollapsedCommissioner(self):\n        \"\"\"start Collapsed Commissioner\n\n        Returns:\n            True: successful to start Commissioner\n            False: fail to start Commissioner\n        \"\"\"\n        print '%s call startCollapsedCommissioner' % self.port\n        if self.__startOpenThread():\n            time.sleep(20)\n            cmd = 'commissioner start'\n            print cmd\n            if self.__sendCommand(cmd)[0] == 'Done':\n                self.isActiveCommissioner = True\n                time.sleep(20) # time for petition process\n                return True\n        return False"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef scanJoiner(self, xEUI='*', strPSKd='threadjpaketest'):\n        print '%s call scanJoiner' % self.port\n\n        # long timeout value to avoid automatic joiner removal (in seconds)\n        timeout = 500\n\n        if not isinstance(xEUI, str):\n            eui64 = self.__convertLongToString(xEUI)\n\n            # prepend 0 at the beginning\n            if len(eui64) < 16:\n                eui64 = eui64.zfill(16)\n                print eui64\n        else:\n            eui64 = xEUI\n\n        cmd = 'commissioner joiner add %s %s %s' % (eui64, strPSKd, str(timeout))\n        print cmd\n        if self.__sendCommand(cmd)[0] == 'Done':\n            if self.logThreadStatus == self.logStatus['stop']:\n                self.logThread = ThreadRunner.run(target=self.__readCommissioningLogs, args=(120,))\n            return True\n        else:\n            return False", "response": "add a new Joiner s steering data to the current Thread"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nstarting joiner s PSKd", "response": "def joinCommissioned(self, strPSKd='threadjpaketest', waitTime=20):\n        \"\"\"start joiner\n\n        Args:\n            strPSKd: Joiner's PSKd\n\n        Returns:\n            True: successful to start joiner\n            False: fail to start joiner\n        \"\"\"\n        print '%s call joinCommissioned' % self.port\n        self.__sendCommand('ifconfig up')\n        cmd = 'joiner start %s %s' %(strPSKd, self.provisioningUrl)\n        print cmd\n        if self.__sendCommand(cmd)[0] == \"Done\":\n            maxDuration = 150 # seconds\n            self.joinCommissionedStatus = self.joinStatus['ongoing']\n\n            if self.logThreadStatus == self.logStatus['stop']:\n                self.logThread = ThreadRunner.run(target=self.__readCommissioningLogs, args=(maxDuration,))\n\n            t_end = time.time() + maxDuration\n            while time.time() < t_end:\n                if self.joinCommissionedStatus == self.joinStatus['succeed']:\n                    break\n                elif self.joinCommissionedStatus == self.joinStatus['failed']:\n                    return False\n\n                time.sleep(1)\n\n            self.__sendCommand('thread start')\n            time.sleep(30)\n            return True\n        else:\n            return False"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nget Commissioning logs in the logThread", "response": "def getCommissioningLogs(self):\n        \"\"\"get Commissioning logs\n\n        Returns:\n           Commissioning logs\n        \"\"\"\n        rawLogs = self.logThread.get()\n        ProcessedLogs = []\n        payload = []\n        while not rawLogs.empty():\n            rawLogEach = rawLogs.get()\n            print rawLogEach\n            if \"[THCI]\" not in rawLogEach:\n                continue\n\n            EncryptedPacket = PlatformDiagnosticPacket()\n            infoList = rawLogEach.split('[THCI]')[1].split(']')[0].split('|')\n            for eachInfo in infoList:\n                print eachInfo\n                info = eachInfo.split(\"=\")\n                infoType = info[0].strip()\n                infoValue = info[1].strip()\n                if \"direction\" in infoType:\n                    EncryptedPacket.Direction = PlatformDiagnosticPacket_Direction.IN if 'recv' in infoValue \\\n                        else PlatformDiagnosticPacket_Direction.OUT if 'send' in infoValue \\\n                        else PlatformDiagnosticPacket_Direction.UNKNOWN\n                elif \"type\" in infoType:\n                    EncryptedPacket.Type = PlatformDiagnosticPacket_Type.JOIN_FIN_req if 'JOIN_FIN.req' in infoValue \\\n                        else PlatformDiagnosticPacket_Type.JOIN_FIN_rsp if 'JOIN_FIN.rsp' in infoValue \\\n                        else PlatformDiagnosticPacket_Type.JOIN_ENT_req if 'JOIN_ENT.ntf' in infoValue \\\n                        else PlatformDiagnosticPacket_Type.JOIN_ENT_rsp if 'JOIN_ENT.rsp' in infoValue \\\n                        else PlatformDiagnosticPacket_Type.UNKNOWN\n                elif \"len\" in infoType:\n                    bytesInEachLine = 16\n                    EncryptedPacket.TLVsLength = int(infoValue)\n                    payloadLineCount = (int(infoValue) + bytesInEachLine - 1)/bytesInEachLine\n                    while payloadLineCount > 0:\n                        payloadLineCount = payloadLineCount - 1\n                        payloadLine = rawLogs.get()\n                        payloadSplit = payloadLine.split('|')\n                        for block in range(1, 3):\n                            payloadBlock = payloadSplit[block]\n                            payloadValues = payloadBlock.split(' ')\n                            for num in range(1, 9):\n                                if \"..\" not in payloadValues[num]:\n                                    payload.append(int(payloadValues[num], 16))\n\n                    EncryptedPacket.TLVs = PlatformPackets.read(EncryptedPacket.Type, payload) if payload != [] else []\n\n            ProcessedLogs.append(EncryptedPacket)\n        return ProcessedLogs"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nsend MGMT_ED_SCAN message to a given destination", "response": "def MGMT_ED_SCAN(self, sAddr, xCommissionerSessionId, listChannelMask, xCount, xPeriod, xScanDuration):\n        \"\"\"send MGMT_ED_SCAN message to a given destinaition.\n\n        Args:\n            sAddr: IPv6 destination address for this message\n            xCommissionerSessionId: commissioner session id\n            listChannelMask: a channel array to indicate which channels to be scaned\n            xCount: number of IEEE 802.15.4 ED Scans (milliseconds)\n            xPeriod: Period between successive IEEE802.15.4 ED Scans (milliseconds)\n            xScanDuration: IEEE 802.15.4 ScanDuration to use when performing an IEEE 802.15.4 ED Scan (milliseconds)\n\n        Returns:\n            True: successful to send MGMT_ED_SCAN message.\n            False: fail to send MGMT_ED_SCAN message\n        \"\"\"\n        print '%s call MGMT_ED_SCAN' % self.port\n        channelMask = ''\n        channelMask = '0x' + self.__convertLongToString(self.__convertChannelMask(listChannelMask))\n        try:\n            cmd = 'commissioner energy %s %s %s %s %s' % (channelMask, xCount, xPeriod, xScanDuration, sAddr)\n            print cmd\n            return self.__sendCommand(cmd) == 'Done'\n        except Exception, e:\n            ModuleHelper.writeintodebuglogger(\"MGMT_ED_SCAN() error: \" + str(e))"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nsend MGMT_PANID_QUERY message to a given destination", "response": "def MGMT_PANID_QUERY(self, sAddr, xCommissionerSessionId, listChannelMask, xPanId):\n        \"\"\"send MGMT_PANID_QUERY message to a given destination\n\n        Args:\n            xPanId: a given PAN ID to check the conflicts\n\n        Returns:\n            True: successful to send MGMT_PANID_QUERY message.\n            False: fail to send MGMT_PANID_QUERY message.\n        \"\"\"\n        print '%s call MGMT_PANID_QUERY' % self.port\n        panid = ''\n        channelMask = ''\n        channelMask = '0x' + self.__convertLongToString(self.__convertChannelMask(listChannelMask))\n\n        if not isinstance(xPanId, str):\n            panid = str(hex(xPanId))\n\n        try:\n            cmd = 'commissioner panid %s %s %s' % (panid, channelMask, sAddr)\n            print cmd\n            return self.__sendCommand(cmd) == 'Done'\n        except Exception, e:\n            ModuleHelper.writeintodebuglogger(\"MGMT_PANID_QUERY() error: \" + str(e))"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef MGMT_ACTIVE_SET(self, sAddr='', xCommissioningSessionId=None, listActiveTimestamp=None, listChannelMask=None, xExtendedPanId=None,\n                        sNetworkName=None, sPSKc=None, listSecurityPolicy=None, xChannel=None, sMeshLocalPrefix=None, xMasterKey=None,\n                        xPanId=None, xTmfPort=None, xSteeringData=None, xBorderRouterLocator=None, BogusTLV=None, xDelayTimer=None):\n        \"\"\"send MGMT_ACTIVE_SET command\n\n        Returns:\n            True: successful to send MGMT_ACTIVE_SET\n            False: fail to send MGMT_ACTIVE_SET\n        \"\"\"\n        print '%s call MGMT_ACTIVE_SET' % self.port\n        try:\n            cmd = 'dataset mgmtsetcommand active'\n\n            if listActiveTimestamp != None:\n                cmd += ' activetimestamp '\n                cmd += str(listActiveTimestamp[0])\n\n            if xExtendedPanId != None:\n                cmd += ' extpanid '\n                xpanid = self.__convertLongToString(xExtendedPanId)\n\n                if len(xpanid) < 16:\n                    xpanid = xpanid.zfill(16)\n\n                cmd += xpanid\n\n            if sNetworkName != None:\n                cmd += ' networkname '\n                cmd += str(sNetworkName)\n\n            if xChannel != None:\n                cmd += ' channel '\n                cmd += str(xChannel)\n\n            if sMeshLocalPrefix != None:\n                cmd += ' localprefix '\n                cmd += str(sMeshLocalPrefix)\n\n            if xMasterKey != None:\n                cmd += ' masterkey '\n                key = self.__convertLongToString(xMasterKey)\n\n                if len(key) < 32:\n                    key = key.zfill(32)\n\n                cmd += key\n\n            if xPanId != None:\n                cmd += ' panid '\n                cmd += str(xPanId)\n\n            if listChannelMask != None:\n                cmd += ' channelmask '\n                cmd += '0x' + self.__convertLongToString(self.__convertChannelMask(listChannelMask))\n\n            if sPSKc != None or listSecurityPolicy != None or \\\n               xCommissioningSessionId != None or xTmfPort != None or xSteeringData != None or xBorderRouterLocator != None or \\\n               BogusTLV != None:\n                cmd += ' binary '\n\n            if sPSKc != None:\n                cmd += '0410'\n                stretchedPskc = Thread_PBKDF2.get(sPSKc,ModuleHelper.Default_XpanId,ModuleHelper.Default_NwkName)\n                pskc = hex(stretchedPskc).rstrip('L').lstrip('0x')\n\n                if len(pskc) < 32:\n                    pskc = pskc.zfill(32)\n\n                cmd += pskc\n\n            if listSecurityPolicy != None:\n                cmd += '0c03'\n\n                rotationTime = 0\n                policyBits = 0\n\n                # previous passing way listSecurityPolicy=[True, True, 3600, False, False, True]\n                if (len(listSecurityPolicy) == 6):\n                    rotationTime = listSecurityPolicy[2]\n\n                    # the last three reserved bits must be 1\n                    policyBits = 0b00000111\n\n                    if listSecurityPolicy[0]:\n                        policyBits = policyBits | 0b10000000\n                    if listSecurityPolicy[1]:\n                        policyBits = policyBits | 0b01000000\n                    if listSecurityPolicy[3]:\n                        policyBits = policyBits | 0b00100000\n                    if listSecurityPolicy[4]:\n                        policyBits = policyBits | 0b00010000\n                    if listSecurityPolicy[5]:\n                        policyBits = policyBits | 0b00001000\n                else:\n                    # new passing way listSecurityPolicy=[3600, 0b11001111]\n                    rotationTime = listSecurityPolicy[0]\n                    policyBits = listSecurityPolicy[1]\n\n                policy = str(hex(rotationTime))[2:]\n\n                if len(policy) < 4:\n                    policy = policy.zfill(4)\n\n                cmd += policy\n\n                cmd += str(hex(policyBits))[2:]\n\n            if xCommissioningSessionId != None:\n                cmd += '0b02'\n                sessionid = str(hex(xCommissioningSessionId))[2:]\n\n                if len(sessionid) < 4:\n                    sessionid = sessionid.zfill(4)\n\n                cmd += sessionid\n\n            if xBorderRouterLocator != None:\n                cmd += '0902'\n                locator = str(hex(xBorderRouterLocator))[2:]\n\n                if len(locator) < 4:\n                    locator = locator.zfill(4)\n\n                cmd += locator\n\n            if xSteeringData != None:\n                steeringData = self.__convertLongToString(xSteeringData)\n                cmd += '08' + str(len(steeringData)/2).zfill(2)\n                cmd += steeringData\n\n            if BogusTLV != None:\n                cmd += \"8202aa55\"\n\n            print cmd\n\n            return self.__sendCommand(cmd)[0] == 'Done'\n\n        except Exception, e:\n            ModuleHelper.WriteIntoDebugLogger(\"MGMT_ACTIVE_SET() Error: \" + str(e))", "response": "send MGMT_ACTIVE_SET command to set the state of the active MESH in the specified session"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nsending MGMT_PENDING_SET command to the device", "response": "def MGMT_PENDING_SET(self, sAddr='', xCommissionerSessionId=None, listPendingTimestamp=None, listActiveTimestamp=None, xDelayTimer=None,\n                         xChannel=None, xPanId=None, xMasterKey=None, sMeshLocalPrefix=None, sNetworkName=None):\n        \"\"\"send MGMT_PENDING_SET command\n\n        Returns:\n            True: successful to send MGMT_PENDING_SET\n            False: fail to send MGMT_PENDING_SET\n        \"\"\"\n        print '%s call MGMT_PENDING_SET' % self.port\n        try:\n            cmd = 'dataset mgmtsetcommand pending'\n\n            if listPendingTimestamp != None:\n                cmd += ' pendingtimestamp '\n                cmd += str(listPendingTimestamp[0])\n\n            if listActiveTimestamp != None:\n                cmd += ' activetimestamp '\n                cmd += str(listActiveTimestamp[0])\n\n            if xDelayTimer != None:\n                cmd += ' delaytimer '\n                cmd += str(xDelayTimer)\n                #cmd += ' delaytimer 3000000'\n\n            if xChannel != None:\n                cmd += ' channel '\n                cmd += str(xChannel)\n\n            if xPanId != None:\n                cmd += ' panid '\n                cmd += str(xPanId)\n\n            if xMasterKey != None:\n                cmd += ' masterkey '\n                key = self.__convertLongToString(xMasterKey)\n\n                if len(key) < 32:\n                    key = key.zfill(32)\n\n                cmd += key\n\n            if sMeshLocalPrefix != None:\n                cmd += ' localprefix '\n                cmd += str(sMeshLocalPrefix)\n\n            if sNetworkName != None:\n                cmd += ' networkname '\n                cmd += str(sNetworkName)\n\n            if xCommissionerSessionId != None:\n                cmd += ' binary '\n                cmd += '0b02'\n                sessionid = str(hex(xCommissionerSessionId))[2:]\n\n                if len(sessionid) < 4:\n                    sessionid = sessionid.zfill(4)\n\n                cmd += sessionid\n\n            print cmd\n\n            return self.__sendCommand(cmd)[0] == 'Done'\n\n        except Exception, e:\n            ModuleHelper.WriteIntoDebugLogger(\"MGMT_PENDING_SET() Error: \" + str(e))"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef MGMT_COMM_GET(self, Addr='ff02::1', TLVs=[]):\n        print '%s call MGMT_COMM_GET' % self.port\n        try:\n            cmd = 'commissioner mgmtget'\n\n            if len(TLVs) != 0:\n                tlvs = \"\".join(hex(tlv).lstrip(\"0x\").zfill(2) for tlv in TLVs)\n                cmd += ' binary '\n                cmd += tlvs\n\n            print cmd\n\n            return self.__sendCommand(cmd)[0] == 'Done'\n\n        except Exception, e:\n            ModuleHelper.WriteIntoDebugLogger(\"MGMT_COMM_GET() Error: \" + str(e))", "response": "send MGMT_COMM_GET command\n\n        Returns:\n            True: successful to send MGMT_COMM_GET\n            False: fail to send MGMT_COMM_GET"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef setUdpJoinerPort(self, portNumber):\n        print '%s call setUdpJoinerPort' % self.port\n        cmd = 'joinerport %d' % portNumber\n        print cmd\n        return self.__sendCommand(cmd)[0] == 'Done'", "response": "set Joiner UDP Port\n           "}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nstops commissioner Returns: True: successful to stop commissioner False: fail to stop commissioner", "response": "def commissionerUnregister(self):\n        \"\"\"stop commissioner\n\n        Returns:\n            True: successful to stop commissioner\n            False: fail to stop commissioner\n        \"\"\"\n        print '%s call commissionerUnregister' % self.port\n        cmd = 'commissioner stop'\n        print cmd\n        return self.__sendCommand(cmd)[0] == 'Done'"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nforce update to router as if there is child id request", "response": "def updateRouterStatus(self):\n        \"\"\"force update to router as if there is child id request\"\"\"\n        print '%s call updateRouterStatus' % self.port\n        cmd = 'state'\n        while True:\n            state = self.__sendCommand(cmd)[0]\n            if state == 'detached':\n                continue\n            elif state == 'child':\n                break\n            else:\n                return False\n\n        cmd = 'state router'\n        return self.__sendCommand(cmd)[0] == 'Done'"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nfinding the expected string within times.", "response": "def _expect(self, expected, times=50):\n        \"\"\"Find the `expected` line within `times` trials.\n\n        Args:\n            expected    str: the expected string\n            times       int: number of trials\n        \"\"\"\n        logger.debug('[%s] Expecting [%s]', self.port, expected)\n        retry_times = 10\n        while times:\n            if not retry_times:\n                break\n\n            line = self._readline()\n\n            if line == expected:\n                return\n\n            if not line:\n                retry_times -= 1\n                time.sleep(0.1)\n\n            times -= 1\n\n        raise Exception('failed to find expected string[%s]' % expected)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _readline(self):\n        if len(self.lines) > 1:\n            return self.lines.pop(0)\n\n        tail = ''\n        if len(self.lines):\n            tail = self.lines.pop()\n\n        try:\n            tail += self._read()\n        except socket.error:\n            logging.exception('No new data')\n            time.sleep(0.1)\n\n        self.lines += linesepx.split(tail)\n        if len(self.lines) > 1:\n            return self.lines.pop(0)", "response": "Read exactly one line from the device nonblocking."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nsending exactly one line to the device", "response": "def _sendline(self, line):\n        \"\"\"Send exactly one line to the device\n\n        Args:\n            line str: data send to device\n        \"\"\"\n        self.lines = []\n        try:\n            self._read()\n        except socket.error:\n            logging.debug('Nothing cleared')\n\n        logger.debug('sending [%s]', line)\n        self._write(line + '\\r\\n')\n\n        # wait for write to complete\n        time.sleep(0.5)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _req(self, req):\n        logger.debug('DUT> %s', req)\n        self._log and self.pause()\n        times = 3\n        res = None\n\n        while times:\n            times = times - 1\n            try:\n                self._sendline(req)\n                self._expect(req)\n\n                line = None\n                res = []\n\n                while True:\n                    line = self._readline()\n                    logger.debug('Got line %s', line)\n\n                    if line == 'Done':\n                        break\n\n                    if line:\n                        res.append(line)\n                break\n\n            except:\n                logger.exception('Failed to send command')\n                self.close()\n                self._init()\n\n        self._log and self.resume()\n        return res", "response": "Send command and wait for response."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef reset(self):\n        logger.debug('DUT> reset')\n        self._log and self.pause()\n        self._sendline('reset')\n        self._read()\n        self._log and self.resume()", "response": "Reset the openthread device not equivalent to stop and start"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef add_prefix(self, prefix, flags, prf):\n        self._req('prefix add %s %s %s' % (prefix, flags, prf))\n        time.sleep(1)\n        self._req('netdataregister')", "response": "Add a network prefix."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ninitializing the telnet connection", "response": "def _init(self):\n        \"\"\"Initialize the telnet connection\n        \"\"\"\n        self.tn = telnetlib.Telnet(self.ip, self.port)\n        self.tn.read_until('User Name')\n        self.tn.write('apc\\r\\n')\n        self.tn.read_until('Password')\n        self.tn.write('apc\\r\\n')\n        self.until_done()"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef open(self, **params):\n        logger.info('opening telnet')\n        self.port = params['port']\n        self.ip = params['ip']\n        self.tn = None\n        self._init()", "response": "Open a new telnet connection"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nwaiting until the regex encountered", "response": "def until(self, regex):\n        \"\"\"Wait until the regex encountered\n        \"\"\"\n        logger.debug('waiting for %s', regex)\n        r = re.compile(regex, re.M)\n        self.tn.expect([r])"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef __sendCommand(self, cmd):\n        logging.info('%s: sendCommand[%s]', self.port, cmd)\n        if self.logThreadStatus == self.logStatus['running']:\n            self.logThreadStatus = self.logStatus['pauseReq']\n            while self.logThreadStatus != self.logStatus['paused'] and self.logThreadStatus != self.logStatus['stop']:\n                pass\n\n        ssh_stdin = None\n        ssh_stdout = None\n        ssh_stderr = None\n        try:\n            # command retransmit times\n            retry_times = 3\n            while retry_times > 0:\n                retry_times -= 1\n                try:\n                    if self._is_net:\n                        ssh_stdin, ssh_stdout, ssh_stderr = self.handle.exec_command(cmd)\n                    else:\n                        self._sendline(cmd)\n                        self._expect(cmd)\n                except Exception as e:\n                    logging.exception('%s: failed to send command[%s]: %s', self.port, cmd, str(e))\n                    if retry_times == 0:\n                        raise\n                else:\n                    break\n\n            line = None\n            response = []\n            retry_times = 20\n            stdout_lines = []\n            stderr_lines = []\n            if self._is_net:\n                stdout_lines = ssh_stdout.readlines()\n                stderr_lines = ssh_stderr.readlines()\n                if stderr_lines:\n                    for stderr_line in stderr_lines:\n                        if re.search(r'Not\\s+Found|failed\\s+with\\s+error', stderr_line.strip(), re.M | re.I):\n                            print \"Command failed:\" + stderr_line\n                            return 'Fail'\n                        print \"Got line: \" + stderr_line\n                        logging.info('%s: the read line is[%s]', self.port, stderr_line)\n                        response.append(str(stderr_line.strip()))\n                elif stdout_lines:\n                    for stdout_line in stdout_lines:\n                        logging.info('%s: the read line is[%s]', self.port, stdout_line)\n                        if re.search(r'Not\\s+Found|failed\\s+with\\s+error', stdout_line.strip(), re.M | re.I):\n                            print \"Command failed\"\n                            return 'Fail'\n                        print \"Got line: \" + stdout_line\n                        logging.info('%s: send command[%s] done!', self.port, cmd)\n                        response.append(str(stdout_line.strip()))\n                response.append(WPAN_CARRIER_PROMPT)\n                return response\n            else:\n                while retry_times > 0:\n                    line = self._readline()\n                    print \"read line: %s\" % line\n                    logging.info('%s: the read line is[%s]', self.port, line)\n                    if line:\n                        response.append(line)\n                        if re.match(WPAN_CARRIER_PROMPT, line):\n                            break\n                        elif re.search(r'Not\\s+Found|failed\\s+with\\s+error', line, re.M | re.I):\n                            print \"Command failed\"\n                            return 'Fail'\n\n                    retry_times -= 1\n                    time.sleep(0.1)\n\n                if retry_times == 0:\n                    raise Exception('%s: failed to find end of response' % self.port)\n                logging.info('%s: send command[%s] done!', self.port, cmd)\n                return response\n        except Exception, e:\n            ModuleHelper.WriteIntoDebugLogger('sendCommand() Error: ' + str(e))\n            raise", "response": "send specific command to the reference unit and parse the result"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef __stripValue(self, value):\n        if isinstance(value, str):\n            if ( value[0] == '\"' and value[-1] == '\"' ) or ( value[0] == '[' and value[-1] == ']' ):\n                return value[1:-1]\n        return value", "response": "strip the special characters in the value\nCOOKIENAME"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef __getIp6Address(self, addressType):\n        addrType = ['link local', 'global', 'rloc', 'mesh EID']\n        addrs = []\n        globalAddr = []\n        linkLocal64Addr = ''\n        rlocAddr = ''\n        meshEIDAddr = ''\n\n        addrs = self.__sendCommand(WPANCTL_CMD + 'getprop -v IPv6:AllAddresses')\n        for ip6AddrItem in addrs:\n            if re.match('\\[|\\]', ip6AddrItem):\n                continue\n            if re.match(WPAN_CARRIER_PROMPT, ip6AddrItem, re.M|re.I):\n                break\n            ip6AddrItem = ip6AddrItem.strip()\n            ip6Addr = self.__stripValue(ip6AddrItem).split(' ')[0]\n            ip6AddrPrefix = ip6Addr.split(':')[0]\n            if ip6AddrPrefix == 'fe80':\n                # link local address\n                if ip6Addr.split(':')[4] != '0':\n                    linkLocal64Addr = ip6Addr\n            elif ip6Addr.startswith(self.meshLocalPrefix):\n                # mesh local address\n                if ip6Addr.split(':')[4] == '0':\n                    # rloc\n                    rlocAddr = ip6Addr\n                else:\n                    # mesh EID\n                    meshEIDAddr = ip6Addr\n                    print 'meshEIDAddr:' + meshEIDAddr\n            else:\n                # global ipv6 address\n                if ip6Addr:\n                    print 'globalAddr: ' + ip6Addr\n                    globalAddr.append(ip6Addr)\n                else:\n                    pass\n\n        if addressType == addrType[0]:\n            return linkLocal64Addr\n        elif addressType == addrType[1]:\n            return globalAddr\n        elif addressType == addrType[2]:\n            return rlocAddr\n        elif addressType == addrType[3]:\n            return meshEIDAddr\n        else:\n            pass", "response": "get specific type of IPv6 address configured on OpenThread_WpanCtl\n           "}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nset the thread device mode", "response": "def __setDeviceMode(self, mode):\n        \"\"\"set thread device mode:\n\n        Args:\n           mode: thread device mode. 15=rsdn, 13=rsn, 4=s\n           r: rx-on-when-idle\n           s: secure IEEE 802.15.4 data request\n           d: full thread device\n           n: full network data\n\n        Returns:\n           True: successful to set the device mode\n           False: fail to set the device mode\n        \"\"\"\n        print 'call __setDeviceMode'\n\n        try:\n            cmd = WPANCTL_CMD + 'setprop Thread:DeviceMode %d' % mode\n            return self.__sendCommand(cmd)[0] != 'Fail'\n        except Exception, e:\n            ModuleHelper.WriteIntoDebugLogger('setDeviceMode() Error: ' + str(e))"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef __setAddressfilterMode(self, mode):\n        print 'call setAddressFilterMode() ' +  mode\n        try:\n            if re.match('list', mode, re.M|re.I):\n                cmd = WPANCTL_CMD + 'setprop MAC:' + mode + ':Enabled 1'\n            elif mode == 'disabled':\n                cmd = WPANCTL_CMD + 'setprop MAC:' + mode + ':Enabled 0'\n            else:\n                print 'no such option'\n                return False\n            if self.__sendCommand(cmd)[0] != 'Fail':\n                return True\n            return False\n        except Exception, e:\n            ModuleHelper.WriteIntoDebugLogger('__setAddressFilterMode() Error: ' + str(e))", "response": "set the address filter mode for the current user"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef __startOpenThreadWpan(self):\n        print 'call startOpenThreadWpan'\n        try:\n\n            # restore whitelist/blacklist address filter mode if rejoin after reset\n            if self.isPowerDown:\n                if self._addressfilterMode == 'whitelist':\n                    if self.__setAddressfilterMode('whitelist'):\n                        for addr in self._addressfilterSet:\n                            self.addAllowMAC(addr)\n                elif self._addressfilterMode == 'blacklist':\n                    if self.__setAddressfilterMode('blacklist'):\n                        for addr in self._addressfilterSet:\n                            self.addBlockedMAC(addr)\n            time.sleep(1)\n            if ModuleHelper.LeaderDutChannelFound:\n                self.channel = ModuleHelper.Default_Channel\n            nodeType = 'router'\n            startType = 'join'\n            if self.deviceRole == Thread_Device_Role.Leader:\n                nodeType = 'router'\n                startType = 'form'\n            elif self.deviceRole == Thread_Device_Role.Router:\n                nodeType = 'router'\n            elif self.deviceRole == Thread_Device_Role.SED:\n                nodeType = 'sed'\n            elif self.deviceRole == Thread_Device_Role.EndDevice:\n                nodeType = 'end'\n            elif self.deviceRole == Thread_Device_Role.REED:\n                nodeType = 'router'\n            elif self.deviceRole == Thread_Device_Role.EndDevice_FED:\n                nodeType = 'router'\n            elif self.deviceRole == Thread_Device_Role.EndDevice_MED:\n                nodeType = 'end'\n            else:\n                pass\n\n            if self.deviceRole in [Thread_Device_Role.Leader, Thread_Device_Role.Router, Thread_Device_Role.REED]:\n                self.__setRouterSelectionJitter(1)\n\n            if startType == 'form':\n                startCmd = WPANCTL_CMD + '%s %s -c %s -T %s ' % (startType, self.networkName, str(self.channel), nodeType)\n            else:\n                startCmd = WPANCTL_CMD + '%s %s -p %s -c %s -T %s ' % (startType, self.networkName, str(hex(self.panId)), str(self.channel), nodeType)\n            if self.__sendCommand(startCmd)[0] != 'Fail':\n                if self.__isOpenThreadWpanRunning():\n                    self.isPowerDown = False\n                    if self.hasActiveDatasetToCommit:\n                        if self.__sendCommand(WPANCTL_CMD + 'setprop Dataset:Command SetActive')[0] == 'Fail':\n                            raise Exception('failed to commit active dataset')\n                        else:\n                            self.hasActiveDatasetToCommit = False\n\n                    return True\n            else:\n                return False\n        except Exception, e:\n            ModuleHelper.WriteIntoDebugLogger('startOpenThreadWpan() Error: ' + str(e))", "response": "start OpenThreadWpan with the current Thread_Device_Role"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nstop OpenThreadWpan Returns: True: successfully stop OpenThreadWpan False: failed to stop OpenThreadWpan", "response": "def __stopOpenThreadWpan(self):\n        \"\"\"stop OpenThreadWpan\n\n        Returns:\n            True: successfully stop OpenThreadWpan\n            False: failed to stop OpenThreadWpan\n        \"\"\"\n        print 'call stopOpenThreadWpan'\n        try:\n            if self.__sendCommand(WPANCTL_CMD + 'leave')[0] != 'Fail' and self.__sendCommand(WPANCTL_CMD + 'dataset erase')[0] != 'Fail':\n                return True\n            else:\n                return False\n        except Exception, e:\n            ModuleHelper.WriteIntoDebugLogger('stopOpenThreadWpan() Error: ' + str(e))"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nchecking whether or not the OpenThreadWpan is running", "response": "def __isOpenThreadWpanRunning(self):\n        \"\"\"check whether or not OpenThreadWpan is running\n\n        Returns:\n            True: OpenThreadWpan is running\n            False: OpenThreadWpan is not running\n        \"\"\"\n        print 'call __isOpenThreadWpanRunning'\n        if self.__stripValue(self.__sendCommand(WPANCTL_CMD + 'getprop -v NCP:State')[0]) == 'associated':\n            print '*****OpenThreadWpan is running'\n            return True\n        else:\n            print '*****Wrong OpenThreadWpan state'\n            return False"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef __convertRlocToRouterId(self, xRloc16):\n        routerList = []\n        routerList = self.__sendCommand(WPANCTL_CMD + 'getprop -v Thread:RouterTable')\n        print routerList\n        print xRloc16\n\n        for line in routerList:\n            if re.match('\\[|\\]', line):\n                continue\n            if re.match(WPAN_CARRIER_PROMPT, line, re.M|re.I):\n                break\n            router = []\n            router = self.__stripValue(line).split(',')\n\n            for item in router:\n                if 'RouterId' in item:\n                    routerid = item.split(':')[1]\n                elif 'RLOC16' in line:\n                    rloc16 = line.split(':')[1]\n                else:\n                    pass\n\n            # process input rloc16\n            if isinstance(xRloc16, str):\n                rloc16 = '0x' + rloc16\n                if rloc16 == xRloc16:\n                    return routerid\n            elif isinstance(xRloc16, int):\n                if int(rloc16, 16) == xRloc16:\n                    return routerid\n            else:\n                pass\n\n        return None", "response": "converts RLOC16 to RouterId"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef __ChannelMaskListToStr(self, channelList):\n        chan_mask_range = ''\n        if isinstance(channelList, list):\n            if len(channelList):\n                chan_mask_range = \",\".join(str(chan) for chan in channelList)\n            else:\n                print 'empty list'\n        else:\n            print 'not a valid channel list:', channelList\n\n        return chan_mask_range", "response": "convert a list of channels to a comma separated string"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef intialize(self):\n        print '%s call intialize' % self.port\n        try:\n            # init serial port\n            self.deviceConnected = False\n            self._connect()\n\n            if self.deviceConnected:\n                self.UIStatusMsg = self.getVersionNumber()\n                if self.firmwarePrefix not in self.UIStatusMsg:\n                    self.deviceConnected = False\n                    self.UIStatusMsg = 'Firmware Not Matching Expecting ' + self.firmwarePrefix + ' Now is ' + self.UIStatusMsg\n                    raise Exception('Err: OpenThread device Firmware not matching..')\n                self.__sendCommand(WPANCTL_CMD + 'leave')\n                self.__sendCommand(WPANCTL_CMD + 'dataset erase')\n            else:\n                raise Exception('Err: Device not connected ..')\n\n        except Exception, e:\n            ModuleHelper.WriteIntoDebugLogger('intialize() Error: ' + str(e))", "response": "initialize the serial port with baudrate timeout parameters"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef getNetworkName(self):\n        print '%s call getNetworkname' % self.port\n        networkName = self.__sendCommand(WPANCTL_CMD + 'getprop -v Network:Name')[0]\n        return self.__stripValue(networkName)", "response": "get Thread Network name"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nsets the extended address of Thread device", "response": "def setMAC(self, xEUI):\n        \"\"\"set the extended addresss of Thread device\n\n        Args:\n            xEUI: extended address in hex format\n\n        Returns:\n            True: successful to set the extended address\n            False: fail to set the extended address\n        \"\"\"\n        print '%s call setMAC' % self.port\n\n        address64 = ''\n        try:\n            if not xEUI:\n                address64 = self.mac\n\n            if not isinstance(xEUI, str):\n                address64 = self.__convertLongToString(xEUI)\n\n                # prepend 0 at the beginning\n                if len(address64) < 16:\n                    address64 = address64.zfill(16)\n                    print address64\n            else:\n                address64 = xEUI\n\n            cmd = WPANCTL_CMD + 'setprop NCP:MACAddress %s' % address64\n\n            if self.__sendCommand(cmd)[0] != 'Fail':\n                self.mac = address64\n                return True\n            else:\n                return False\n        except Exception, e:\n            ModuleHelper.WriteIntoDebugLogger('setMAC() Error: ' + str(e))"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef getMAC(self, bType=MacType.RandomMac):\n        print '%s call getMAC' % self.port\n\n        # if power down happens, return extended address assigned previously\n        if self.isPowerDown:\n            macAddr64 = self.mac\n        else:\n            if bType == MacType.FactoryMac:\n                macAddr64 = self.__stripValue(self.__sendCommand(WPANCTL_CMD + 'getprop -v NCP:HardwareAddress')[0])\n            elif bType == MacType.HashMac:\n                macAddr64 = self.__stripValue(self.__sendCommand(WPANCTL_CMD + 'getprop -v NCP:MACAddress')[0])\n            else:\n                macAddr64 = self.__stripValue(self.__sendCommand(WPANCTL_CMD + 'getprop -v NCP:ExtendedAddress')[0])\n\n        return int(macAddr64, 16)", "response": "get one specific type of MAC address"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef getLL64(self):\n        print '%s call getLL64' % self.port\n        return self.__stripValue(self.__sendCommand(WPANCTL_CMD + 'getprop -v IPv6:LinkLocalAddress')[0])", "response": "get link local unicast IPv6 address"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ngetting router locator unicast IPv6 address", "response": "def getRloc(self):\n        \"\"\"get router locator unicast IPv6 address\"\"\"\n        print '%s call getRloc' % self.port\n        prefix = self.__stripValue(self.__sendCommand(WPANCTL_CMD + 'getprop -v IPv6:MeshLocalPrefix')[0])\n        mlprefix = prefix.split('/')[0]\n        rloc16 = self.__sendCommand(WPANCTL_CMD + 'getprop -v Thread:RLOC16')[0].lstrip('0x')\n        print \"prefix: \" + prefix\n        print \"mlprefix: \" + mlprefix\n        print \"rloc16: \" + rloc16\n\n        rloc = self.__padIp6Addr(mlprefix + '00ff:fe00:' + rloc16)\n        print \"rloc: \" + rloc\n        return rloc"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nset Thread Network master key used in secure the MLE / 802. 15. 4 packet", "response": "def setNetworkKey(self, key):\n        \"\"\"set Thread Network master key\n\n        Args:\n            key: Thread Network master key used in secure the MLE/802.15.4 packet\n\n        Returns:\n            True: successful to set the Thread Network master key\n            False: fail to set the Thread Network master key\n        \"\"\"\n        masterKey = ''\n        print '%s call setNetworkKey' % self.port\n\n        try:\n            if not isinstance(key, str):\n                masterKey = self.__convertLongToString(key)\n\n                # prpend '0' at the beginning\n                if len(masterKey) < 32:\n                    masterKey = masterKey.zfill(32)\n\n                cmd = WPANCTL_CMD + 'setprop Network:Key %s' % masterKey\n                datasetCmd = WPANCTL_CMD + 'setprop Dataset:MasterKey %s' % masterKey\n            else:\n                masterKey = key\n                cmd = WPANCTL_CMD + 'setprop Network:Key %s' % masterKey\n                datasetCmd = WPANCTL_CMD + 'setprop Dataset:MasterKey %s' % masterKey\n\n            self.networkKey = masterKey\n            self.hasActiveDatasetToCommit = True\n            return self.__sendCommand(cmd)[0] != 'Fail' and self.__sendCommand(datasetCmd)[0] != 'Fail'\n        except Exception, e:\n            ModuleHelper.WriteIntoDebugLogger('setNetworkkey() Error: ' + str(e))"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef addAllowMAC(self, xEUI):\n        print '%s call addAllowMAC' % self.port\n        print xEUI\n        if isinstance(xEUI, str):\n            macAddr = xEUI\n        else:\n            macAddr = self.__convertLongToString(xEUI)\n\n        try:\n            if self._addressfilterMode != 'whitelist':\n                if self.__setAddressfilterMode('Whitelist'):\n                    self._addressfilterMode = 'whitelist'\n\n            cmd = WPANCTL_CMD + 'insert MAC:Whitelist:Entries %s' % macAddr\n            ret = self.__sendCommand(cmd)[0] != 'Fail'\n\n            self._addressfilterSet.add(macAddr)\n            print 'current whitelist entries:'\n            for addr in self._addressfilterSet:\n                print addr\n            return ret\n\n        except Exception, e:\n            ModuleHelper.WriteIntoDebugLogger('addAllowMAC() Error: ' + str(e))", "response": "add a given extended address to the whitelist"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nclearing all entries in whitelist table", "response": "def clearAllowList(self):\n        \"\"\"clear all entries in whitelist table\n\n        Returns:\n            True: successful to clear the whitelist\n            False: fail to clear the whitelist\n        \"\"\"\n        print '%s call clearAllowList' % self.port\n\n        # remove all entries in whitelist\n        try:\n            print 'clearing whitelist entries:'\n            for addr in self._addressfilterSet:\n                print addr\n\n            # disable whitelist\n            if self.__setAddressfilterMode('disable'):\n                self._addressfilterMode = 'disable'\n                # clear ops\n                cmd = WPANCTL_CMD + 'insert MAC:Whitelist:Entries'\n                if self.__sendCommand(cmd)[0] != 'Fail':\n                    self._addressfilterSet.clear()\n                    return True\n            return False\n        except Exception, e:\n            ModuleHelper.WriteIntoDebugLogger('clearAllowList() Error: ' + str(e))"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nget current device role in Thread Network", "response": "def getDeviceRole(self):\n        \"\"\"get current device role in Thread Network\"\"\"\n        print '%s call getDeviceRole' % self.port\n        return self.__stripValue(self.__sendCommand(WPANCTL_CMD + 'getprop -v Network:NodeType')[0])"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nmakes device ready to join the Thread Network with a given role id", "response": "def joinNetwork(self, eRoleId):\n        \"\"\"make device ready to join the Thread Network with a given role\n\n        Args:\n            eRoleId: a given device role id\n\n        Returns:\n            True: ready to set Thread Network parameter for joining desired Network\n        \"\"\"\n        print '%s call joinNetwork' % self.port\n        print eRoleId\n\n        self.deviceRole = eRoleId\n        mode = 15\n        try:\n            if ModuleHelper.LeaderDutChannelFound:\n                self.channel = ModuleHelper.Default_Channel\n\n            # FIXME: when Harness call setNetworkDataRequirement()?\n            # only sleep end device requires stable networkdata now\n            if eRoleId == Thread_Device_Role.Leader:\n                print 'join as leader'\n                #rsdn\n                mode = 15\n                if self.AutoDUTEnable is False:\n                    # set ROUTER_DOWNGRADE_THRESHOLD\n                    self.__setRouterDowngradeThreshold(33)\n            elif eRoleId == Thread_Device_Role.Router:\n                print 'join as router'\n                #rsdn\n                mode = 15\n                if self.AutoDUTEnable is False:\n                    # set ROUTER_DOWNGRADE_THRESHOLD\n                    self.__setRouterDowngradeThreshold(33)\n            elif eRoleId == Thread_Device_Role.SED:\n                print 'join as sleepy end device'\n                #s\n                mode = 4\n                self.setPollingRate(self.sedPollingRate)\n            elif eRoleId == Thread_Device_Role.EndDevice:\n                print 'join as end device'\n                #rsn\n                mode = 13\n            elif eRoleId == Thread_Device_Role.REED:\n                print 'join as REED'\n                #rsdn\n                mode = 15\n                # set ROUTER_UPGRADE_THRESHOLD\n                self.__setRouterUpgradeThreshold(0)\n            elif eRoleId == Thread_Device_Role.EndDevice_FED:\n                # always remain an ED, never request to be a router\n                print 'join as FED'\n                #rsdn\n                mode = 15\n                # set ROUTER_UPGRADE_THRESHOLD\n                self.__setRouterUpgradeThreshold(0)\n            elif eRoleId == Thread_Device_Role.EndDevice_MED:\n                print 'join as MED'\n                #rsn\n                mode = 13\n            else:\n                pass\n\n            # set Thread device mode with a given role\n            self.__setDeviceMode(mode)\n            self.__setKeySwitchGuardTime(0)  #temporally\n            time.sleep(0.1)\n            # start OpenThreadWpan\n            self.__startOpenThreadWpan()\n            time.sleep(3)\n            return True\n        except Exception, e:\n            ModuleHelper.WriteIntoDebugLogger('joinNetwork() Error: ' + str(e))"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ngetting current Thread Network Partition from LeaderData", "response": "def getNetworkFragmentID(self):\n        \"\"\"get current partition id of Thread Network Partition from LeaderData\n\n        Returns:\n            The Thread network Partition Id\n        \"\"\"\n        print '%s call getNetworkFragmentID' % self.port\n        if not self.____isOpenThreadWpanRunning():\n            print 'OpenThreadWpan is not running'\n            return None\n\n        return self.__sendCommand(WPANCTL_CMD + 'getprop -v Network:PartitionId')[0]"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ngetting Thread device s parent extended address and rloc16 short address", "response": "def getParentAddress(self):\n        \"\"\"get Thread device's parent extended address and rloc16 short address\n\n        Returns:\n            The extended address of parent in hex format\n        \"\"\"\n        print '%s call getParentAddress' % self.port\n        parentInfo = []\n        parentInfo = self.__stripValue(self.__sendCommand(WPANCTL_CMD + 'getporp -v Thread:Parent')).split(' ')\n\n        return parentInfo[0]"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\npowering down the OpenThreadWpan", "response": "def powerDown(self):\n        \"\"\"power down the OpenThreadWpan\"\"\"\n        print '%s call powerDown' % self.port\n        if self.__sendCommand(WPANCTL_CMD + 'setprop Daemon:AutoAssociateAfterReset false')[0] != 'Fail':\n            time.sleep(0.5)\n            if self.__sendCommand(WPANCTL_CMD + 'reset')[0] != 'Fail':\n                self.isPowerDown = True\n                return True\n            else:\n                return False\n        else:\n            return False"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef powerUp(self):\n        print '%s call powerUp' % self.port\n        if not self.handle:\n            self._connect()\n\n        self.isPowerDown = False\n        if self.__sendCommand(WPANCTL_CMD + 'attach')[0] != 'Fail':\n            time.sleep(3)\n        else:\n            return False\n\n        if self.__sendCommand(WPANCTL_CMD + 'setprop Daemon:AutoAssociateAfterReset true')[0] == 'Fail':\n            return False\n\n        if self.__stripValue(self.__sendCommand(WPANCTL_CMD + 'getprop -v NCP:State')[0]) != 'associated':\n            print 'powerUp failed'\n            return False\n        else:\n            return True", "response": "power up the Thread device"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nresetting and rejoin Thread Network with any timeout", "response": "def reboot(self):\n        \"\"\"reset and rejoin to Thread Network without any timeout\n\n        Returns:\n            True: successful to reset and rejoin the Thread Network\n            False: fail to reset and rejoin the Thread Network\n        \"\"\"\n        print '%s call reboot' % self.port\n        try:\n            self._sendline(WPANCTL_CMD + 'reset')\n            self.isPowerDown = True\n\n\n            if self.__sendCommand(WPANCTL_CMD + 'getprop -v NCP:State')[0] != 'associated':\n                print '[FAIL] reboot'\n                return False\n            else:\n                return True\n        except Exception, e:\n            ModuleHelper.WriteIntoDebugLogger('reboot() Error: ' + str(e))"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nsend ICMPv6 echo request with a given length to a unicast destination", "response": "def ping(self, destination, length=20):\n        \"\"\" send ICMPv6 echo request with a given length to a unicast destination\n            address\n\n        Args:\n            destination: the unicast destination address of ICMPv6 echo request\n            length: the size of ICMPv6 echo request payload\n        \"\"\"\n        print '%s call ping' % self.port\n        print 'destination: %s' %destination\n        try:\n            cmd = 'ping %s -c 1 -s %s -I %s'  % (destination, str(length), WPAN_INTERFACE)\n            if self._is_net:\n                ssh_stdin, ssh_stdout, ssh_stderr = self.handle.exec_command(cmd)\n            else:\n                self._sendline(cmd)\n                self._expect(cmd)\n            # wait echo reply\n            time.sleep(1)\n        except Exception, e:\n            ModuleHelper.WriteIntoDebugLogger('ping() Error: ' + str(e))"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ngetting OpenThreadWpan stack firmware version number", "response": "def getVersionNumber(self):\n        \"\"\"get OpenThreadWpan stack firmware version number\"\"\"\n        print '%s call getVersionNumber' % self.port\n        versionStr = self.__sendCommand(WPANCTL_CMD + 'getprop -v NCP:Version')[0]\n\n        return self.__stripValue(versionStr)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nsetting Thread Network PAN ID to a given PAN ID", "response": "def setPANID(self, xPAN):\n        \"\"\"set Thread Network PAN ID\n\n        Args:\n            xPAN: a given PAN ID in hex format\n\n        Returns:\n            True: successful to set the Thread Network PAN ID\n            False: fail to set the Thread Network PAN ID\n        \"\"\"\n        print '%s call setPANID' % self.port\n        print xPAN\n        panid = ''\n        try:\n            if not isinstance(xPAN, str):\n                panid = str(hex(xPAN))\n                print panid\n\n            cmd = WPANCTL_CMD + 'setprop -s Network:PANID %s' % panid\n            datasetCmd = WPANCTL_CMD + 'setprop Dataset:PanId %s' % panid\n            self.hasActiveDatasetToCommit = True\n            return self.__sendCommand(cmd)[0] != 'Fail' and self.__sendCommand(datasetCmd)[0] != 'Fail'\n        except Exception, e:\n            ModuleHelper.WriteIntoDebugLogger('setPANID() Error: ' + str(e))"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nresets the internal state of the object", "response": "def reset(self):\n        \"\"\"factory reset\"\"\"\n        print '%s call reset' % self.port\n        try:\n            if self._is_net:\n                self.__sendCommand(WPANCTL_CMD + 'leave')\n            else:\n                self._sendline(WPANCTL_CMD + 'leave')\n\n            self.__sendCommand(WPANCTL_CMD + 'dataset erase')\n            time.sleep(2)\n            if not self._is_net:\n                self._read()\n        except Exception, e:\n            ModuleHelper.WriteIntoDebugLogger('reset() Error: ' + str(e))"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef removeRouter(self, xRouterId):\n        print '%s call removeRouter' % self.port\n        print xRouterId\n        routerId = ''\n        routerId = self.__convertRlocToRouterId(xRouterId)\n        print routerId\n\n        if routerId == None:\n            print 'no matched xRouterId'\n            return False\n\n        try:\n            cmd = 'releaserouterid %s' % routerId\n            return self.__sendCommand(cmd)[0] != 'Fail'\n        except Exception, e:\n            ModuleHelper.WriteIntoDebugLogger('removeRouter() Error: ' + str(e))", "response": "kick router with a given router id from the Thread Network Network\n            returns True if the router was removed False otherwise"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef setDefaultValues(self):\n        print '%s call setDefaultValues' % self.port\n\n        # initialize variables\n        self.networkName = ModuleHelper.Default_NwkName\n        self.networkKey = ModuleHelper.Default_NwkKey\n        self.channel = ModuleHelper.Default_Channel\n        self.channelMask = \"0x7fff800\" #(0xffff << 11)\n        self.panId = ModuleHelper.Default_PanId\n        self.xpanId = ModuleHelper.Default_XpanId\n        self.meshLocalPrefix = ModuleHelper.Default_MLPrefix\n        self.pskc = \"00000000000000000000000000000000\"  # OT only accept hex format PSKc for now\n        self.securityPolicySecs = ModuleHelper.Default_SecurityPolicy\n        self.securityPolicyFlags = \"onrcb\"\n        self.activetimestamp = ModuleHelper.Default_ActiveTimestamp\n        #self.sedPollingRate = ModuleHelper.Default_Harness_SED_Polling_Rate\n        self.sedPollingRate = 3\n        self.deviceRole = None\n        self.provisioningUrl = ''\n        self.hasActiveDatasetToCommit = False\n        self.logThread = Queue()\n        self.logThreadStatus = self.logStatus['stop']\n        self.networkDataRequirement = ''      # indicate Thread device requests full or stable network data\n        self.isPowerDown = False              # indicate if Thread device experiences a power down event\n        self._addressfilterMode = 'disable'   # indicate AddressFilter mode ['disable', 'whitelist', 'blacklist']\n        self._addressfilterSet = set()        # cache filter entries\n        self.isActiveCommissioner = False     # indicate if Thread device is an active commissioner\n        self._lines = None                    # buffered lines read from device\n\n        # initialize device configuration\n        try:\n            self.setMAC(self.mac)\n            self.__setChannelMask(self.channelMask)\n            self.__setSecurityPolicy(self.securityPolicySecs, self.securityPolicyFlags)\n            self.setChannel(self.channel)\n            self.setPANID(self.panId)\n            self.setXpanId(self.xpanId)\n            self.setNetworkName(self.networkName)\n            self.setNetworkKey(self.networkKey)\n            self.setMLPrefix(self.meshLocalPrefix)\n            self.setPSKc(self.pskc)\n            self.setActiveTimestamp(self.activetimestamp)\n        except Exception, e:\n            ModuleHelper.WriteIntoDebugLogger('setDefaultValue() Error: ' + str(e))", "response": "set default mandatory Thread Network parameter value"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nsets data polling rate for sleepy end device", "response": "def setPollingRate(self, iPollingRate):\n        \"\"\"set data polling rate for sleepy end device\n\n        Args:\n            iPollingRate: data poll period of sleepy end device\n\n        Returns:\n            True: successful to set the data polling rate for sleepy end device\n            False: fail to set the data polling rate for sleepy end device\n        \"\"\"\n        print '%s call setPollingRate' % self.port\n        print iPollingRate\n        try:\n            cmd = WPANCTL_CMD + 'setprop NCP:SleepyPollInterval %s' % str(iPollingRate*1000)\n            print cmd\n            return self.__sendCommand(cmd)[0] != 'Fail'\n        except Exception, e:\n            ModuleHelper.WriteIntoDebugLogger('setPollingRate() Error: ' + str(e))"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef resetAndRejoin(self, timeout):\n        print '%s call resetAndRejoin' % self.port\n        print timeout\n        try:\n            if self.__sendCommand(WPANCTL_CMD + 'setprop Daemon:AutoAssociateAfterReset false')[0] != 'Fail':\n                time.sleep(0.5)\n                if self.__sendCommand(WPANCTL_CMD + 'reset')[0] != 'Fail':\n                    self.isPowerDown = True\n                else:\n                    return False\n            else:\n                return False\n            time.sleep(timeout)\n\n            if self.deviceRole == Thread_Device_Role.SED:\n                self.setPollingRate(self.sedPollingRate)\n\n            if self.__sendCommand(WPANCTL_CMD + 'attach')[0] != 'Fail':\n                time.sleep(3)\n            else:\n                return False\n\n            if self.__sendCommand(WPANCTL_CMD + 'setprop Daemon:AutoAssociateAfterReset true')[0] == 'Fail':\n                return False\n\n            if self.__stripValue(self.__sendCommand(WPANCTL_CMD + 'getprop -v NCP:State')[0]) != 'associated':\n                print '[FAIL] reset and rejoin'\n                return False\n            return True\n        except Exception, e:\n            ModuleHelper.WriteIntoDebugLogger('resetAndRejoin() Error: ' + str(e))", "response": "reset and rejoin Thread Network with a given timeout delay"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nconfigure the border router with a given prefix entry parameters", "response": "def configBorderRouter(self, P_Prefix, P_stable=1, P_default=1, P_slaac_preferred=0, P_Dhcp=0, P_preference=0, P_on_mesh=1, P_nd_dns=0):\n        \"\"\"configure the border router with a given prefix entry parameters\n\n        Args:\n            P_Prefix: IPv6 prefix that is available on the Thread Network\n            P_stable: is true if the default router is expected to be stable network data\n            P_default: is true if border router offers the default route for P_Prefix\n            P_slaac_preferred: is true if Thread device is allowed auto-configure address using P_Prefix\n            P_Dhcp: is true if border router is a DHCPv6 Agent\n            P_preference: is two-bit signed integer indicating router preference\n            P_on_mesh: is true if P_Prefix is considered to be on-mesh\n            P_nd_dns: is true if border router is able to supply DNS information obtained via ND\n\n        Returns:\n            True: successful to configure the border router with a given prefix entry\n            False: fail to configure the border router with a given prefix entry\n        \"\"\"\n        print '%s call configBorderRouter' % self.port\n        prefix = self.__convertIp6PrefixStringToIp6Address(str(P_Prefix))\n        print prefix\n        try:\n            parameter = ''\n\n            if P_slaac_preferred == 1:\n                parameter += ' -a -f'\n\n            if P_stable == 1:\n                parameter += ' -s'\n\n            if P_default == 1:\n                parameter += ' -r'\n\n            if P_Dhcp == 1:\n                parameter += ' -d'\n\n            if P_on_mesh == 1:\n                parameter += ' -o'\n\n            cmd = WPANCTL_CMD + 'add-prefix %s %s -P %d' % (prefix, parameter, P_preference)\n            print parameter\n            print cmd\n            if self.__sendCommand(cmd)[0] != 'Fail':\n                return True\n            else:\n                return False\n        except Exception, e:\n            ModuleHelper.WriteIntoDebugLogger('configBorderRouter() Error: ' + str(e))"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef setKeepAliveTimeOut(self, iTimeOut):\n        print '%s call setKeepAliveTimeOut' % self.port\n        print iTimeOut\n        try:\n            cmd = WPANCTL_CMD + 'setprop NCP:SleepyPollInterval %s' % str(iTimeOut*1000)\n            print cmd\n            return self.__sendCommand(cmd)[0] != 'Fail'\n        except Exception, e:\n            ModuleHelper.WriteIntoDebugLogger('setKeepAliveTimeOut() Error: ' + str(e))", "response": "set keep alive timeout for a specific NCP device"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nset the Thread Network master key sequence counter corresponding to Thread Network master key sequence value", "response": "def setKeySequenceCounter(self, iKeySequenceValue):\n        \"\"\" set the Key sequence counter corresponding to Thread Network master key\n\n        Args:\n            iKeySequenceValue: key sequence value\n\n        Returns:\n            True: successful to set the key sequence\n            False: fail to set the key sequence\n        \"\"\"\n        print '%s call setKeySequenceCounter' % self.port\n        print iKeySequenceValue\n        try:\n            cmd = WPANCTL_CMD + 'setprop Network:KeyIndex %s' % str(iKeySequenceValue)\n            if self.__sendCommand(cmd)[0] != 'Fail':\n                time.sleep(1)\n                return True\n            else:\n                return False\n        except Exception, e:\n            ModuleHelper.WriteIntoDebugLogger('setKeySequenceCounter() Error: ' + str(e))"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ngetting current Thread Network key sequence", "response": "def getKeySequenceCounter(self):\n        \"\"\"get current Thread Network key sequence\"\"\"\n        print '%s call getKeySequenceCounter' % self.port\n        keySequence = ''\n        keySequence = self.__sendCommand(WPANCTL_CMD + 'getprop -v Network:KeyIndex')[0]\n        return keySequence"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef incrementKeySequenceCounter(self, iIncrementValue=1):\n        print '%s call incrementKeySequenceCounter' % self.port\n        print iIncrementValue\n        currentKeySeq = ''\n        try:\n            currentKeySeq = self.getKeySequenceCounter()\n            keySequence = int(currentKeySeq, 10) + iIncrementValue\n            print keySequence\n            return self.setKeySequenceCounter(keySequence)\n        except Exception, e:\n            ModuleHelper.WriteIntoDebugLogger('incrementKeySequenceCounter() Error: ' + str(e))", "response": "increment the key sequence counter for a given value"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nsets the extended PAN ID of Thread Network Network", "response": "def setXpanId(self, xPanId):\n        \"\"\"set extended PAN ID of Thread Network\n\n        Args:\n            xPanId: extended PAN ID in hex format\n\n        Returns:\n            True: successful to set the extended PAN ID\n            False: fail to set the extended PAN ID\n        \"\"\"\n        xpanid = ''\n        print '%s call setXpanId' % self.port\n        print xPanId\n        try:\n            if not isinstance(xPanId, str):\n                xpanid = self.__convertLongToString(xPanId)\n\n                # prepend '0' at the beginning\n                if len(xpanid) < 16:\n                    xpanid = xpanid.zfill(16)\n                    print xpanid\n                    cmd = WPANCTL_CMD + 'setprop Network:XPANID %s' % xpanid\n                    datasetCmd = WPANCTL_CMD + 'setprop Dataset:ExtendedPanId %s' % xpanid\n            else:\n                xpanid = xPanId\n                cmd = WPANCTL_CMD + 'setprop Network:XPANID %s' % xpanid\n                datasetCmd = WPANCTL_CMD + 'setprop Dataset:ExtendedPanId %s' % xpanid\n\n            self.xpanId = xpanid\n            self.hasActiveDatasetToCommit = True\n            return self.__sendCommand(cmd)[0] != 'Fail' and self.__sendCommand(datasetCmd)[0] != 'Fail'\n        except Exception, e:\n            ModuleHelper.WriteIntoDebugLogger('setXpanId() Error: ' + str(e))"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nsetting the Thread Network Partition ID", "response": "def setPartationId(self, partationId):\n        \"\"\"set Thread Network Partition ID\n\n        Args:\n            partitionId: partition id to be set by leader\n\n        Returns:\n            True: successful to set the Partition ID\n            False: fail to set the Partition ID\n        \"\"\"\n        print '%s call setPartationId' % self.port\n        print partationId\n\n        cmd = WPANCTL_CMD + 'setprop Network:PartitionId %s' %(str(hex(partationId)).rstrip('L'))\n        print cmd\n        return self.__sendCommand(cmd)[0] != 'Fail'"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ngets the global unicast IPv6 address of OpenThreadWpan", "response": "def getGUA(self, filterByPrefix=None):\n        \"\"\"get expected global unicast IPv6 address of OpenThreadWpan\n\n        Args:\n            filterByPrefix: a given expected global IPv6 prefix to be matched\n\n        Returns:\n            a global IPv6 address\n        \"\"\"\n        print '%s call getGUA' % self.port\n        print filterByPrefix\n        globalAddrs = []\n        try:\n            # get global addrs set if multiple\n            globalAddrs = self.getGlobal()\n\n            if filterByPrefix is None:\n                return self.__padIp6Addr(globalAddrs[0])\n            else:\n                for line in globalAddrs:\n                    line = self.__padIp6Addr(line)\n                    print \"Padded IPv6 Address:\" + line\n                    if line.startswith(filterByPrefix):\n                        return line\n                print 'no global address matched'\n                return str(globalAddrs[0])\n        except Exception, e:\n            ModuleHelper.WriteIntoDebugLogger('getGUA() Error: ' + str(e))\n            return e"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nget child timeout value", "response": "def getChildTimeoutValue(self):\n        \"\"\"get child timeout\"\"\"\n        print '%s call getChildTimeoutValue' % self.port\n        childTimeout = self.__sendCommand(WPANCTL_CMD + 'getporp -v Thread:ChildTimeout')[0]\n        return int(childTimeout)"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nstarts Collapsed Commissioner Returns: True: successful to start Commissioner False: fail to start Commissioner", "response": "def startCollapsedCommissioner(self):\n        \"\"\"start Collapsed Commissioner\n\n        Returns:\n            True: successful to start Commissioner\n            False: fail to start Commissioner\n        \"\"\"\n        print '%s call startCollapsedCommissioner' % self.port\n        startCmd = WPANCTL_CMD + 'form %s -c %s -T router' % (self.networkName, str(self.channel))\n        if self.__sendCommand(startCmd) != 'Fail':\n            time.sleep(2)\n            cmd = WPANCTL_CMD + 'commissioner start'\n            print cmd\n            if self.__sendCommand(cmd)[0] != 'Fail':\n                self.isActiveCommissioner = True\n                time.sleep(20) # time for petition process\n                return True\n        return False"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef scanJoiner(self, xEUI='*', strPSKd='threadjpaketest'):\n        print '%s call scanJoiner' % self.port\n        if not isinstance(xEUI, str):\n            eui64 = self.__convertLongToString(xEUI)\n\n            # prepend 0 at the beginning\n            if len(eui64) < 16:\n                eui64 = eui64.zfill(16)\n                print eui64\n        else:\n            eui64 = xEUI\n\n        # long timeout value to avoid automatic joiner removal (in seconds)\n        timeout = 500\n\n        cmd = WPANCTL_CMD + 'commissioner joiner-add %s %s %s' % (eui64, str(timeout), strPSKd)\n        print cmd\n        if not self.isActiveCommissioner:\n            self.startCollapsedCommissioner()\n        if self.__sendCommand(cmd)[0] != 'Fail':\n            return True\n        else:\n            return False", "response": "add a new Joiner s steering data to the current Thread."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef setProvisioningUrl(self, strURL='grl.com'):\n        print '%s call setProvisioningUrl' % self.port\n        self.provisioningUrl = strURL\n        if self.deviceRole == Thread_Device_Role.Commissioner:\n            cmd = WPANCTL_CMD + 'setprop Commissioner:ProvisioningUrl %s' %(strURL)\n            print cmd\n            return self.__sendCommand(cmd)[0] != \"Fail\"\n        return True", "response": "set provisioning Url\n\n        Args:\n            strURL: Provisioning Url string\n\n        Returns:\n            True: successful to set provisioning Url\n            False: fail to set provisioning Url"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef allowCommission(self):\n        print '%s call allowCommission' % self.port\n        try:\n            cmd = WPANCTL_CMD + 'commissioner start'\n            print cmd\n            if self.isActiveCommissioner:\n                return True\n            if self.__sendCommand(cmd)[0] != 'Fail':\n                self.isActiveCommissioner = True\n                time.sleep(40) # time for petition process and at least one keep alive\n                return True\n            else:\n                return False\n        except Exception, e:\n            ModuleHelper.WriteIntoDebugLogger('allowcommission() error: ' + str(e))", "response": "start commissioner candidate petition process\n           "}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nstart joiner s PSKd", "response": "def joinCommissioned(self, strPSKd='threadjpaketest', waitTime=20):\n        \"\"\"start joiner\n\n        Args:\n            strPSKd: Joiner's PSKd\n\n        Returns:\n            True: successful to start joiner\n            False: fail to start joiner\n        \"\"\"\n        print '%s call joinCommissioned' % self.port\n        cmd = WPANCTL_CMD + 'joiner --start %s %s' %(strPSKd, self.provisioningUrl)\n        print cmd\n        if self.__sendCommand(cmd)[0] != \"Fail\":\n            if self.__getJoinerState():\n                self.__sendCommand(WPANCTL_CMD + 'joiner --attach')\n                time.sleep(30)\n                return True\n            else:\n                return False\n        else:\n            return False"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nsend MGMT_PANID_QUERY message to a given destination", "response": "def MGMT_PANID_QUERY(self, sAddr, xCommissionerSessionId, listChannelMask, xPanId):\n        \"\"\"send MGMT_PANID_QUERY message to a given destination\n\n        Args:\n            xPanId: a given PAN ID to check the conflicts\n\n        Returns:\n            True: successful to send MGMT_PANID_QUERY message.\n            False: fail to send MGMT_PANID_QUERY message.\n        \"\"\"\n        print '%s call MGMT_PANID_QUERY' % self.port\n        panid = ''\n        channelMask = ''\n        channelMask = self.__ChannelMaskListToStr(listChannelMask)\n\n        if not isinstance(xPanId, str):\n            panid = str(hex(xPanId))\n\n        try:\n            cmd = WPANCTL_CMD + 'commissioner pan-id-query %s %s %s' % (panid, channelMask, sAddr)\n            print cmd\n            return self.__sendCommand(cmd) != 'Fail'\n        except Exception, e:\n            ModuleHelper.WriteIntoDebugLogger('MGMT_PANID_QUERY() error: ' + str(e))"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef MGMT_ANNOUNCE_BEGIN(self, sAddr, xCommissionerSessionId, listChannelMask, xCount, xPeriod):\n        print '%s call MGMT_ANNOUNCE_BEGIN' % self.port\n        channelMask = ''\n        channelMask = self.__ChannelMaskListToStr(listChannelMask)\n        try:\n            cmd = WPANCTL_CMD + 'commissioner announce-begin %s %s %s %s' % (channelMask, xCount, xPeriod, sAddr)\n            print cmd\n            return self.__sendCommand(cmd) != 'Fail'\n        except Exception, e:\n            ModuleHelper.WriteIntoDebugLogger('MGMT_ANNOUNCE_BEGIN() error: ' + str(e))", "response": "send MGMT_ANNOUNCE_BEGIN message to a given destination"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef MGMT_ACTIVE_GET(self, Addr='', TLVs=[]):\n        print '%s call MGMT_ACTIVE_GET' % self.port\n        try:\n\n            cmd = WPANCTL_CMD + 'dataset mgmt-get-active'\n\n            if len(TLVs) != 0:\n                tlvs = \"\".join(hex(tlv).lstrip(\"0x\").zfill(2) for tlv in TLVs)\n                setTLVCmd = WPANCTL_CMD + 'setprop Dataset:RawTlvs ' + tlvs\n                if self.__sendCommand(setTLVCmd)[0] == 'Fail':\n                    return False\n            else:\n                if self.__sendCommand(WPANCTL_CMD + 'dataset erase')[0] == 'Fail':\n                    return False\n\n            if Addr != '':\n                setAddressCmd = WPANCTL_CMD + 'setprop Dataset:DestIpAddress ' + Addr\n                if self.__sendCommand(setAddressCmd)[0] == 'Fail':\n                    return False\n\n            print cmd\n\n            return self.__sendCommand(cmd)[0] != 'Fail'\n\n        except Exception, e:\n            ModuleHelper.WriteIntoDebugLogger('MGMT_ACTIVE_GET() Error: ' + str(e))", "response": "send MGMT_ACTIVE_GET command\n\n        Returns:\n            True: successful to send MGMT_ACTIVE_GET\n            False: fail to send MGMT_ACTIVE_GET"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef MGMT_ACTIVE_SET(self, sAddr='', xCommissioningSessionId=None, listActiveTimestamp=None, listChannelMask=None, xExtendedPanId=None,\n                        sNetworkName=None, sPSKc=None, listSecurityPolicy=None, xChannel=None, sMeshLocalPrefix=None, xMasterKey=None,\n                        xPanId=None, xTmfPort=None, xSteeringData=None, xBorderRouterLocator=None, BogusTLV=None, xDelayTimer=None):\n        \"\"\"send MGMT_ACTIVE_SET command\n\n        Returns:\n            True: successful to send MGMT_ACTIVE_SET\n            False: fail to send MGMT_ACTIVE_SET\n        \"\"\"\n        print '%s call MGMT_ACTIVE_SET' % self.port\n        try:\n            cmd = WPANCTL_CMD + 'dataset mgmt-set-active'\n\n            if self.__sendCommand(WPANCTL_CMD + 'dataset erase')[0] == 'Fail':\n                return False\n\n            if listActiveTimestamp != None:\n                sActiveTimestamp = str(hex(listActiveTimestamp[0]))\n                if len(sActiveTimestamp) < 18:\n                    sActiveTimestamp = sActiveTimestamp.lstrip('0x').zfill(16)\n                setActiveTimeCmd = WPANCTL_CMD + 'setprop Dataset:ActiveTimestamp ' + sActiveTimestamp\n                if self.__sendCommand(setActiveTimeCmd)[0] == 'Fail':\n                    return False\n\n            if xExtendedPanId != None:\n                xpanid = self.__convertLongToString(xExtendedPanId)\n                if len(xpanid) < 16:\n                    xpanid = xpanid.zfill(16)\n                setExtendedPanIdCmd = WPANCTL_CMD + 'setprop Dataset:ExtendedPanId ' + xpanid\n                if self.__sendCommand(setExtendedPanIdCmd)[0] == 'Fail':\n                    return False\n\n            if sNetworkName != None:\n                setNetworkNameCmd = WPANCTL_CMD + 'setprop Dataset:NetworkName ' + str(sNetworkName)\n                if self.__sendCommand(setNetworkNameCmd)[0] == 'Fail':\n                    return False\n\n            if xChannel != None:\n                setChannelCmd = WPANCTL_CMD + 'setprop Dataset:Channel ' + str(xChannel)\n                if self.__sendCommand(setChannelCmd)[0] == 'Fail':\n                    return False\n\n            if sMeshLocalPrefix != None:\n                setMLPrefixCmd = WPANCTL_CMD + 'setprop Dataset:MeshLocalPrefix ' + str(sMeshLocalPrefix)\n                if self.__sendCommand(setMLPrefixCmd)[0] == 'Fail':\n                    return False\n\n            if xMasterKey != None:\n                key = self.__convertLongToString(xMasterKey)\n\n                if len(key) < 32:\n                    key = key.zfill(32)\n\n                setMasterKeyCmd = WPANCTL_CMD + 'setprop Dataset:MasterKey ' + key\n                if self.__sendCommand(setMasterKeyCmd)[0] == 'Fail':\n                    return False\n\n            if xPanId != None:\n                setPanIdCmd = WPANCTL_CMD + 'setprop Dataset:PanId ' + str(xPanId)\n                if self.__sendCommand(setPanIdCmd)[0] == 'Fail':\n                    return False\n\n            if listChannelMask != None:\n                setChannelMaskCmd = WPANCTL_CMD + 'setprop Dataset:ChannelMaskPage0 ' \\\n                                    + '0x' + self.__convertLongToString(self.__convertChannelMask(listChannelMask))\n                if self.__sendCommand(setChannelMaskCmd)[0] == 'Fail':\n                    return False\n\n            if sPSKc != None or listSecurityPolicy != None or \\\n                    xCommissioningSessionId != None or xTmfPort != None or xSteeringData != None or xBorderRouterLocator != None or \\\n                    BogusTLV != None:\n                setRawTLVCmd = WPANCTL_CMD + 'setprop Dataset:RawTlvs '\n\n            if sPSKc != None:\n                setRawTLVCmd += '0410'\n                stretchedPskc = Thread_PBKDF2.get(sPSKc, ModuleHelper.Default_XpanId, ModuleHelper.Default_NwkName)\n                pskc = hex(stretchedPskc).rstrip('L').lstrip('0x')\n\n                if len(pskc) < 32:\n                    pskc = pskc.zfill(32)\n\n                setRawTLVCmd += pskc\n\n            if listSecurityPolicy != None:\n                setRawTLVCmd += '0c03'\n\n                rotationTime = 0\n                policyBits = 0\n\n                # previous passing way listSecurityPolicy=[True, True, 3600, False, False, True]\n                if (len(listSecurityPolicy) == 6):\n                    rotationTime = listSecurityPolicy[2]\n\n                    # the last three reserved bits must be 1\n                    policyBits = 0b00000111\n\n                    if listSecurityPolicy[0]:\n                        policyBits = policyBits | 0b10000000\n                    if listSecurityPolicy[1]:\n                        policyBits = policyBits | 0b01000000\n                    if listSecurityPolicy[3]:\n                        policyBits = policyBits | 0b00100000\n                    if listSecurityPolicy[4]:\n                        policyBits = policyBits | 0b00010000\n                    if listSecurityPolicy[5]:\n                        policyBits = policyBits | 0b00001000\n                else:\n                    # new passing way listSecurityPolicy=[3600, 0b11001111]\n                    rotationTime = listSecurityPolicy[0]\n                    policyBits = listSecurityPolicy[1]\n\n                policy = str(hex(rotationTime))[2:]\n\n                if len(policy) < 4:\n                    policy = policy.zfill(4)\n\n                setRawTLVCmd += policy\n\n                setRawTLVCmd += str(hex(policyBits))[2:]\n\n            if xCommissioningSessionId != None:\n                setRawTLVCmd += '0b02'\n                sessionid = str(hex(xCommissioningSessionId))[2:]\n\n                if len(sessionid) < 4:\n                    sessionid = sessionid.zfill(4)\n\n                setRawTLVCmd += sessionid\n\n            if xBorderRouterLocator != None:\n                setRawTLVCmd += '0902'\n                locator = str(hex(xBorderRouterLocator))[2:]\n\n                if len(locator) < 4:\n                    locator = locator.zfill(4)\n\n                setRawTLVCmd += locator\n\n            if xSteeringData != None:\n                steeringData = self.__convertLongToString(xSteeringData)\n                setRawTLVCmd += '08' + str(len(steeringData)/2).zfill(2)\n                setRawTLVCmd += steeringData\n\n            if BogusTLV != None:\n                setRawTLVCmd += \"8202aa55\"\n\n            print setRawTLVCmd\n            print cmd\n\n            if self.__sendCommand(setRawTLVCmd)[0] == 'Fail':\n                return False\n\n            return self.__sendCommand(cmd)[0] != 'Fail'\n\n        except Exception, e:\n            ModuleHelper.WriteIntoDebugLogger('MGMT_ACTIVE_SET() Error: ' + str(e))", "response": "send MGMT_ACTIVE_SET command to set the state of the active MESH object"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nsending MGMT_PENDING_SET command to set the state of the current MGMT_PENDING_SET to the current state of the MGMT_PENDING_SET", "response": "def MGMT_PENDING_SET(self, sAddr='', xCommissionerSessionId=None, listPendingTimestamp=None, listActiveTimestamp=None, xDelayTimer=None,\n                         xChannel=None, xPanId=None, xMasterKey=None, sMeshLocalPrefix=None, sNetworkName=None):\n        \"\"\"send MGMT_PENDING_SET command\n\n        Returns:\n            True: successful to send MGMT_PENDING_SET\n            False: fail to send MGMT_PENDING_SET\n        \"\"\"\n        print '%s call MGMT_PENDING_SET' % self.port\n        try:\n            cmd = WPANCTL_CMD + 'dataset mgmt-set-pending'\n            if self.__sendCommand(WPANCTL_CMD + 'dataset erase')[0] == 'Fail':\n                return False\n\n            if listPendingTimestamp != None:\n                sActiveTimestamp = str(hex(listPendingTimestamp[0]))\n                if len(sActiveTimestamp) < 18:\n                    sActiveTimestamp = sActiveTimestamp.lstrip('0x').zfill(16)\n                setPendingTimeCmd = WPANCTL_CMD + 'setprop Dataset:PendingTimestamp ' + sActiveTimestamp\n                if self.__sendCommand(setPendingTimeCmd)[0] == 'Fail':\n                    return False\n\n            if listActiveTimestamp != None:\n                sActiveTimestamp = str(hex(listActiveTimestamp[0]))\n                if len(sActiveTimestamp) < 18:\n                    sActiveTimestamp = sActiveTimestamp.lstrip('0x').zfill(16)\n                setActiveTimeCmd = WPANCTL_CMD + 'setprop Dataset:ActiveTimestamp ' + sActiveTimestamp\n                if self.__sendCommand(setActiveTimeCmd)[0] == 'Fail':\n                    return False\n\n            if xDelayTimer != None:\n                setDelayTimerCmd = WPANCTL_CMD + 'setprop Dataset:Delay ' + str(xDelayTimer)\n                if self.__sendCommand(setDelayTimerCmd)[0] == 'Fail':\n                    return False\n\n            if sNetworkName != None:\n                setNetworkNameCmd = WPANCTL_CMD + 'setprop Dataset:NetworkName ' + str(sNetworkName)\n                if self.__sendCommand(setNetworkNameCmd)[0] == 'Fail':\n                    return False\n\n            if xChannel != None:\n                setChannelCmd = WPANCTL_CMD + 'setprop Dataset:Channel ' + str(xChannel)\n                if self.__sendCommand(setChannelCmd)[0] == 'Fail':\n                    return False\n\n            if sMeshLocalPrefix != None:\n                setMLPrefixCmd = WPANCTL_CMD + 'setprop Dataset:MeshLocalPrefix ' + str(sMeshLocalPrefix)\n                if self.__sendCommand(setMLPrefixCmd)[0] == 'Fail':\n                    return False\n\n            if xMasterKey != None:\n                key = self.__convertLongToString(xMasterKey)\n\n                if len(key) < 32:\n                    key = key.zfill(32)\n\n                setMasterKeyCmd = WPANCTL_CMD + 'setprop Dataset:MasterKey ' + key\n                if self.__sendCommand(setMasterKeyCmd)[0] == 'Fail':\n                    return False\n\n            if xPanId != None:\n                setPanIdCmd = WPANCTL_CMD + 'setprop Dataset:PanId ' + str(xPanId)\n                if self.__sendCommand(setPanIdCmd)[0] == 'Fail':\n                    return False\n\n            if xCommissionerSessionId != None:\n               print 'not handle xCommissionerSessionId'\n\n            print cmd\n\n            return self.__sendCommand(cmd)[0] != 'Fail'\n\n        except Exception, e:\n            ModuleHelper.WriteIntoDebugLogger('MGMT_PENDING_SET() Error: ' + str(e))"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef commissionerUnregister(self):\n        print '%s call commissionerUnregister' % self.port\n        cmd = WPANCTL_CMD + 'commissioner stop'\n        print cmd\n        if self.__sendCommand(cmd)[0] != 'Fail':\n            self.isActiveCommissioner = False\n            return True\n        else:\n            return False", "response": "stop commissioner\n\n        Returns:\n            True: successful to stop commissioner\n            False: fail to stop commissioner"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef start(self, host, nornir):\n        self.host = host\n        self.nornir = nornir\n\n        try:\n            logger.debug(\"Host %r: running task %r\", self.host.name, self.name)\n            r = self.task(self, **self.params)\n            if not isinstance(r, Result):\n                r = Result(host=host, result=r)\n\n        except NornirSubTaskError as e:\n            tb = traceback.format_exc()\n            logger.error(\n                \"Host %r: task %r failed with traceback:\\n%s\",\n                self.host.name,\n                self.name,\n                tb,\n            )\n            r = Result(host, exception=e, result=str(e), failed=True)\n\n        except Exception as e:\n            tb = traceback.format_exc()\n            logger.error(\n                \"Host %r: task %r failed with traceback:\\n%s\",\n                self.host.name,\n                self.name,\n                tb,\n            )\n            r = Result(host, exception=e, result=tb, failed=True)\n\n        r.name = self.name\n        r.severity_level = logging.ERROR if r.failed else self.severity_level\n\n        self.results.insert(0, r)\n        return self.results", "response": "Starts the task for the given host and returns a list of the results and the subtasks that were executed."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef is_dry_run(self, override: bool = None) -> bool:\n        return override if override is not None else self.nornir.data.dry_run", "response": "Returns whether the current task is a dry_run or not."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef failed_hosts(self):\n        return {h: r for h, r in self.items() if r.failed}", "response": "Returns a dict of hosts that failed during the execution of the task."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef napalm_validate(\n    task: Task,\n    src: Optional[str] = None,\n    validation_source: ValidationSourceData = None,\n) -> Result:\n    \"\"\"\n    Gather information with napalm and validate it:\n\n        http://napalm.readthedocs.io/en/develop/validate/index.html\n\n    Arguments:\n        src: file to use as validation source\n        validation_source (list): data to validate device's state\n\n    Returns:\n        Result object with the following attributes set:\n        * result (``dict``): dictionary with the result of the validation\n        * complies (``bool``): Whether the device complies or not\n    \"\"\"\n    device = task.host.get_connection(\"napalm\", task.nornir.config)\n    r = device.compliance_report(\n        validation_file=src, validation_source=validation_source\n    )\n    return Result(host=task.host, result=r)", "response": "Gather information with napalm and validate it"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef netmiko_save_config(\n    task: Task, cmd: str = \"\", confirm: bool = False, confirm_response: str = \"\"\n) -> Result:\n    \"\"\"\n    Execute Netmiko save_config method\n    Arguments:\n        cmd(str, optional): Command used to save the configuration.\n        confirm(bool, optional): Does device prompt for confirmation before executing save operation\n        confirm_response(str, optional): Response send to device when it prompts for confirmation\n\n    Returns:\n        :obj: `nornir.core.task.Result`:\n          * result (``str``): String showing the CLI output from the save operation\n    \"\"\"\n    conn = task.host.get_connection(\"netmiko\", task.nornir.config)\n    if cmd:\n        result = conn.save_config(\n            cmd=cmd, confirm=confirm, confirm_response=confirm_response\n        )\n    else:\n        result = conn.save_config(confirm=confirm, confirm_response=confirm_response)\n    return Result(host=task.host, result=result, changed=True)", "response": "Execute Netmiko save_config method on the node with optional command"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef netmiko_send_command(\n    task: Task,\n    command_string: str,\n    use_timing: bool = False,\n    enable: bool = False,\n    **kwargs: Any\n) -> Result:\n    \"\"\"\n    Execute Netmiko send_command method (or send_command_timing)\n\n    Arguments:\n        command_string: Command to execute on the remote network device.\n        use_timing: Set to True to switch to send_command_timing method.\n        enable: Set to True to force Netmiko .enable() call.\n        kwargs: Additional arguments to pass to send_command method.\n\n    Returns:\n        Result object with the following attributes set:\n          * result: Result of the show command (generally a string, but depends on use of TextFSM).\n    \"\"\"\n    net_connect = task.host.get_connection(\"netmiko\", task.nornir.config)\n    if enable:\n        net_connect.enable()\n    if use_timing:\n        result = net_connect.send_command_timing(command_string, **kwargs)\n    else:\n        result = net_connect.send_command(command_string, **kwargs)\n    return Result(host=task.host, result=result)", "response": "Execute a Netmiko command on the remote network device."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning the value of the given item from the host or hosts group variables.", "response": "def get(self, item, default=None):\n        \"\"\"\n        Returns the value ``item`` from the host or hosts group variables.\n\n        Arguments:\n            item(``str``): The variable to get\n            default(``any``): Return value if item not found\n        \"\"\"\n        if hasattr(self, item):\n            return getattr(self, item)\n        try:\n            return self.__getitem__(item)\n\n        except KeyError:\n            return default"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning an existing connection for the given connection type.", "response": "def get_connection(self, connection: str, configuration: Config) -> Any:\n        \"\"\"\n        The function of this method is twofold:\n\n            1. If an existing connection is already established for the given type return it\n            2. If none exists, establish a new connection of that type with default parameters\n               and return it\n\n        Raises:\n            AttributeError: if it's unknown how to establish a connection for the given type\n\n        Arguments:\n            connection: Name of the connection, for instance, netmiko, paramiko, napalm...\n\n        Returns:\n            An already established connection\n        \"\"\"\n        if connection not in self.connections:\n            conn = self.get_connection_parameters(connection)\n            self.open_connection(\n                connection=connection,\n                configuration=configuration,\n                hostname=conn.hostname,\n                port=conn.port,\n                username=conn.username,\n                password=conn.password,\n                platform=conn.platform,\n                extras=conn.extras,\n            )\n        return self.connections[connection].connection"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nget the state of a connection.", "response": "def get_connection_state(self, connection: str) -> Dict[str, Any]:\n        \"\"\"\n        For an already established connection return its state.\n        \"\"\"\n        if connection not in self.connections:\n            raise ConnectionNotOpen(connection)\n\n        return self.connections[connection].state"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nopen a new connection.", "response": "def open_connection(\n        self,\n        connection: str,\n        configuration: Config,\n        hostname: Optional[str] = None,\n        username: Optional[str] = None,\n        password: Optional[str] = None,\n        port: Optional[int] = None,\n        platform: Optional[str] = None,\n        extras: Optional[Dict[str, Any]] = None,\n        default_to_host_attributes: bool = True,\n    ) -> ConnectionPlugin:\n        \"\"\"\n        Open a new connection.\n\n        If ``default_to_host_attributes`` is set to ``True`` arguments will default to host\n        attributes if not specified.\n\n        Raises:\n            AttributeError: if it's unknown how to establish a connection for the given type\n\n        Returns:\n            An already established connection\n        \"\"\"\n        if connection in self.connections:\n            raise ConnectionAlreadyOpen(connection)\n\n        self.connections[connection] = self.connections.get_plugin(connection)()\n        if default_to_host_attributes:\n            conn_params = self.get_connection_parameters(connection)\n            self.connections[connection].open(\n                hostname=hostname if hostname is not None else conn_params.hostname,\n                username=username if username is not None else conn_params.username,\n                password=password if password is not None else conn_params.password,\n                port=port if port is not None else conn_params.port,\n                platform=platform if platform is not None else conn_params.platform,\n                extras=extras if extras is not None else conn_params.extras,\n                configuration=configuration,\n            )\n        else:\n            self.connections[connection].open(\n                hostname=hostname,\n                username=username,\n                password=password,\n                port=port,\n                platform=platform,\n                extras=extras,\n                configuration=configuration,\n            )\n        return self.connections[connection]"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef children_of_group(self, group: Union[str, Group]) -> Set[Host]:\n        hosts: List[Host] = set()\n        for host in self.hosts.values():\n            if host.has_parent_group(group):\n                hosts.add(host)\n        return hosts", "response": "Returns the set of hosts that belong to a group including those that belong to those that belong to a group including those that belong to a group including those that belong to the group indirectly via inheritance\n       ."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nadds a host to the inventory after initialization", "response": "def add_host(self, name: str, **kwargs) -> None:\n        \"\"\"\n        Add a host to the inventory after initialization\n        \"\"\"\n        host = {\n            name: deserializer.inventory.InventoryElement.deserialize_host(\n                name=name, defaults=self.defaults, **kwargs\n            )\n        }\n        self.hosts.update(host)"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nadds a group to the inventory after initialization", "response": "def add_group(self, name: str, **kwargs) -> None:\n        \"\"\"\n        Add a group to the inventory after initialization\n        \"\"\"\n        group = {\n            name: deserializer.inventory.InventoryElement.deserialize_group(\n                name=name, defaults=self.defaults, **kwargs\n            )\n        }\n        self.groups.update(group)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_defaults_dict(self) -> Dict:\n        return deserializer.inventory.Defaults.serialize(self.defaults).dict()", "response": "Returns serialized dictionary of defaults from inventory\n       "}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_groups_dict(self) -> Dict:\n        return {\n            k: deserializer.inventory.InventoryElement.serialize(v).dict()\n            for k, v in self.groups.items()\n        }", "response": "Returns serialized dictionary of groups from inventory\n       "}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn serialized dictionary of hosts from inventory", "response": "def get_hosts_dict(self) -> Dict:\n        \"\"\"\n        Returns serialized dictionary of hosts from inventory\n        \"\"\"\n        return {\n            k: deserializer.inventory.InventoryElement.serialize(v).dict()\n            for k, v in self.hosts.items()\n        }"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ntransfers files from src to dst using sftp protocol.", "response": "def sftp(\n    task: Task, src: str, dst: str, action: str, dry_run: Optional[bool] = None\n) -> Result:\n    \"\"\"\n    Transfer files from/to the device using sftp protocol\n\n    Example::\n\n        nornir.run(files.sftp,\n                    action=\"put\",\n                    src=\"README.md\",\n                    dst=\"/tmp/README.md\")\n\n    Arguments:\n        dry_run: Whether to apply changes or not\n        src: source file\n        dst: destination\n        action: ``put``, ``get``.\n\n    Returns:\n        Result object with the following attributes set:\n          * changed (``bool``):\n          * files_changed (``list``): list of files that changed\n    \"\"\"\n    dry_run = task.is_dry_run(dry_run)\n    actions = {\"put\": put, \"get\": get}\n    client = task.host.get_connection(\"paramiko\", task.nornir.config)\n    scp_client = SCPClient(client.get_transport())\n    sftp_client = paramiko.SFTPClient.from_transport(client.get_transport())\n    files_changed = actions[action](task, scp_client, sftp_client, src, dst, dry_run)\n    return Result(\n        host=task.host, changed=bool(files_changed), files_changed=files_changed\n    )"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nrenders a file with jinja2 and returns the rendered text", "response": "def template_file(\n    task: Task,\n    template: str,\n    path: str,\n    jinja_filters: FiltersDict = None,\n    **kwargs: Any\n) -> Result:\n    \"\"\"\n    Renders contants of a file with jinja2. All the host data is available in the template\n\n    Arguments:\n        template: filename\n        path: path to dir with templates\n        jinja_filters: jinja filters to enable. Defaults to nornir.config.jinja2.filters\n        **kwargs: additional data to pass to the template\n\n    Returns:\n        Result object with the following attributes set:\n          * result (``string``): rendered string\n    \"\"\"\n    jinja_filters = jinja_filters or {} or task.nornir.config.jinja2.filters\n    text = jinja_helper.render_from_file(\n        template=template,\n        path=path,\n        host=task.host,\n        jinja_filters=jinja_filters,\n        **kwargs\n    )\n    return Result(host=task.host, result=text)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef http_method(\n    task: Optional[Task] = None,\n    method: str = \"get\",\n    url: str = \"\",\n    raise_for_status: bool = True,\n    **kwargs: Any\n) -> Result:\n    \"\"\"\n    This is a convenience task that uses `requests <http://docs.python-requests.org/en/master/>`_ to\n    interact with an HTTP server.\n\n    Arguments:\n        method: HTTP method to call\n        url: URL to connect to\n        raise_for_status: Whether to call `raise_for_status\n            <http://docs.python-requests.org/en/master/api/#requests.Response.raise_for_status>`_\n            method automatically or not. For quick reference, raise_for_status will consider an\n            error if the return code is any of 4xx or 5xx\n        **kwargs: Keyword arguments will be passed to the `request\n            <http://docs.python-requests.org/en/master/api/#requests.request>`_\n            method\n\n    Returns:\n        Result object with the following attributes set:\n          * result (``string/dict``): Body of the response. Either text or a dict if the\n            response was a json object\n          * response (``requests.Response``): Original `Response\n            <http://docs.python-requests.org/en/master/api/#requests.Response>`_\n    \"\"\"\n    r = requests.request(method, url, **kwargs)\n\n    if raise_for_status:\n        r.raise_for_status()\n\n    try:\n        content_type = r.headers[\"Content-type\"]\n    except KeyError:\n        content_type = \"text\"\n\n    result = r.json() if \"application/json\" == content_type else r.text\n\n    return Result(host=task.host if task else None, response=r, result=result)", "response": "This is a convenience task that interacts with an HTTP server and returns the response body of the HTTP request."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef netmiko_send_config(\n    task: Task,\n    config_commands: Optional[List[str]] = None,\n    config_file: Optional[str] = None,\n    **kwargs: Any\n) -> Result:\n    \"\"\"\n    Execute Netmiko send_config_set method (or send_config_from_file)\n\n    Arguments:\n        config_commands: Commands to configure on the remote network device.\n        config_file: File to read configuration commands from.\n        kwargs: Additional arguments to pass to method.\n\n    Returns:\n        Result object with the following attributes set:\n          * result (``str``): string showing the CLI from the configuration changes.\n    \"\"\"\n    net_connect = task.host.get_connection(\"netmiko\", task.nornir.config)\n    net_connect.enable()\n    if config_commands:\n        result = net_connect.send_config_set(config_commands=config_commands, **kwargs)\n    elif config_file:\n        result = net_connect.send_config_from_file(config_file=config_file, **kwargs)\n    else:\n        raise ValueError(\"Must specify either config_commands or config_file\")\n\n    return Result(host=task.host, result=result, changed=True)", "response": "Execute Netmiko send_config_set or send_config_from_file method on the remote network device."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef napalm_configure(\n    task: Task,\n    dry_run: Optional[bool] = None,\n    filename: Optional[str] = None,\n    configuration: Optional[str] = None,\n    replace: bool = False,\n) -> Result:\n    \"\"\"\n    Loads configuration into a network devices using napalm\n\n    Arguments:\n        dry_run: Whether to apply changes or not\n        filename: filename containing the configuration to load into the device\n        configuration: configuration to load into the device\n        replace: whether to replace or merge the configuration\n\n    Returns:\n        Result object with the following attributes set:\n          * changed (``bool``): whether the task is changing the system or not\n          * diff (``string``): change in the system\n    \"\"\"\n    device = task.host.get_connection(\"napalm\", task.nornir.config)\n\n    if replace:\n        device.load_replace_candidate(filename=filename, config=configuration)\n    else:\n        device.load_merge_candidate(filename=filename, config=configuration)\n    diff = device.compare_config()\n\n    dry_run = task.is_dry_run(dry_run)\n    if not dry_run and diff:\n        device.commit_config()\n    else:\n        device.discard_config()\n    return Result(host=task.host, diff=diff, changed=len(diff) > 0)", "response": "Configure the nornir system with the given configuration."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef load_json(task: Task, file: str) -> Result:\n    kwargs: Dict[str, Type[MutableMapping[str, Any]]] = {}\n    with open(file, \"r\") as f:\n        data = json.loads(f.read(), **kwargs)\n\n    return Result(host=task.host, result=data)", "response": "Loads a json file."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nprint the result of a previous task to screen.", "response": "def print_result(\n    result: Result,\n    host: Optional[str] = None,\n    vars: List[str] = None,\n    failed: bool = False,\n    severity_level: int = logging.INFO,\n) -> None:\n    \"\"\"\n    Prints the :obj:`nornir.core.task.Result` from a previous task to screen\n\n    Arguments:\n        result: from a previous task\n        host: # TODO\n        vars: Which attributes you want to print\n        failed: if ``True`` assume the task failed\n        severity_level: Print only errors with this severity level or higher\n    \"\"\"\n    LOCK.acquire()\n    try:\n        _print_result(result, host, vars, failed, severity_level)\n    finally:\n        LOCK.release()"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nexecutes a command locally and returns the result.", "response": "def command(task: Task, command: str) -> Result:\n    \"\"\"\n    Executes a command locally\n\n    Arguments:\n        command: command to execute\n\n    Returns:\n        Result object with the following attributes set:\n          * result (``str``): stderr or stdout\n          * stdout (``str``): stdout\n          * stderr (``str``): stderr\n\n    Raises:\n        :obj:`nornir.core.exceptions.CommandError`: when there is a command error\n    \"\"\"\n    cmd = subprocess.Popen(\n        shlex.split(command),\n        stdout=subprocess.PIPE,\n        stderr=subprocess.PIPE,\n        shell=False,\n    )\n\n    stdout, stderr = cmd.communicate()\n    stdout = stdout.decode()\n    stderr = stderr.decode()\n\n    if cmd.poll():\n        raise CommandError(command, cmd.returncode, stdout, stderr)\n\n    result = stderr if stderr else stdout\n    return Result(result=result, host=task.host, stderr=stderr, stdout=stdout)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef netmiko_file_transfer(\n    task: Task, source_file: str, dest_file: str, **kwargs: Any\n) -> Result:\n    \"\"\"\n    Execute Netmiko file_transfer method\n\n    Arguments:\n        source_file: Source file.\n        dest_file: Destination file.\n        kwargs: Additional arguments to pass to file_transfer\n\n    Returns:\n        Result object with the following attributes set:\n          * result (``bool``): file exists and MD5 is valid\n          * changed (``bool``): the destination file was changed\n\n    \"\"\"\n    net_connect = task.host.get_connection(\"netmiko\", task.nornir.config)\n    kwargs.setdefault(\"direction\", \"put\")\n    scp_result = file_transfer(\n        net_connect, source_file=source_file, dest_file=dest_file, **kwargs\n    )\n    if kwargs.get(\"disable_md5\") is True:\n        file_valid = scp_result[\"file_exists\"]\n    else:\n        file_valid = scp_result[\"file_exists\"] and scp_result[\"file_verified\"]\n    return Result(\n        host=task.host, result=file_valid, changed=scp_result[\"file_transferred\"]\n    )", "response": "Execute Netmiko file_transfer method on the node with optional arguments."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nruns commands on remote devices using napalm", "response": "def napalm_cli(task: Task, commands: List[str]) -> Result:\n    \"\"\"\n    Run commands on remote devices using napalm\n\n    Arguments:\n        commands: commands to execute\n\n    Returns:\n        Result object with the following attributes set:\n          * result (``dict``): result of the commands execution\n    \"\"\"\n    device = task.host.get_connection(\"napalm\", task.nornir.config)\n    result = device.cli(commands)\n    return Result(host=task.host, result=result)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef filter(self, *args, **kwargs):\n        b = Nornir(**self.__dict__)\n        b.inventory = self.inventory.filter(*args, **kwargs)\n        return b", "response": "Returns a new object with same configuration as self but filtered inventory."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef run(\n        self,\n        task,\n        num_workers=None,\n        raise_on_error=None,\n        on_good=True,\n        on_failed=False,\n        **kwargs,\n    ):\n        \"\"\"\n        Run task over all the hosts in the inventory.\n\n        Arguments:\n            task (``callable``): function or callable that will be run against each device in\n              the inventory\n            num_workers(``int``): Override for how many hosts to run in paralell for this task\n            raise_on_error (``bool``): Override raise_on_error behavior\n            on_good(``bool``): Whether to run or not this task on hosts marked as good\n            on_failed(``bool``): Whether to run or not this task on hosts marked as failed\n            **kwargs: additional argument to pass to ``task`` when calling it\n\n        Raises:\n            :obj:`nornir.core.exceptions.NornirExecutionError`: if at least a task fails\n              and self.config.core.raise_on_error is set to ``True``\n\n        Returns:\n            :obj:`nornir.core.task.AggregatedResult`: results of each execution\n        \"\"\"\n        num_workers = num_workers or self.config.core.num_workers\n\n        run_on = []\n        if on_good:\n            for name, host in self.inventory.hosts.items():\n                if name not in self.data.failed_hosts:\n                    run_on.append(host)\n        if on_failed:\n            for name, host in self.inventory.hosts.items():\n                if name in self.data.failed_hosts:\n                    run_on.append(host)\n\n        num_hosts = len(self.inventory.hosts)\n        task_name = kwargs.get(\"name\") or task.__name__\n        if num_hosts:\n            logger.info(\n                f\"Running task %r with args %s on %d hosts\",\n                task_name,\n                kwargs,\n                num_hosts,\n            )\n        else:\n            logger.warning(\"Task %r has not been run \u2013 0 hosts selected\", task_name)\n\n        if num_workers == 1:\n            result = self._run_serial(task, run_on, **kwargs)\n        else:\n            result = self._run_parallel(task, run_on, num_workers, **kwargs)\n\n        raise_on_error = (\n            raise_on_error\n            if raise_on_error is not None\n            else self.config.core.raise_on_error\n        )  # noqa\n        if raise_on_error:\n            result.raise_on_error()\n        else:\n            self.data.failed_hosts.update(result.failed_hosts.keys())\n        return result", "response": "Runs a task over all the hosts in the inventory."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nexposes some of the Gitlab API functionality for operations on files in a Gitlab repository. Example: nornir.run(files.gitlab, action=\"create\", url=\"https://gitlab.localhost.com\", token=\"ABCD1234\", repository=\"test\", filename=\"config\", ref=\"master\") Arguments: dry_run: Whether to apply changes or not url: Gitlab instance URL token: Personal access token repository: source/destination repository filename: source/destination file name content: content to write action: ``create``, ``update``, ``get`` branch: destination branch destination: local destination filename (only used in get action) ref: branch, commit hash or tag (only used in get action) commit_message: commit message Returns: Result object with the following attributes set: * changed (``bool``): * diff (``str``): unified diff", "response": "def gitlab(\n    task: Task,\n    url: str,\n    token: str,\n    repository: str,\n    filename: str,\n    content: str = \"\",\n    action: str = \"create\",\n    dry_run: bool = False,\n    branch: str = \"master\",\n    destination: str = \"\",\n    ref: str = \"master\",\n    commit_message: str = \"\",\n) -> Result:\n    \"\"\"\n    Exposes some of the Gitlab API functionality for operations on files\n    in a Gitlab repository.\n\n    Example:\n\n        nornir.run(files.gitlab,\n                   action=\"create\",\n                   url=\"https://gitlab.localhost.com\",\n                   token=\"ABCD1234\",\n                   repository=\"test\",\n                   filename=\"config\",\n                   ref=\"master\")\n\n    Arguments:\n        dry_run: Whether to apply changes or not\n        url: Gitlab instance URL\n        token: Personal access token\n        repository: source/destination repository\n        filename: source/destination file name\n        content: content to write\n        action: ``create``, ``update``, ``get``\n        branch: destination branch\n        destination: local destination filename (only used in get action)\n        ref: branch, commit hash or tag (only used in get action)\n        commit_message: commit message\n\n    Returns:\n        Result object with the following attributes set:\n            * changed (``bool``):\n            * diff (``str``): unified diff\n\n    \"\"\"\n    dry_run = task.is_dry_run(dry_run)\n\n    session = requests.session()\n    session.headers.update({\"PRIVATE-TOKEN\": token})\n\n    if commit_message == \"\":\n        commit_message = \"File created with nornir\"\n\n    pid = _get_repository(session, url, repository)\n\n    if action == \"create\":\n        diff = _create(\n            task, session, url, pid, filename, content, branch, commit_message, dry_run\n        )\n    elif action == \"update\":\n        diff = _update(\n            task, session, url, pid, filename, content, branch, commit_message, dry_run\n        )\n    elif action == \"get\":\n        diff = _get(task, session, url, pid, filename, destination, ref, dry_run)\n    return Result(host=task.host, diff=diff, changed=bool(diff))"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef failed_hosts(self) -> Dict[str, \"MultiResult\"]:\n        return {k: v for k, v in self.result.items() if v.failed}", "response": "Returns a dict of hosts that failed to complete the task."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef template_string(\n    task: Task, template: str, jinja_filters: FiltersDict = None, **kwargs: Any\n) -> Result:\n    \"\"\"\n    Renders a string with jinja2. All the host data is available in the template\n\n    Arguments:\n        template (string): template string\n        jinja_filters (dict): jinja filters to enable. Defaults to nornir.config.jinja2.filters\n        **kwargs: additional data to pass to the template\n\n    Returns:\n        Result object with the following attributes set:\n          * result (``string``): rendered string\n    \"\"\"\n    jinja_filters = jinja_filters or {} or task.nornir.config.jinja2.filters\n    text = jinja_helper.render_from_string(\n        template=template, host=task.host, jinja_filters=jinja_filters, **kwargs\n    )\n    return Result(host=task.host, result=text)", "response": "Renders a string with jinja2."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef build_configuration_parameters(app):\n    env = Environment(loader=FileSystemLoader(\"{0}/_data_templates\".format(BASEPATH)))\n    template_file = env.get_template(\"configuration-parameters.j2\")\n    data = {}\n    data[\"schema\"] = Config.schema()\n    rendered_template = template_file.render(**data)\n    output_dir = \"{0}/configuration/generated\".format(BASEPATH)\n    with open(\"{}/parameters.rst\".format(output_dir), \"w\") as f:\n        f.write(rendered_template)", "response": "Create documentation for configuration parameters."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef setup(app):\n    app.connect(\"builder-inited\", build_configuration_parameters)\n    app.connect(\"autodoc-skip-member\", skip_slots)\n    app.add_stylesheet(\"css/custom.css\")", "response": "Map methods to states of the documentation build."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef load_yaml(task: Task, file: str) -> Result:\n    with open(file, \"r\") as f:\n        yml = ruamel.yaml.YAML(typ=\"safe\")\n        data = yml.load(f)\n\n    return Result(host=task.host, result=data)", "response": "Load a yaml file."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef tcp_ping(\n    task: Task, ports: List[int], timeout: int = 2, host: Optional[str] = None\n) -> Result:\n    \"\"\"\n    Tests connection to a tcp port and tries to establish a three way\n    handshake. To be used for network discovery or testing.\n\n    Arguments:\n        ports (list of int): tcp ports to ping\n        timeout (int, optional): defaults to 2\n        host (string, optional): defaults to ``hostname``\n\n\n    Returns:\n        Result object with the following attributes set:\n          * result (``dict``): Contains port numbers as keys with True/False as values\n    \"\"\"\n\n    if isinstance(ports, int):\n        ports = [ports]\n\n    if isinstance(ports, list):\n        if not all(isinstance(port, int) for port in ports):\n            raise ValueError(\"Invalid value for 'ports'\")\n\n    else:\n        raise ValueError(\"Invalid value for 'ports'\")\n\n    host = host or task.host.hostname\n\n    result = {}\n    for port in ports:\n        s = socket.socket()\n        s.settimeout(timeout)\n        try:\n            status = s.connect_ex((host, port))\n            if status == 0:\n                connection = True\n            else:\n                connection = False\n        except (socket.gaierror, socket.timeout, socket.error):\n            connection = False\n        finally:\n            s.close()\n        result[port] = connection\n\n    return Result(host=task.host, result=result)", "response": "Tests connection to a tcp port and attempts to establish a three way TCP connection."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ngathers information from network devices using napalm", "response": "def napalm_get(\n    task: Task,\n    getters: List[str],\n    getters_options: GetterOptionsDict = None,\n    **kwargs: Any\n) -> Result:\n    \"\"\"\n    Gather information from network devices using napalm\n\n    Arguments:\n        getters: getters to use\n        getters_options (dict of dicts): When passing multiple getters you\n            pass a dictionary where the outer key is the getter name\n            and the included dictionary represents the options to pass\n            to the getter\n        **kwargs: will be passed as they are to the getters\n\n    Examples:\n\n        Simple example::\n\n            > nr.run(task=napalm_get,\n            >        getters=[\"interfaces\", \"facts\"])\n\n        Passing options using ``**kwargs``::\n\n            > nr.run(task=napalm_get,\n            >        getters=[\"config\"],\n            >        retrieve=\"all\")\n\n        Passing options using ``getters_options``::\n\n            > nr.run(task=napalm_get,\n            >        getters=[\"config\", \"interfaces\"],\n            >        getters_options={\"config\": {\"retrieve\": \"all\"}})\n\n    Returns:\n        Result object with the following attributes set:\n          * result (``dict``): dictionary with the result of the getter\n    \"\"\"\n    device = task.host.get_connection(\"napalm\", task.nornir.config)\n    getters_options = getters_options or {}\n\n    if isinstance(getters, str):\n        getters = [getters]\n\n    result = {}\n    for g in getters:\n        options = copy.deepcopy(kwargs)\n        options.update(getters_options.get(g, {}))\n        getter = g if g.startswith(\"get_\") else \"get_{}\".format(g)\n        method = getattr(device, getter)\n        result[g] = method(**options)\n    return Result(host=task.host, result=result)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nexecuting a command remotely on the host", "response": "def remote_command(task: Task, command: str) -> Result:\n    \"\"\"\n    Executes a command remotely on the host\n\n    Arguments:\n        command (``str``): command to execute\n\n    Returns:\n        Result object with the following attributes set:\n          * result (``str``): stderr or stdout\n          * stdout (``str``): stdout\n          * stderr (``str``): stderr\n\n    Raises:\n        :obj:`nornir.core.exceptions.CommandError`: when there is a command error\n    \"\"\"\n    client = task.host.get_connection(\"paramiko\", task.nornir.config)\n    connection_state = task.host.get_connection_state(\"paramiko\")\n\n    chan = client.get_transport().open_session()\n\n    if connection_state[\"ssh_forward_agent\"]:\n        AgentRequestHandler(chan)\n\n    chan.exec_command(command)\n\n    with chan.makefile() as f:\n        stdout = f.read().decode()\n    with chan.makefile_stderr() as f:\n        stderr = f.read().decode()\n\n    exit_status_code = chan.recv_exit_status()\n\n    if exit_status_code:\n        raise CommandError(command, exit_status_code, stdout, stderr)\n\n    result = stderr if stderr else stdout\n    return Result(result=result, host=task.host, stderr=stderr, stdout=stdout)"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nopens the connection to the device and populate the attribute connection with the underlying connection.", "response": "def open(\n        self,\n        hostname: Optional[str],\n        username: Optional[str],\n        password: Optional[str],\n        port: Optional[int],\n        platform: Optional[str],\n        extras: Optional[Dict[str, Any]] = None,\n        configuration: Optional[Config] = None,\n    ) -> None:\n        \"\"\"\n        Connect to the device and populate the attribute :attr:`connection` with\n        the underlying connection\n        \"\"\"\n        pass"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nregistering a connection plugin with a specified name.", "response": "def register(cls, name: str, plugin: Type[ConnectionPlugin]) -> None:\n        \"\"\"Registers a connection plugin with a specified name\n\n        Args:\n            name: name of the connection plugin to register\n            plugin: defined connection plugin class\n\n        Raises:\n            :obj:`nornir.core.exceptions.ConnectionPluginAlreadyRegistered` if\n                another plugin with the specified name was already registered\n        \"\"\"\n        existing_plugin = cls.available.get(name)\n        if existing_plugin is None:\n            cls.available[name] = plugin\n        elif existing_plugin != plugin:\n            raise ConnectionPluginAlreadyRegistered(\n                f\"Connection plugin {plugin.__name__} can't be registered as \"\n                f\"{name!r} because plugin {existing_plugin.__name__} \"\n                f\"was already registered under this name\"\n            )"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef deregister(cls, name: str) -> None:\n        if name not in cls.available:\n            raise ConnectionPluginNotRegistered(\n                f\"Connection {name!r} is not registered\"\n            )\n        cls.available.pop(name)", "response": "Deregisters a registered connection plugin by its name."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_plugin(cls, name: str) -> Type[ConnectionPlugin]:\n        if name not in cls.available:\n            raise ConnectionPluginNotRegistered(\n                f\"Connection {name!r} is not registered\"\n            )\n        return cls.available[name]", "response": "Fetches the connection plugin by name Raises ConnectionPluginNotRegistered if the connection plugin is not registered"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ninitialize a new Nornir object.", "response": "def InitNornir(\n    config_file: str = \"\",\n    dry_run: bool = False,\n    configure_logging: Optional[bool] = None,\n    **kwargs: Dict[str, Any],\n) -> Nornir:\n    \"\"\"\n    Arguments:\n        config_file(str): Path to the configuration file (optional)\n        dry_run(bool): Whether to simulate changes or not\n        configure_logging: Whether to configure logging or not. This argument is being\n            deprecated. Please use logging.enabled parameter in the configuration\n            instead.\n        **kwargs: Extra information to pass to the\n            :obj:`nornir.core.configuration.Config` object\n\n    Returns:\n        :obj:`nornir.core.Nornir`: fully instantiated and configured\n    \"\"\"\n    register_default_connection_plugins()\n\n    if callable(kwargs.get(\"inventory\", {}).get(\"plugin\", \"\")):\n        kwargs[\"inventory\"][\"plugin\"] = cls_to_string(kwargs[\"inventory\"][\"plugin\"])\n\n    if callable(kwargs.get(\"inventory\", {}).get(\"transform_function\", \"\")):\n        kwargs[\"inventory\"][\"transform_function\"] = cls_to_string(\n            kwargs[\"inventory\"][\"transform_function\"]\n        )\n\n    conf = Config.load_from_file(config_file, **kwargs)\n\n    data = GlobalState(dry_run=dry_run)\n\n    if configure_logging is not None:\n        msg = (\n            \"'configure_logging' argument is deprecated, please use \"\n            \"'logging.enabled' parameter in the configuration instead: \"\n            \"https://nornir.readthedocs.io/en/stable/configuration/index.html\"\n        )\n        warnings.warn(msg, DeprecationWarning)\n\n    if conf.logging.enabled is None:\n        if configure_logging is not None:\n            conf.logging.enabled = configure_logging\n        else:\n            conf.logging.enabled = True\n\n    conf.logging.configure()\n\n    inv = conf.inventory.plugin.deserialize(\n        transform_function=conf.inventory.transform_function,\n        transform_function_options=conf.inventory.transform_function_options,\n        config=conf,\n        **conf.inventory.options,\n    )\n\n    return Nornir(inventory=inv, config=conf, data=data)"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nwrites contents to a file", "response": "def write_file(\n    task: Task,\n    filename: str,\n    content: str,\n    append: bool = False,\n    dry_run: Optional[bool] = None,\n) -> Result:\n    \"\"\"\n    Write contents to a file (locally)\n\n    Arguments:\n        dry_run: Whether to apply changes or not\n        filename: file you want to write into\n        content: content you want to write\n        append: whether you want to replace the contents or append to it\n\n    Returns:\n        Result object with the following attributes set:\n          * changed (``bool``):\n          * diff (``str``): unified diff\n    \"\"\"\n    diff = _generate_diff(filename, content, append)\n\n    if not task.is_dry_run(dry_run):\n        mode = \"a+\" if append else \"w+\"\n        with open(filename, mode=mode) as f:\n            f.write(content)\n\n    return Result(host=task.host, diff=diff, changed=bool(diff))"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef load_cli_args(self):\n        parser = argparse.ArgumentParser(description='QTPyLib Reporting',\n                                         formatter_class=argparse.ArgumentDefaultsHelpFormatter)\n\n        parser.add_argument('--port', default=self.args[\"port\"],\n                            help='HTTP port to use', type=int)\n        parser.add_argument('--host', default=self.args[\"host\"],\n                            help='Host to bind the http process to')\n        parser.add_argument('--blotter',\n                            help='Use this Blotter\\'s MySQL server settings')\n        parser.add_argument('--nopass',\n                            help='Skip password for web app (flag)',\n                            action='store_true')\n\n        # only return non-default cmd line args\n        # (meaning only those actually given)\n        cmd_args, _ = parser.parse_known_args()\n        args = {arg: val for arg, val in vars(\n            cmd_args).items() if val != parser.get_default(arg)}\n        return args", "response": "Parse command line arguments and return only the non - default ones"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef run(self):\n\n        # -----------------------------------\n        # assign view\n        app.add_url_rule('/', 'index', view_func=self.index)\n        app.add_url_rule('/<path:start>', 'index', view_func=self.index)\n        app.add_url_rule('/<start>/<path:end>', 'index', view_func=self.index)\n\n        app.add_url_rule('/algos', 'algos', view_func=self.algos)\n        app.add_url_rule('/symbols', 'symbols', view_func=self.symbols)\n\n        app.add_url_rule('/positions', 'positions', view_func=self.positions)\n        app.add_url_rule('/positions/<path:algo_id>',\n                         'positions', view_func=self.positions)\n\n        app.add_url_rule('/algo/<path:algo_id>',\n                         'trades_by_algo', view_func=self.trades_by_algo)\n        app.add_url_rule('/algo/<algo_id>/<path:start>',\n                         'trades_by_algo', view_func=self.trades_by_algo)\n        app.add_url_rule('/algo/<algo_id>/<start>/<path:end>',\n                         'trades_by_algo', view_func=self.trades_by_algo)\n\n        app.add_url_rule('/bars/<resolution>/<symbol>',\n                         'bars', view_func=self.bars)\n        app.add_url_rule(\n            '/bars/<resolution>/<symbol>/<path:start>', 'bars', view_func=self.bars)\n        app.add_url_rule('/bars/<resolution>/<symbol>/<start>/<path:end>',\n                         'bars', view_func=self.bars)\n\n        app.add_url_rule('/trades', 'trades', view_func=self.trades)\n        app.add_url_rule('/trades/<path:start>',\n                         'trades', view_func=self.trades)\n        app.add_url_rule('/trades/<start>/<path:end>',\n                         'trades', view_func=self.trades)\n        app.add_url_rule('/login/<password>', 'login', view_func=self.login)\n        app.add_url_rule('/static/<url_path>', 'send_static',\n                         view_func=self.send_static)\n\n        # let user know what the temp password is\n        if 'nopass' not in self.args and self._password != \"\":\n            print(\" * Web app password is:\", self._password)\n\n        # notice\n        # print(\" * Running on http://\"+ str(self.host) +\":\"+str(self.port)+\"/ (CTRL+C to quit)\")\n\n        # -----------------------------------\n        # run flask app\n        app.run(\n            debug=True,\n            host=str(self.host),\n            port=int(self.port)\n        )", "response": "Starts the reporting module."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef add_instruments(self, *instruments):\n        for instrument in instruments:\n            if isinstance(instrument, ezibpy.utils.Contract):\n                instrument = self.ibConn.contract_to_tuple(instrument)\n                contractString = self.ibConn.contractString(instrument)\n                self.instruments[contractString] = instrument\n                self.ibConn.createContract(instrument)\n\n        self.symbols = list(self.instruments.keys())", "response": "add instruments after initialization"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nadds contracts to groups", "response": "def register_combo(self, parent, legs):\n        \"\"\" add contracts to groups \"\"\"\n        parent = self.ibConn.contractString(parent)\n        legs_dict = {}\n        for leg in legs:\n            leg = self.ibConn.contractString(leg)\n            legs_dict[leg] = self.get_instrument(leg)\n        self.instrument_combos[parent] = legs_dict"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nget group by child symbol", "response": "def get_combo(self, symbol):\n        \"\"\" get group by child symbol \"\"\"\n        for parent, legs in self.instrument_combos.items():\n            if symbol == parent or symbol in legs.keys():\n                return {\n                    \"parent\": self.get_instrument(parent),\n                    \"legs\": legs,\n                }\n        return {\n            \"parent\": None,\n            \"legs\": {},\n        }"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _register_trade(self, order):\n        if order['id'] in self.orders.recent:\n            orderId = order['id']\n        else:\n            orderId = order['parentId']\n        # entry / exit?\n        symbol = order[\"symbol\"]\n        order_data = self.orders.recent[orderId]\n        position = self.get_positions(symbol)['position']\n\n        if position != 0:\n            # entry\n            order_data['action'] = \"ENTRY\"\n            order_data['position'] = position\n            order_data['entry_time'] = tools.datetime_to_timezone(\n                order['time'])\n            order_data['exit_time'] = None\n            order_data['entry_order'] = order_data['order_type']\n            order_data['entry_price'] = order['avgFillPrice']\n            order_data['exit_price'] = 0\n            order_data['exit_reason'] = None\n\n        else:\n            order_data['action'] = \"EXIT\"\n            order_data['position'] = 0\n            order_data['exit_time'] = tools.datetime_to_timezone(order['time'])\n            order_data['exit_price'] = order['avgFillPrice']\n\n            # target / stop?\n            if order['id'] == order_data['targetOrderId']:\n                order_data['exit_reason'] = \"TARGET\"\n            elif order['id'] == order_data['stopOrderId']:\n                order_data['exit_reason'] = \"STOP\"\n            else:\n                order_data['exit_reason'] = \"SIGNAL\"\n\n            # remove from collection\n            del self.orders.recent[orderId]\n\n        if order_data is None:\n            return None\n\n        # trade identifier\n        tradeId = self.strategy.upper() + '_' + symbol.upper()\n        tradeId = hashlib.sha1(tradeId.encode()).hexdigest()\n\n        # existing trade?\n        if tradeId not in self.active_trades:\n            self.active_trades[tradeId] = {\n                \"strategy\": self.strategy,\n                \"action\": order_data['action'],\n                \"quantity\": abs(order_data['position']),\n                \"position\": order_data['position'],\n                \"symbol\": order_data[\"symbol\"].split('_')[0],\n                \"direction\": order_data['direction'],\n                \"entry_time\": None,\n                \"exit_time\": None,\n                \"duration\": \"0s\",\n                \"exit_reason\": order_data['exit_reason'],\n                \"order_type\": order_data['order_type'],\n                \"market_price\": order_data['price'],\n                \"target\": order_data['target'],\n                \"stop\": order_data['initial_stop'],\n                \"entry_price\": 0,\n                \"exit_price\": order_data['exit_price'],\n                \"realized_pnl\": 0\n            }\n            if \"entry_time\" in order_data:\n                self.active_trades[tradeId][\"entry_time\"] = order_data['entry_time']\n            if \"entry_price\" in order_data:\n                self.active_trades[tradeId][\"entry_price\"] = order_data['entry_price']\n        else:\n            # self.active_trades[tradeId]['direction']   = order_data['direction']\n            self.active_trades[tradeId]['action'] = order_data['action']\n            self.active_trades[tradeId]['position'] = order_data['position']\n            self.active_trades[tradeId]['exit_price'] = order_data['exit_price']\n            self.active_trades[tradeId]['exit_reason'] = order_data['exit_reason']\n            self.active_trades[tradeId]['exit_time'] = order_data['exit_time']\n\n            # calculate trade duration\n            try:\n                delta = int((self.active_trades[tradeId]['exit_time'] -\n                             self.active_trades[tradeId]['entry_time']).total_seconds())\n                days, remainder = divmod(delta, 86400)\n                hours, remainder = divmod(remainder, 3600)\n                minutes, seconds = divmod(remainder, 60)\n                duration = ('%sd %sh %sm %ss' %\n                            (days, hours, minutes, seconds))\n                self.active_trades[tradeId]['duration'] = duration.replace(\n                    \"0d \", \"\").replace(\"0h \", \"\").replace(\"0m \", \"\")\n            except Exception as e:\n                pass\n\n            trade = self.active_trades[tradeId]\n            if trade['entry_price'] > 0 and trade['position'] == 0:\n                if trade['direction'] == \"SELL\":\n                    pnl = trade['entry_price'] - trade['exit_price']\n                else:\n                    pnl = trade['exit_price'] - trade['entry_price']\n\n                pnl = tools.to_decimal(pnl)\n                # print(\"1)\", pnl)\n                self.active_trades[tradeId]['realized_pnl'] = pnl\n\n        # print(\"\\n\\n-----------------\")\n        # print(self.active_trades[tradeId])\n        # print(\"-----------------\\n\\n\")\n\n        # get trade\n        trade = self.active_trades[tradeId].copy()\n\n        # sms trades\n        sms._send_trade(trade, self.sms_numbers, self.timezone)\n\n        # rename trade direction\n        trade['direction'] = trade['direction'].replace(\n            \"BUY\", \"LONG\").replace(\"SELL\", \"SHORT\")\n\n        # log\n        self.log_trade(trade)\n\n        # remove from active trades and add to trade\n        if trade['action'] == \"EXIT\":\n            del self.active_trades[tradeId]\n            self.trades.append(trade)\n\n        # return trade\n        return trade", "response": "registers a new trade in the database"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _cancel_orphan_orders(self, orderId):\n        orders = self.ibConn.orders\n        for order in orders:\n            order = orders[order]\n            if order['parentId'] != orderId:\n                self.ibConn.cancelOrder(order['id'])", "response": "cancel orphan orders when parent is gone"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncancel expired pending orders", "response": "def _cancel_expired_pending_orders(self):\n        \"\"\" expires pending orders \"\"\"\n        # use a copy to prevent errors\n        pending = self.orders.pending.copy()\n        for symbol in pending:\n            orderId = pending[symbol][\"orderId\"]\n            expiration = pending[symbol][\"expires\"]\n\n            delta = expiration - datetime.now()\n            delta = self._milliseconds_delta(delta)\n\n            # cancel order if expired\n            if delta < 0:\n                self.ibConn.cancelOrder(orderId)\n                if orderId in self.orders.pending_ttls:\n                    if orderId in self.orders.pending_ttls:\n                        del self.orders.pending_ttls[orderId]\n                    if symbol in self.orders.pending:\n                        if self.orders.pending[symbol]['orderId'] == orderId:\n                            del self.orders.pending[symbol]"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a new Instrument object for the given symbol.", "response": "def get_instrument(self, symbol):\n        \"\"\"\n        A string subclass that provides easy access to misc\n        symbol-related methods and information using shorthand.\n        Refer to the `Instruments API <#instrument-api>`_\n        for available methods and properties\n\n        Call from within your strategy:\n        ``instrument = self.get_instrument(\"SYMBOL\")``\n\n        :Parameters:\n            symbol : string\n                instrument symbol\n\n        \"\"\"\n        instrument = Instrument(self.get_symbol(symbol))\n        instrument._set_parent(self)\n        instrument._set_windows(ticks=self.tick_window, bars=self.bar_window)\n\n        return instrument"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef run(self):\n        if self.init_sec:\n            sleep(self.init_sec)\n        self._functime = time()\n        while self._running:\n            start = time()\n            self._func()\n            self._functime += self.interval_sec\n            if self._functime - start > 0:\n                sleep(self._functime - start)", "response": "Start the recurring task."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nload running blotter s arguments.", "response": "def load_blotter_args(blotter_name=None, logger=None):\n    \"\"\" Load running blotter's settings (used by clients)\n\n    :Parameters:\n        blotter_name : str\n            Running Blotter's name (defaults to \"auto-detect\")\n        logger : object\n            Logger to be use (defaults to Blotter's)\n\n    :Returns:\n        args : dict\n            Running Blotter's arguments\n    \"\"\"\n    if logger is None:\n        logger = tools.createLogger(__name__, logging.WARNING)\n\n    # find specific name\n    if blotter_name is not None:  # and blotter_name != 'auto-detect':\n        args_cache_file = tempfile.gettempdir() + \"/\" + blotter_name.lower() + \".qtpylib\"\n        if not os.path.exists(args_cache_file):\n            logger.critical(\n                \"Cannot connect to running Blotter [%s]\", blotter_name)\n            if os.isatty(0):\n                sys.exit(0)\n            return []\n\n    # no name provided - connect to last running\n    else:\n        blotter_files = sorted(\n            glob.glob(tempfile.gettempdir() + \"/*.qtpylib\"), key=os.path.getmtime)\n\n        if not blotter_files:\n            logger.critical(\n                \"Cannot connect to running Blotter [%s]\", blotter_name)\n            if os.isatty(0):\n                sys.exit(0)\n            return []\n\n        args_cache_file = blotter_files[-1]\n\n    args = pickle.load(open(args_cache_file, \"rb\"))\n    args['as_client'] = True\n\n    return args"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_symbol_id(symbol, dbconn, dbcurr, ibConn=None):\n    def _get_contract_expiry(symbol, ibConn=None):\n        # parse w/p ibConn\n        if ibConn is None or isinstance(symbol, str):\n            return tools.contract_expiry_from_symbol(symbol)\n\n        # parse with ibConn\n        contract_details = ibConn.contractDetails(symbol)[\"m_summary\"]\n        if contract_details[\"m_expiry\"] == \"\":\n            ibConn.createContract(symbol)\n            return _get_contract_expiry(symbol, ibConn)\n\n        if contract_details[\"m_expiry\"]:\n            return datetime.strptime(str(contract_details[\"m_expiry\"]), '%Y%m%d'\n                                     ).strftime(\"%Y-%m-%d\")\n\n        return contract_details[\"m_expiry\"]\n\n    # start\n    asset_class = tools.gen_asset_class(symbol)\n    symbol_group = tools.gen_symbol_group(symbol)\n    clean_symbol = symbol.replace(\"_\" + asset_class, \"\")\n    expiry = None\n\n    if asset_class in (\"FUT\", \"OPT\", \"FOP\"):\n        expiry = _get_contract_expiry(symbol, ibConn)\n\n        # look for symbol w/ expiry\n        sql = \"\"\"SELECT id FROM `symbols` WHERE\n            `symbol`=%s AND `symbol_group`=%s AND `asset_class`=%s  AND `expiry`=%s LIMIT 1\"\"\"\n        dbcurr.execute(sql, (clean_symbol, symbol_group, asset_class, expiry))\n\n    else:\n        # look for symbol w/o expiry\n        sql = \"\"\"SELECT id FROM `symbols` WHERE\n            `symbol`=%s AND `symbol_group`=%s AND `asset_class`=%s LIMIT 1\"\"\"\n        dbcurr.execute(sql, (clean_symbol, symbol_group, asset_class))\n\n    row = dbcurr.fetchone()\n\n    # symbol already in db\n    if row is not None:\n        return row[0]\n\n    # symbol/expiry not in db... insert new/update expiry\n    else:\n        # need to update the expiry?\n        if expiry is not None:\n            sql = \"\"\"SELECT id FROM `symbols` WHERE\n                `symbol`=%s AND `symbol_group`=%s AND `asset_class`=%s LIMIT 1\"\"\"\n            dbcurr.execute(sql, (clean_symbol, symbol_group, asset_class))\n\n            row = dbcurr.fetchone()\n            if row is not None:\n                sql = \"UPDATE `symbols` SET `expiry`='\" + \\\n                    str(expiry) + \"' WHERE id=\" + str(row[0])\n                dbcurr.execute(sql)\n                try:\n                    dbconn.commit()\n                except Exception as e:\n                    return False\n                return int(row[0])\n\n        # insert new symbol\n        sql = \"\"\"INSERT IGNORE INTO `symbols`\n            (`symbol`, `symbol_group`, `asset_class`, `expiry`) VALUES (%s, %s, %s, %s)\n            ON DUPLICATE KEY UPDATE `symbol`=`symbol`, `expiry`=%s\n        \"\"\"\n\n        dbcurr.execute(sql, (clean_symbol, symbol_group,\n                             asset_class, expiry, expiry))\n        try:\n            dbconn.commit()\n        except Exception as e:\n            return False\n\n        return dbcurr.lastrowid", "response": "Get the ID of the given symbol from the Database or create it if it doesn t exist."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef run(self):\n\n        self._check_unique_blotter()\n\n        # connect to mysql\n        self.mysql_connect()\n\n        self.context = zmq.Context(zmq.REP)\n        self.socket = self.context.socket(zmq.PUB)\n        self.socket.bind(\"tcp://*:\" + str(self.args['zmqport']))\n\n        db_modified = 0\n        contracts = []\n        prev_contracts = []\n        first_run = True\n\n        self.log_blotter.info(\"Connecting to Interactive Brokers...\")\n        self.ibConn = ezIBpy()\n        self.ibConn.ibCallback = self.ibCallback\n\n        while not self.ibConn.connected:\n            self.ibConn.connect(clientId=int(self.args['ibclient']),\n                                port=int(self.args['ibport']), host=str(self.args['ibserver']))\n            time.sleep(1)\n            if not self.ibConn.connected:\n                print('*', end=\"\", flush=True)\n        self.log_blotter.info(\"Connection established...\")\n\n        try:\n            while True:\n\n                if not os.path.exists(self.args['symbols']):\n                    pd.DataFrame(columns=['symbol', 'sec_type', 'exchange',\n                                          'currency', 'expiry', 'strike', 'opt_type']\n                                 ).to_csv(self.args['symbols'], header=True, index=False)\n                    tools.chmod(self.args['symbols'])\n                else:\n                    time.sleep(0.1)\n\n                    # read db properties\n                    db_data = os.stat(self.args['symbols'])\n                    db_size = db_data.st_size\n                    db_last_modified = db_data.st_mtime\n\n                    # empty file\n                    if db_size == 0:\n                        if prev_contracts:\n                            self.log_blotter.info('Cancel market data...')\n                            self.ibConn.cancelMarketData()\n                            time.sleep(0.1)\n                            prev_contracts = []\n                        continue\n\n                    # modified?\n                    if not first_run and db_last_modified == db_modified:\n                        continue\n\n                    # continue...\n                    db_modified = db_last_modified\n\n                    # read contructs db\n                    df = pd.read_csv(self.args['symbols'], header=0)\n                    if df.empty:\n                        continue\n\n                    # removed expired\n                    df = df[(\n                            (df['expiry'] < 1000000) & (\n                                df['expiry'] >= int(datetime.now().strftime('%Y%m')))) | (\n                            (df['expiry'] >= 1000000) & (\n                                df['expiry'] >= int(datetime.now().strftime('%Y%m%d')))) |\n                            np_isnan(df['expiry'])\n                            ]\n\n                    # fix expiry formatting (no floats)\n                    df['expiry'] = df['expiry'].fillna(\n                        0).astype(int).astype(str)\n                    df.loc[df['expiry'] == \"0\", 'expiry'] = \"\"\n                    df = df[df['sec_type'] != 'BAG']\n\n                    df.fillna(\"\", inplace=True)\n                    df.to_csv(self.args['symbols'], header=True, index=False)\n                    tools.chmod(self.args['symbols'])\n\n                    # ignore commentee\n                    df = df[~df['symbol'].str.contains(\"#\")]\n                    contracts = [tuple(x) for x in df.values]\n\n                    if first_run:\n                        first_run = False\n\n                    else:\n                        if contracts != prev_contracts:\n                            # cancel market data for removed contracts\n                            for contract in prev_contracts:\n                                if contract not in contracts:\n                                    self.ibConn.cancelMarketData(\n                                        self.ibConn.createContract(contract))\n                                    if self.args['orderbook']:\n                                        self.ibConn.cancelMarketDepth(\n                                            self.ibConn.createContract(contract))\n                                    time.sleep(0.1)\n                                    contract_string = self.ibConn.contractString(\n                                        contract).split('_')[0]\n                                    self.log_blotter.info(\n                                        'Contract Removed [%s]', contract_string)\n\n                    # request market data\n                    for contract in contracts:\n                        if contract not in prev_contracts:\n                            self.ibConn.requestMarketData(\n                                self.ibConn.createContract(contract))\n                            if self.args['orderbook']:\n                                self.ibConn.requestMarketDepth(\n                                    self.ibConn.createContract(contract))\n                            time.sleep(0.1)\n                            contract_string = self.ibConn.contractString(\n                                contract).split('_')[0]\n                            self.log_blotter.info(\n                                'Contract Added [%s]', contract_string)\n\n                    # update latest contracts\n                    prev_contracts = contracts\n\n                time.sleep(2)\n\n        except (KeyboardInterrupt, SystemExit):\n            self.quitting = True  # don't display connection errors on ctrl+c\n            print(\n                \"\\n\\n>>> Interrupted with Ctrl-c...\\n(waiting for running tasks to be completed)\\n\")\n            # asynctools.multitasking.killall() # stop now\n            asynctools.multitasking.wait_for_tasks()  # wait for threads to complete\n            sys.exit(1)", "response": "Starts the blotter thread"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _fix_history_sequence(self, df, table):\n\n        # remove \"Unnamed: x\" columns\n        cols = df.columns[df.columns.str.startswith('Unnamed:')].tolist()\n        df.drop(cols, axis=1, inplace=True)\n\n        # remove future dates\n        df['datetime'] = pd.to_datetime(df['datetime'], utc=True)\n        blacklist = df[df['datetime'] > pd.to_datetime('now', utc=True)]\n        df = df.loc[set(df.index) - set(blacklist)]  # .tail()\n\n        # loop through data, symbol by symbol\n        dfs = []\n        bad_ids = [blacklist['id'].values.tolist()]\n\n        for symbol_id in list(df['symbol_id'].unique()):\n\n            data = df[df['symbol_id'] == symbol_id].copy()\n\n            # sort by id\n            data.sort_values('id', axis=0, ascending=True, inplace=False)\n\n            # convert index to column\n            data.loc[:, \"ix\"] = data.index\n            data.reset_index(inplace=True)\n\n            # find out of sequence ticks/bars\n            malformed = data.shift(1)[(data['id'] > data['id'].shift(1)) & (\n                data['datetime'] < data['datetime'].shift(1))]\n\n            # cleanup rows\n            if malformed.empty:\n                # if all rows are in sequence, just remove last row\n                dfs.append(data)\n            else:\n                # remove out of sequence rows + last row from data\n                index = [\n                    x for x in data.index.values if x not in malformed['ix'].values]\n                dfs.append(data.loc[index])\n\n                # add to bad id list (to remove from db)\n                bad_ids.append(list(malformed['id'].values))\n\n        # combine all lists\n        data = pd.concat(dfs, sort=True)\n\n        # flatten bad ids\n        bad_ids = sum(bad_ids, [])\n\n        # remove bad ids from db\n        if bad_ids:\n            bad_ids = list(map(str, map(int, bad_ids)))\n            self.dbcurr.execute(\"DELETE FROM greeks WHERE %s IN (%s)\" % (\n                table.lower()[:-1] + \"_id\", \",\".join(bad_ids)))\n            self.dbcurr.execute(\"DELETE FROM \" + table.lower() +\n                                \" WHERE id IN (%s)\" % (\",\".join(bad_ids)))\n            try:\n                self.dbconn.commit()\n            except Exception as e:\n                self.dbconn.rollback()\n\n        # return\n        return data.drop(['id', 'ix', 'index'], axis=1)", "response": "fix out - of - sequence ticks and bars in history table"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef session(df, start='17:00', end='16:00'):\n    if df.empty:\n        return df\n\n    # get start/end/now as decimals\n    int_start = list(map(int, start.split(':')))\n    int_start = (int_start[0] + int_start[1] - 1 / 100) - 0.0001\n    int_end = list(map(int, end.split(':')))\n    int_end = int_end[0] + int_end[1] / 100\n    int_now = (df[-1:].index.hour[0] + (df[:1].index.minute[0]) / 100)\n\n    # same-dat session?\n    is_same_day = int_end > int_start\n\n    # set pointers\n    curr = prev = df[-1:].index[0].strftime('%Y-%m-%d')\n\n    # globex/forex session\n    if not is_same_day:\n        prev = (datetime.strptime(curr, '%Y-%m-%d') -\n                timedelta(1)).strftime('%Y-%m-%d')\n\n    # slice\n    if int_now >= int_start:\n        df = df[df.index >= curr + ' ' + start]\n    else:\n        df = df[df.index >= prev + ' ' + start]\n\n    return df.copy()", "response": "remove previous globex day from df"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ncalculates vwap of entire time series", "response": "def vwap(bars):\n    \"\"\"\n    calculate vwap of entire time series\n    (input can be pandas series or numpy array)\n    bars are usually mid [ (h+l)/2 ] or typical [ (h+l+c)/3 ]\n    \"\"\"\n    typical = ((bars['high'] + bars['low'] + bars['close']) / 3).values\n    volume = bars['volume'].values\n\n    return pd.Series(index=bars.index,\n                     data=np.cumsum(volume * typical) / np.cumsum(volume))"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncalculates vwap using moving window", "response": "def rolling_vwap(bars, window=200, min_periods=None):\n    \"\"\"\n    calculate vwap using moving window\n    (input can be pandas series or numpy array)\n    bars are usually mid [ (h+l)/2 ] or typical [ (h+l+c)/3 ]\n    \"\"\"\n    min_periods = window if min_periods is None else min_periods\n\n    typical = ((bars['high'] + bars['low'] + bars['close']) / 3)\n    volume = bars['volume']\n\n    left = (volume * typical).rolling(window=window,\n                                      min_periods=min_periods).sum()\n    right = volume.rolling(window=window, min_periods=min_periods).sum()\n\n    return pd.Series(index=bars.index, data=(left / right)).replace([np.inf, -np.inf], float('NaN')).ffill()"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef rsi(series, window=14):\n\n    # 100-(100/relative_strength)\n    deltas = np.diff(series)\n    seed = deltas[:window + 1]\n\n    # default values\n    ups = seed[seed > 0].sum() / window\n    downs = -seed[seed < 0].sum() / window\n    rsival = np.zeros_like(series)\n    rsival[:window] = 100. - 100. / (1. + ups / downs)\n\n    # period values\n    for i in range(window, len(series)):\n        delta = deltas[i - 1]\n        if delta > 0:\n            upval = delta\n            downval = 0\n        else:\n            upval = 0\n            downval = -delta\n\n        ups = (ups * (window - 1) + upval) / window\n        downs = (downs * (window - 1.) + downval) / window\n        rsival[i] = 100. - 100. / (1. + ups / downs)\n\n    # return rsival\n    return pd.Series(index=series.index, data=rsival)", "response": "compute the n period relative strength indicator\n   "}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef macd(series, fast=3, slow=10, smooth=16):\n    macd_line = rolling_weighted_mean(series, window=fast) - \\\n        rolling_weighted_mean(series, window=slow)\n    signal = rolling_weighted_mean(macd_line, window=smooth)\n    histogram = macd_line - signal\n    # return macd_line, signal, histogram\n    return pd.DataFrame(index=series.index, data={\n        'macd': macd_line.values,\n        'signal': signal.values,\n        'histogram': histogram.values\n    })", "response": "compute the MACD of a single object"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef roc(series, window=14):\n    res = (series - series.shift(window)) / series.shift(window)\n    return pd.Series(index=series.index, data=res)", "response": "compute rate of change"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef cci(series, window=14):\n    price = typical_price(series)\n    typical_mean = rolling_mean(price, window)\n    res = (price - typical_mean) / (.015 * np.std(typical_mean))\n    return pd.Series(index=series.index, data=res)", "response": "compute commodity channel index"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nget zscore of price", "response": "def zscore(bars, window=20, stds=1, col='close'):\n    \"\"\" get zscore of price \"\"\"\n    std = numpy_rolling_std(bars[col], window)\n    mean = numpy_rolling_mean(bars[col], window)\n    return (bars[col] - mean) / (std * stds)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef pvt(bars):\n    trend = ((bars['close'] - bars['close'].shift(1)) /\n             bars['close'].shift(1)) * bars['volume']\n    return trend.cumsum()", "response": "Calculates the Price Volume Trend"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef run(self):\n\n        history = pd.DataFrame()\n\n        # get history from csv dir\n        if self.backtest and self.backtest_csv:\n            kind = \"TICK\" if self.resolution[-1] in (\"S\", \"K\", \"V\") else \"BAR\"\n            dfs = []\n            for symbol in self.symbols:\n                file = \"%s/%s.%s.csv\" % (self.backtest_csv, symbol, kind)\n                if not os.path.exists(file):\n                    self.log_algo.error(\n                        \"Can't load data for %s (%s doesn't exist)\",\n                        symbol, file)\n                    sys.exit(0)\n                try:\n                    df = pd.read_csv(file)\n                    if \"expiry\" not in df.columns:\n                        df.loc[:, \"expiry\"] = nan\n\n                    if not validate_csv_columns(df, kind, raise_errors=False):\n                        self.log_algo.error(\n                            \"%s isn't a QTPyLib-compatible format\", file)\n                        sys.exit(0)\n\n                    if df['symbol'].values[-1] != symbol:\n                        self.log_algo.error(\n                            \"%s Doesn't content data for %s\", file, symbol)\n                        sys.exit(0)\n\n                    dfs.append(df)\n\n                except Exception as e:\n                    self.log_algo.error(\n                        \"Error reading data for %s (%s)\", symbol, file)\n                    sys.exit(0)\n\n            history = prepare_history(\n                data=pd.concat(dfs, sort=True),\n                resolution=self.resolution,\n                tz=self.timezone,\n                continuous=self.continuous\n            )\n            history = history[history.index >= self.backtest_start]\n\n        elif not self.blotter_args[\"dbskip\"] and (\n                self.backtest or self.preload):\n\n            start = self.backtest_start if self.backtest else tools.backdate(\n                self.preload)\n            end = self.backtest_end if self.backtest else None\n\n            history = self.blotter.history(\n                symbols=self.symbols,\n                start=start,\n                end=end,\n                resolution=self.resolution,\n                tz=self.timezone,\n                continuous=self.continuous\n            )\n\n            # history needs backfilling?\n            # self.blotter.backfilled = True\n            if not self.blotter.backfilled:\n                # \"loan\" Blotter our ibConn\n                self.blotter.ibConn = self.ibConn\n\n                # call the back fill\n                self.blotter.backfill(data=history,\n                                      resolution=self.resolution,\n                                      start=start, end=end)\n\n                # re-get history from db\n                history = self.blotter.history(\n                    symbols=self.symbols,\n                    start=start,\n                    end=end,\n                    resolution=self.resolution,\n                    tz=self.timezone,\n                    continuous=self.continuous\n                )\n\n                # take our ibConn back :)\n                self.blotter.ibConn = None\n\n        # optimize pandas\n        if not history.empty:\n            history['symbol'] = history['symbol'].astype('category')\n            history['symbol_group'] = history['symbol_group'].astype('category')\n            history['asset_class'] = history['asset_class'].astype('category')\n\n        if self.backtest:\n            # initiate strategy\n            self.on_start()\n\n            # drip history\n            drip_handler = self._tick_handler if self.resolution[-1] in (\n                \"S\", \"K\", \"V\") else self._bar_handler\n            self.blotter.drip(history, drip_handler)\n\n        else:\n            # place history self.bars\n            self.bars = history\n\n            # add instruments to blotter in case they do not exist\n            self.blotter.register(self.instruments)\n\n            # initiate strategy\n            self.on_start()\n\n            # listen for RT data\n            self.blotter.stream(\n                symbols=self.symbols,\n                tz=self.timezone,\n                quote_handler=self._quote_handler,\n                tick_handler=self._tick_handler,\n                bar_handler=self._bar_handler,\n                book_handler=self._book_handler\n            )", "response": "This function is called by the main function of the main function. It processes market data and processes bar data and returns the result of the main function."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_history(self, symbols, start, end=None, resolution=\"1T\", tz=\"UTC\"):\n        return self.blotter.history(symbols, start, end, resolution, tz)", "response": "Get historical market data for a list of symbols"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef order(self, signal, symbol, quantity=0, **kwargs):\n        self.log_algo.debug('ORDER: %s %4d %s %s', signal,\n                            quantity, symbol, kwargs)\n        if signal.upper() == \"EXIT\" or signal.upper() == \"FLATTEN\":\n            position = self.get_positions(symbol)\n            if position['position'] == 0:\n                return\n\n            kwargs['symbol'] = symbol\n            kwargs['quantity'] = abs(position['position'])\n            kwargs['direction'] = \"BUY\" if position['position'] < 0 else \"SELL\"\n\n            # print(\"EXIT\", kwargs)\n\n            try:\n                self.record({symbol+'_POSITION': 0})\n            except Exception as e:\n                pass\n\n            if not self.backtest:\n                self._create_order(**kwargs)\n\n        else:\n            if quantity == 0:\n                return\n\n            kwargs['symbol'] = symbol\n            kwargs['quantity'] = abs(quantity)\n            kwargs['direction'] = signal.upper()\n\n            # print(signal.upper(), kwargs)\n\n            # record\n            try:\n                quantity = abs(quantity)\n                if kwargs['direction'] != \"BUY\":\n                    quantity = -quantity\n                self.record({symbol+'_POSITION': quantity})\n            except Exception as e:\n                pass\n\n            if not self.backtest:\n                self._create_order(**kwargs)", "response": "Send an order for the selected instrument and symbol."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef record(self, *args, **kwargs):\n        if self.record_output:\n            try:\n                self.datastore.record(self.record_ts, *args, **kwargs)\n            except Exception as e:\n                pass", "response": "Records data for later analysis."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nsends an SMS message.", "response": "def sms(self, text):\n        \"\"\"Sends an SMS message.\n        Relies on properly setting up an SMS provider (refer to the\n        SMS section of the documentation for more information about this)\n\n        Call from within your strategy:\n        ``self.sms(\"message text\")``\n\n        :Parameters:\n            text : string\n                The body of the SMS message to send\n\n        \"\"\"\n        logging.info(\"SMS: %s\", str(text))\n        sms.send_text(self.name + ': ' + str(text), self.sms_numbers)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _base_bar_handler(self, bar):\n        # bar symbol\n        symbol = bar['symbol'].values\n        if len(symbol) == 0:\n            return\n        symbol = symbol[0]\n        self_bars = self.bars.copy()  # work on copy\n\n        is_tick_or_volume_bar = False\n        handle_bar = True\n\n        if self.resolution[-1] in (\"S\", \"K\", \"V\"):\n            is_tick_or_volume_bar = True\n            handle_bar = self._caller(\"_tick_handler\")\n\n        # drip is also ok\n        handle_bar = handle_bar or self._caller(\"drip\")\n\n        if is_tick_or_volume_bar:\n            # just add a bar (used by tick bar bandler)\n            if self.threads == 0:\n                self.bars = self._update_window(self.bars, bar,\n                                                window=self.bar_window)\n            else:\n                self_bars = self._update_window(self_bars, bar,\n                                                window=self.bar_window)\n        else:\n            # add the bar and resample to resolution\n            if self.threads == 0:\n                self.bars = self._update_window(self.bars, bar,\n                                                window=self.bar_window,\n                                                resolution=self.resolution)\n            else:\n                self_bars = self._update_window(self_bars, bar,\n                                                window=self.bar_window,\n                                                resolution=self.resolution)\n\n        # assign new data to self.bars if threaded\n        if self.threads > 0:\n            self.bars = self._thread_safe_merge(symbol, self.bars, self_bars)\n\n        # optimize pandas\n        if len(self.bars) == 1:\n            self.bars['symbol'] = self.bars['symbol'].astype('category')\n            self.bars['symbol_group'] = self.bars['symbol_group'].astype('category')\n            self.bars['asset_class'] = self.bars['asset_class'].astype('category')\n\n        # new bar?\n        hash_string = bar[:1]['symbol'].to_string().translate(\n            str.maketrans({key: None for key in \"\\n -:+\"}))\n        this_bar_hash = abs(hash(hash_string)) % (10 ** 8)\n\n        newbar = True\n        if symbol in self.bar_hashes.keys():\n            newbar = self.bar_hashes[symbol] != this_bar_hash\n        self.bar_hashes[symbol] = this_bar_hash\n\n        if newbar and handle_bar:\n            if self.bars[(self.bars['symbol'] == symbol) | (\n                    self.bars['symbol_group'] == symbol)].empty:\n                return\n            bar_instrument = self.get_instrument(symbol)\n            if bar_instrument:\n                self.record_ts = bar.index[0]\n                self.on_bar(bar_instrument)\n                # if self.resolution[-1] not in (\"S\", \"K\", \"V\"):\n                self.record(bar)", "response": "non threaded bar handler"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _add_signal_history(self, df, symbol):\n        if symbol not in self.signals.keys() or len(self.signals[symbol]) == 0:\n            self.signals[symbol] = [nan] * len(df.index)\n        else:\n            self.signals[symbol].append(nan)\n\n        self.signals[symbol] = self.signals[symbol][-len(df.index):]\n        signal_count = len(self.signals[symbol])\n        df.loc[-signal_count:, 'signal'] = self.signals[symbol][-signal_count:]\n\n        return df", "response": "Add signal history to the DataFrame"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nshift the dataframe to a given window", "response": "def multi_shift(df, window):\n    \"\"\"\n    get last N rows RELATIVE to another row in pandas\n    http://stackoverflow.com/questions/25724056/how-to-get-last-n-rows-relative-to-another-row-in-pandas-vector-solution\n    \"\"\"\n    if isinstance(df, pd.Series):\n        df = pd.DataFrame(df)\n\n    dfs = [df.shift(i) for i in np.arange(window)]\n    for ix, df_item in enumerate(dfs[1:]):\n        dfs[ix + 1].columns = [str(col) for col in df_item.columns + str(ix + 1)]\n    return pd.concat(dfs, 1, sort=True)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nchecking if a string is a number", "response": "def is_number(string):\n    \"\"\" checks if a string is a number (int/float) \"\"\"\n    string = str(string)\n    if string.isnumeric():\n        return True\n    try:\n        float(string)\n        return True\n    except ValueError:\n        return False"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef to_decimal(number, points=None):\n    if not is_number(number):\n        return number\n\n    number = float(decimal.Decimal(number * 1.))  # can't Decimal an int\n    if is_number(points):\n        return round(number, points)\n    return number", "response": "converts a number into a decimal. Decimal"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncreates ib contract tuple", "response": "def create_ib_tuple(instrument):\n    \"\"\" create ib contract tuple \"\"\"\n    from qtpylib import futures\n\n    if isinstance(instrument, str):\n        instrument = instrument.upper()\n\n        if \"FUT.\" not in instrument:\n            # symbol stock\n            instrument = (instrument, \"STK\", \"SMART\", \"USD\", \"\", 0.0, \"\")\n\n        else:\n            # future contract\n            try:\n                symdata = instrument.split(\".\")\n\n                # is this a CME future?\n                if symdata[1] not in futures.futures_contracts.keys():\n                    raise ValueError(\n                        \"Un-supported symbol. Please use full contract tuple.\")\n\n                # auto get contract details\n                spec = futures.get_ib_futures(symdata[1])\n                if not isinstance(spec, dict):\n                    raise ValueError(\"Un-parsable contract tuple\")\n\n                # expiry specified?\n                if len(symdata) == 3 and symdata[2] != '':\n                    expiry = symdata[2]\n                else:\n                    # default to most active\n                    expiry = futures.get_active_contract(symdata[1])\n\n                instrument = (spec['symbol'].upper(), \"FUT\",\n                              spec['exchange'].upper(), spec['currency'].upper(),\n                              int(expiry), 0.0, \"\")\n\n            except Exception as e:\n                raise ValueError(\"Un-parsable contract tuple\")\n\n    # tuples without strike/right\n    elif len(instrument) <= 7:\n        instrument_list = list(instrument)\n        if len(instrument_list) < 3:\n            instrument_list.append(\"SMART\")\n        if len(instrument_list) < 4:\n            instrument_list.append(\"USD\")\n        if len(instrument_list) < 5:\n            instrument_list.append(\"\")\n        if len(instrument_list) < 6:\n            instrument_list.append(0.0)\n        if len(instrument_list) < 7:\n            instrument_list.append(\"\")\n\n        try:\n            instrument_list[4] = int(instrument_list[4])\n        except Exception as e:\n            pass\n\n        instrument_list[5] = 0. if isinstance(instrument_list[5], str) \\\n            else float(instrument_list[5])\n\n        instrument = tuple(instrument_list)\n\n    return instrument"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nchanges mod to writeable", "response": "def chmod(f):\n    \"\"\" change mod to writeable \"\"\"\n    try:\n        os.chmod(f, S_IWRITE)  # windows (cover all)\n    except Exception as e:\n        pass\n    try:\n        os.chmod(f, 0o777)  # *nix\n    except Exception as e:\n        pass"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nconverts df to dict and adds a datetime field if df is datetime", "response": "def as_dict(df, ix=':'):\n    \"\"\" converts df to dict and adds a datetime field if df is datetime \"\"\"\n    if isinstance(df.index, pd.DatetimeIndex):\n        df['datetime'] = df.index\n    return df.to_dict(orient='records')[ix]"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nget an IB - compatible durationStr for reqHistoricalData", "response": "def ib_duration_str(start_date=None):\n    \"\"\"\n    Get a datetime object or a epoch timestamp and return\n    an IB-compatible durationStr for reqHistoricalData()\n    \"\"\"\n    now = datetime.datetime.utcnow()\n\n    if is_number(start_date):\n        diff = now - datetime.datetime.fromtimestamp(float(start_date))\n    elif isinstance(start_date, str):\n        diff = now - parse_date(start_date)\n    elif isinstance(start_date, datetime.datetime):\n        diff = now - start_date\n    else:\n        return None\n\n    # get diff\n    second_diff = diff.seconds\n    day_diff = diff.days\n\n    # small diff?\n    if day_diff < 0 or second_diff < 60:\n        return None\n\n    # return str(second_diff)+ \" S\"\n    if day_diff == 0 and second_diff > 1:\n        return str(second_diff) + \" S\"\n    if 31 > day_diff > 0:\n        return str(day_diff) + \" D\"\n    if 365 > day_diff > 31:\n        return str(ceil(day_diff / 30)) + \" M\"\n\n    return str(ceil(day_diff / 365)) + \" Y\""}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef datetime64_to_datetime(dt):\n    dt64 = np.datetime64(dt)\n    ts = (dt64 - np.datetime64('1970-01-01T00:00:00')) / np.timedelta64(1, 's')\n    return datetime.datetime.utcfromtimestamp(ts)", "response": "convert numpy s datetime64 to datetime"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nround to closest resolution", "response": "def round_to_fraction(val, res, decimals=None):\n    \"\"\" round to closest resolution \"\"\"\n    if val is None:\n        return 0.0\n    if decimals is None and \".\" in str(res):\n        decimals = len(str(res).split('.')[1])\n\n    return round(round(val / res) * res, decimals)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef backdate(res, date=None, as_datetime=False, fmt='%Y-%m-%d'):\n    if res is None:\n        return None\n\n    if date is None:\n        date = datetime.datetime.now()\n    else:\n        try:\n            date = parse_date(date)\n        except Exception as e:\n            pass\n\n    new_date = date\n\n    periods = int(\"\".join([s for s in res if s.isdigit()]))\n\n    if periods > 0:\n\n        if \"K\" in res:\n            new_date = date - datetime.timedelta(microseconds=periods)\n        elif \"S\" in res:\n            new_date = date - datetime.timedelta(seconds=periods)\n        elif \"T\" in res:\n            new_date = date - datetime.timedelta(minutes=periods)\n        elif \"H\" in res or \"V\" in res:\n            new_date = date - datetime.timedelta(hours=periods)\n        elif \"W\" in res:\n            new_date = date - datetime.timedelta(weeks=periods)\n        else:  # days\n            new_date = date - datetime.timedelta(days=periods)\n\n        # not a week day:\n        while new_date.weekday() > 4:  # Mon-Fri are 0-4\n            new_date = backdate(res=\"1D\", date=new_date, as_datetime=True)\n\n    if as_datetime:\n        return new_date\n\n    return new_date.strftime(fmt)", "response": "get past date based on currect date"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef previous_weekday(day=None, as_datetime=False):\n    if day is None:\n        day = datetime.datetime.now()\n    else:\n        day = datetime.datetime.strptime(day, '%Y-%m-%d')\n\n    day -= datetime.timedelta(days=1)\n    while day.weekday() > 4:  # Mon-Fri are 0-4\n        day -= datetime.timedelta(days=1)\n\n    if as_datetime:\n        return day\n    return day.strftime(\"%Y-%m-%d\")", "response": "get the most recent business day"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nchecking if day is month s 3rd friday", "response": "def is_third_friday(day=None):\n    \"\"\" check if day is month's 3rd friday \"\"\"\n    day = day if day is not None else datetime.datetime.now()\n    defacto_friday = (day.weekday() == 4) or (\n        day.weekday() == 3 and day.hour() >= 17)\n    return defacto_friday and 14 < day.day < 22"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef after_third_friday(day=None):\n    day = day if day is not None else datetime.datetime.now()\n    now = day.replace(day=1, hour=16, minute=0, second=0, microsecond=0)\n    now += relativedelta.relativedelta(weeks=2, weekday=relativedelta.FR)\n    return day > now", "response": "check if day is after month s 3rd friday"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_timezone(as_timedelta=False):\n    try:\n        offset_hour = -(time.altzone if time.daylight else time.timezone)\n    except Exception as e:\n        offset_hour = -(datetime.datetime.now() -\n                        datetime.datetime.utcnow()).seconds\n\n    offset_hour = offset_hour // 3600\n    offset_hour = offset_hour if offset_hour < 10 else offset_hour // 10\n\n    if as_timedelta:\n        return datetime.timedelta(hours=offset_hour)\n\n    return 'Etc/GMT%+d' % offset_hour", "response": "utility to get the machine s timezone"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef datetime_to_timezone(date, tz=\"UTC\"):\n    if not date.tzinfo:\n        date = date.replace(tzinfo=timezone(get_timezone()))\n    return date.astimezone(timezone(tz))", "response": "convert naive datetime to timezone - aware datetime"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef convert_timezone(date_str, tz_from, tz_to=\"UTC\", fmt=None):\n    tz_offset = datetime_to_timezone(\n        datetime.datetime.now(), tz=tz_from).strftime('%z')\n    tz_offset = tz_offset[:3] + ':' + tz_offset[3:]\n\n    date = parse_date(str(date_str) + tz_offset)\n    if tz_from != tz_to:\n        date = datetime_to_timezone(date, tz_to)\n\n    if isinstance(fmt, str):\n        return date.strftime(fmt)\n    return date", "response": "converts a date string to a timezone offset"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef set_timezone(data, tz=None, from_local=False):\n    # pandas object?\n    if isinstance(data, pd.DataFrame) | isinstance(data, pd.Series):\n        try:\n            try:\n                data.index = data.index.tz_convert(tz)\n            except Exception as e:\n                if from_local:\n                    data.index = data.index.tz_localize(\n                        get_timezone()).tz_convert(tz)\n                else:\n                    data.index = data.index.tz_localize('UTC').tz_convert(tz)\n        except Exception as e:\n            pass\n\n    # not pandas...\n    else:\n        if isinstance(data, str):\n            data = parse_date(data)\n        try:\n            try:\n                data = data.astimezone(tz)\n            except Exception as e:\n                data = timezone('UTC').localize(data).astimezone(timezone(tz))\n        except Exception as e:\n            pass\n\n    return data", "response": "change the timeozone of the object to specified one without converting"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nfixes timezone for pandas", "response": "def fix_timezone(df, freq, tz=None):\n    \"\"\" set timezone for pandas \"\"\"\n    index_name = df.index.name\n\n    # fix timezone\n    if isinstance(df.index[0], str):\n        # timezone df exists\n        if (\"-\" in df.index[0][-6:]) | (\"+\" in df.index[0][-6:]):\n            df.index = pd.to_datetime(df.index, utc=False)\n            df.index = df.index.tz_localize('UTC').tz_convert(tz)\n\n        # no timezone df - do some resampling\n        else:\n            # original range\n            start_range = df.index[0]\n            end_range = df.index[-1]\n\n            # resample df\n            df.index = pd.to_datetime(df.index, utc=True)\n            df = resample(df, resolution=freq, ffill=False, dropna=False)\n\n            # create date range\n            new_freq = ''.join(i for i in freq if not i.isdigit())\n            rng = pd.date_range(start=start_range,\n                                end=end_range, tz=tz, freq=new_freq)\n\n            # assign date range to df and drop empty rows\n            df.index = rng\n            df.dropna(inplace=True)\n\n    # finalize timezone (also for timezone-aware df)\n    df = set_timezone(df, tz=tz)\n\n    df.index.name = index_name\n    return df"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nrecording the data for the current record", "response": "def record(self, timestamp, *args, **kwargs):\n        \"\"\" add custom data to data store \"\"\"\n        if self.output_file is None:\n            return\n\n        data = {'datetime': timestamp}\n\n        # append all data\n        if len(args) == 1:\n            if isinstance(args[0], dict):\n                data.update(dict(args[0]))\n            elif isinstance(args[0], pd.DataFrame):\n                data.update(args[0][-1:].to_dict(orient='records')[0])\n\n        # add kwargs\n        if kwargs:\n            data.update(dict(kwargs))\n\n        data['datetime'] = timestamp\n        # self.rows.append(pd.DataFrame(data=data, index=[timestamp]))\n\n        new_data = {}\n        if \"symbol\" not in data.keys():\n            new_data = dict(data)\n        else:\n            sym = data[\"symbol\"]\n            new_data[\"symbol\"] = data[\"symbol\"]\n            for key in data.keys():\n                if key not in ['datetime', 'symbol_group', 'asset_class']:\n                    new_data[sym + '_' + str(key).upper()] = data[key]\n\n        new_data['datetime'] = timestamp\n\n        # append to rows\n        self.rows.append(pd.DataFrame(data=new_data, index=[timestamp]))\n\n        # create dataframe\n        recorded = pd.concat(self.rows, sort=True)\n\n        if \"symbol\" not in recorded.columns:\n            return\n\n\n        # group by symbol\n        recorded['datetime'] = recorded.index\n        data = recorded.groupby(['symbol', 'datetime'], as_index=False).sum()\n        data.set_index('datetime', inplace=True)\n\n        symbols = data['symbol'].unique().tolist()\n        data.drop(columns=['symbol'], inplace=True)\n\n\n        # cleanup:\n\n        # remove symbols\n        recorded.drop(['symbol'] + [sym + '_SYMBOL' for sym in symbols],\n                      axis=1, inplace=True)\n\n        # remove non-option data if not working with options\n        for sym in symbols:\n            try:\n                opt_cols = recorded.columns[\n                    recorded.columns.str.startswith(sym + '_OPT_')].tolist()\n                if len(opt_cols) == len(recorded[opt_cols].isnull().all()):\n                    recorded.drop(opt_cols, axis=1, inplace=True)\n            except Exception as e:\n                pass\n\n        # group df\n        recorded = recorded.groupby(recorded['datetime']).first()\n\n        # shift position\n        for sym in symbols:\n            recorded[sym + '_POSITION'] = recorded[sym + '_POSITION'\n                                                   ].shift(1).fillna(0)\n\n        # make this public\n        self.recorded = recorded.copy()\n\n        # cleanup columns names before saving...\n        recorded.columns = [col.replace('_FUT_', '_').replace(\n                            '_OPT_OPT_', '_OPT_') for col in recorded.columns]\n\n        # save\n        if \".csv\" in self.output_file:\n            recorded.to_csv(self.output_file)\n        elif \".h5\" in self.output_file:\n            recorded.to_hdf(self.output_file, 0)\n        elif (\".pickle\" in self.output_file) | (\".pkl\" in self.output_file):\n            recorded.to_pickle(self.output_file)\n\n        chmod(self.output_file)"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ndownloads historical data from Interactive Brokers and saves it in a QTPyLib compatible format.", "response": "def get_data_ib(instrument, start, resolution=\"1 min\",\n                blotter=None, output_path=None):\n    \"\"\"\n    Downloads historical data from Interactive Brokers\n\n    :Parameters:\n        instrument : mixed\n            IB contract tuple / string (same as that given to strategy)\n        start : str\n            Backtest start date (YYYY-MM-DD [HH:MM:SS[.MS])\n\n    :Optional:\n        resolution : str\n            1/5/15/30 secs, 1/2/3/5/15/30 min (default 1min), 1 hour, 1 day\n        blotter : str\n            Store MySQL server used by this Blotter (default is \"auto detect\")\n        output_path : str\n            Path to where the resulting CSV should be saved (optional)\n\n    :Returns:\n        data : pd.DataFrame\n            Pandas DataFrame in a QTPyLib-compatible format and timezone\n    \"\"\"\n    global _IB_HISTORY_DOWNLOADED\n    _IB_HISTORY_DOWNLOADED = False\n\n    # load blotter settings\n    blotter_args = load_blotter_args(\n        blotter, logger=logging.getLogger(__name__))\n\n    # create contract string (no need for connection)\n    ibConn = ezIBpy()\n    ibConn.ibCallback = ibCallback\n\n    if not ibConn.connected:\n        ibConn.connect(clientId=997,\n                       port=int(blotter_args['ibport']),\n                       host=str(blotter_args['ibserver']))\n\n    # generate a valid ib tuple\n    instrument = tools.create_ib_tuple(instrument)\n    contract_string = ibConn.contractString(instrument)\n    contract = ibConn.createContract(instrument)\n\n    ibConn.requestHistoricalData(contracts=[contract],\n                                 data=\"TRADES\", resolution=resolution,\n                                 lookback=tools.ib_duration_str(start),\n                                 rth=False)\n\n    while not _IB_HISTORY_DOWNLOADED:\n        time.sleep(1)\n\n    ibConn.disconnect()\n\n    data = ibConn.historicalData[contract_string]\n    data['datetime'] = data.index\n    return prepare_data(instrument, data, output_path=output_path)"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nconvert given DataFrame to a QTPyLib - compatible format and timezone for a single node.", "response": "def prepare_data(instrument, data, output_path=None,\n                 index=None, colsmap=None, kind=\"BAR\", resample=\"1T\"):\n    \"\"\"\n    Converts given DataFrame to a QTPyLib-compatible format and timezone\n\n    :Parameters:\n        instrument : mixed\n            IB contract tuple / string (same as that given to strategy)\n        data : pd.DataFrame\n            Pandas DataDrame with that instrument's market data\n        output_path : str\n            Path to where the resulting CSV should be saved (optional)\n        index : pd.Series\n            Pandas Series that will be used for df's index (optioanl)\n        colsmap : dict\n            Dict for mapping df's columns to those used by QTPyLib\n            (default assumes same naming convention as QTPyLib's)\n        kind : str\n            Is this ``BAR`` or ``TICK`` data\n        resample : str\n            Pandas resolution (defaults to 1min/1T)\n\n    :Returns:\n        data : pd.DataFrame\n            Pandas DataFrame in a QTPyLib-compatible format and timezone\n    \"\"\"\n\n    global _TICKS_COLSMAP, _BARS_COLSMAP\n\n    # work on copy\n    df = data.copy()\n\n    # ezibpy's csv?\n    if set(df.columns) == set([\n            'datetime', 'C', 'H', 'L', 'O', 'OI', 'V', 'WAP']):\n        df.rename(columns={\n            'datetime': 'datetime',\n            'O': 'open',\n            'H': 'high',\n            'L': 'low',\n            'C': 'close',\n            'OI': 'volume',\n        }, inplace=True)\n        df.index = pd.to_datetime(df['datetime'])\n        df.index = df.index.tz_localize(tools.get_timezone()).tz_convert(\"UTC\")\n        index = None\n\n    # lower case columns\n    df.columns = map(str.lower, df.columns)\n\n    # set index\n    if index is None:\n        index = df.index\n\n    # set defaults columns\n    if not isinstance(colsmap, dict):\n        colsmap = {}\n\n    _colsmap = _TICKS_COLSMAP if kind == \"TICK\" else _BARS_COLSMAP\n    for el in _colsmap:\n        if el not in colsmap:\n            colsmap[el] = _colsmap[el]\n\n    # generate a valid ib tuple\n    instrument = tools.create_ib_tuple(instrument)\n\n    # create contract string (no need for connection)\n    ibConn = ezIBpy()\n    contract_string = ibConn.contractString(instrument)\n    asset_class = tools.gen_asset_class(contract_string)\n    symbol_group = tools.gen_symbol_group(contract_string)\n\n    # add symbol data\n    df.loc[:, 'symbol'] = contract_string\n    df.loc[:, 'symbol_group'] = symbol_group\n    df.loc[:, 'asset_class'] = asset_class\n\n    # validate columns\n    valid_cols = validate_columns(df, kind)\n    if not valid_cols:\n        raise ValueError('Invalid Column list')\n\n    # rename columns to map\n    df.rename(columns=colsmap, inplace=True)\n\n    # force option columns on options\n    if asset_class == \"OPT\":\n        df = tools.force_options_columns(df)\n\n    # remove all other columns\n    known_cols = list(colsmap.values()) + \\\n        ['symbol', 'symbol_group', 'asset_class', 'expiry']\n    for col in df.columns:\n        if col not in known_cols:\n            df.drop(col, axis=1, inplace=True)\n\n    # set UTC index\n    df.index = pd.to_datetime(index)\n    df = tools.set_timezone(df, \"UTC\")\n    df.index.rename(\"datetime\", inplace=True)\n\n    # resample\n    if resample and kind == \"BAR\":\n        df = tools.resample(df, resolution=resample, tz=\"UTC\")\n\n    # add expiry\n    df.loc[:, 'expiry'] = np.nan\n    if asset_class in (\"FUT\", \"OPT\", \"FOP\"):\n        df.loc[:, 'expiry'] = contract_expiry_from_symbol(contract_string)\n\n    # save csv\n    if output_path is not None:\n        output_path = output_path[\n            :-1] if output_path.endswith('/') else output_path\n        df.to_csv(\"%s/%s.%s.csv\" % (output_path, contract_string, kind))\n\n    # return df\n    return df"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nstoring data in MySQL using the provided blotter.", "response": "def store_data(df, blotter=None, kind=\"BAR\"):\n    \"\"\"\n    Store QTPyLib-compatible csv files in Blotter's MySQL.\n    TWS/GW data are required for determining futures/options expiration\n\n    :Parameters:\n        df : dict\n            Tick/Bar data\n\n    :Optional:\n        blotter : str\n            Store MySQL server used by this Blotter (default is \"auto detect\")\n        kind : str\n            Is this ``BAR`` or ``TICK`` data\n    \"\"\"\n\n    # validate columns\n    valid_cols = validate_columns(df, kind)\n    if not valid_cols:\n        raise ValueError('Invalid Column list')\n\n    # load blotter settings\n    blotter_args = load_blotter_args(\n        blotter, logger=logging.getLogger(__name__))\n\n    # blotter not running\n    if blotter_args is None:\n        raise Exception(\"Cannot connect to running Blotter.\")\n\n    # cannot continue\n    if blotter_args['dbskip']:\n        raise Exception(\"Cannot continue. Blotter running with --dbskip\")\n\n    # connect to mysql using blotter's settings\n    dbconn = pymysql.connect(\n        client_flag=MULTI_STATEMENTS,\n        host=str(blotter_args['dbhost']),\n        port=int(blotter_args['dbport']),\n        user=str(blotter_args['dbuser']),\n        passwd=str(blotter_args['dbpass']),\n        db=str(blotter_args['dbname']),\n        autocommit=True\n    )\n    dbcurr = dbconn.cursor()\n\n    # loop through symbols and save in db\n    for symbol in list(df['symbol'].unique()):\n        data = df[df['symbol'] == symbol]\n        symbol_id = get_symbol_id(symbol, dbconn, dbcurr)\n\n        # prepare columns for insert\n        data.loc[:, 'timestamp'] = data.index.strftime('%Y-%m-%d %H:%M:%S')\n        data.loc[:, 'symbol_id'] = symbol_id\n\n        # insert row by row to handle greeks\n        data = data.to_dict(orient=\"records\")\n\n        if kind == \"BAR\":\n            for _, row in enumerate(data):\n                mysql_insert_bar(row, symbol_id, dbcurr)\n        else:\n            for _, row in enumerate(data):\n                mysql_insert_tick(row, symbol_id, dbcurr)\n\n        try:\n            dbconn.commit()\n        except Exception as e:\n            return False\n\n    return True"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _set_windows(self, ticks, bars):\n        self.tick_window = ticks\n        self.bar_window = bars", "response": "set ticks and bars for the current time window"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_bars(self, lookback=None, as_dict=False):\n        bars = self._get_symbol_dataframe(self.parent.bars, self)\n\n        # add signal history to bars\n        bars = self.parent._add_signal_history(df=bars, symbol=self)\n\n        lookback = self.bar_window if lookback is None else lookback\n        bars = bars[-lookback:]\n        # if lookback is not None:\n        #     bars = bars[-lookback:]\n\n        if not bars.empty > 0 and bars['asset_class'].values[-1] not in (\"OPT\", \"FOP\"):\n            bars.drop(bars.columns[\n                bars.columns.str.startswith('opt_')].tolist(),\n                inplace=True, axis=1)\n\n        if as_dict:\n            bars.loc[:, 'datetime'] = bars.index\n            bars = bars.to_dict(orient='records')\n            if lookback == 1:\n                bars = None if not bars else bars[0]\n\n        return bars", "response": "Get the available bars for this instrument."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_ticks(self, lookback=None, as_dict=False):\n        ticks = self._get_symbol_dataframe(self.parent.ticks, self)\n\n        lookback = self.tick_window if lookback is None else lookback\n        ticks = ticks[-lookback:]\n        # if lookback is not None:\n        #     ticks = ticks[-lookback:]\n\n        if not ticks.empty and ticks['asset_class'].values[-1] not in (\"OPT\", \"FOP\"):\n            ticks.drop(ticks.columns[\n                ticks.columns.str.startswith('opt_')].tolist(),\n                inplace=True, axis=1)\n\n        if as_dict:\n            ticks.loc[:, 'datetime'] = ticks.index\n            ticks = ticks.to_dict(orient='records')\n            if lookback == 1:\n                ticks = None if not ticks else ticks[0]\n\n        return ticks", "response": "Get the ticks for this instrument."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nget the last price of the current time series", "response": "def get_price(self):\n        \"\"\" Shortcut to self.get_ticks(lookback=1, as_dict=True)['last'] \"\"\"\n        tick = self.get_ticks(lookback=1, as_dict=True)\n        return None if tick is None else tick['last']"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_quote(self):\n        if self in self.parent.quotes.keys():\n            return self.parent.quotes[self]\n        return None", "response": "Get the last quote for this instrument."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_orderbook(self):\n        if self in self.parent.books.keys():\n            return self.parent.books[self]\n\n        return {\n            \"bid\": [0], \"bidsize\": [0],\n            \"ask\": [0], \"asksize\": [0]\n        }", "response": "Get orderbook for the instrument\n       "}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef limit_order(self, direction, quantity, price, **kwargs):\n        kwargs['limit_price'] = price\n        kwargs['order_type'] = \"LIMIT\"\n        self.parent.order(direction.upper(), self, quantity=quantity, **kwargs)", "response": "Shortcut for instrument. order(... ) and accepts all of its\n        optional parameters"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef buy(self, quantity, **kwargs):\n        self.parent.order(\"BUY\", self, quantity=quantity, **kwargs)", "response": "Shortcut for instrument. order ( BUY"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef sell(self, quantity, **kwargs):\n        self.parent.order(\"SELL\", self, quantity=quantity, **kwargs)", "response": "Shortcut for instrument. order ( SELLL"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef sell_limit(self, quantity, price, **kwargs):\n        kwargs['limit_price'] = price\n        kwargs['order_type'] = \"LIMIT\"\n        self.parent.order(\"SELL\", self, quantity=quantity, **kwargs)", "response": "Shortcut for instrument. order. SELLL"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_positions(self, attr=None):\n        pos = self.parent.get_positions(self)\n        try:\n            if attr is not None:\n                attr = attr.replace(\"quantity\", \"position\")\n            return pos[attr]\n        except Exception as e:\n            return pos", "response": "Get the positions data for the instrument\n           "}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nmodify quantity and limit price of an active order for the instrument", "response": "def modify_order(self, orderId, quantity=None, limit_price=None):\n        \"\"\"Modify quantity and/or limit price of an active order for the instrument\n\n        :Parameters:\n            orderId : int\n                the order id to modify\n\n        :Optional:\n            quantity : int\n                the required quantity of the modified order\n            limit_price : int\n                the new limit price of the modified order\n        \"\"\"\n        return self.parent.modify_order(self, orderId, quantity, limit_price)"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nmodifies bracket order :Parameters: orderId : int the order id to modify :Optional: entry: float new entry limit price (for unfilled limit orders only) target: float new target limit price (for unfilled limit orders only) stop: float new stop limit price (for unfilled limit orders only) quantity : int the required quantity of the modified order", "response": "def modify_order_group(self, orderId, entry=None, target=None, stop=None, quantity=None):\n        \"\"\"Modify bracket order\n\n        :Parameters:\n            orderId : int\n                the order id to modify\n\n        :Optional:\n            entry: float\n                new entry limit price (for unfilled limit orders only)\n            target: float\n                new target limit price (for unfilled limit orders only)\n            stop: float\n                new stop limit price (for unfilled limit orders only)\n            quantity : int\n                the required quantity of the modified order\n        \"\"\"\n        return self.parent.modify_order_group(self, orderId=orderId,\n                                              entry=entry, target=target,\n                                              stop=stop, quantity=quantity)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nmodify stop order. Auto-discover **orderId** and **quantity** and invokes ``self.modify_order(...)``. :Parameters: stoploss : float the new stoploss limit price", "response": "def move_stoploss(self, stoploss):\n        \"\"\"Modify stop order.\n        Auto-discover **orderId** and **quantity** and invokes ``self.modify_order(...)``.\n\n        :Parameters:\n            stoploss : float\n                the new stoploss limit price\n\n        \"\"\"\n        stopOrder = self.get_active_order(order_type=\"STOP\")\n\n        if stopOrder is not None and \"orderId\" in stopOrder.keys():\n            self.modify_order(orderId=stopOrder['orderId'],\n                              quantity=stopOrder['quantity'], limit_price=stoploss)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ngetting the margin requirements for intrument", "response": "def get_margin_requirement(self):\n        \"\"\" Get margin requirements for intrument (futures only)\n\n        :Retruns:\n            margin : dict\n                margin requirements for instrument\n                (all values are ``None`` for non-futures instruments)\n        \"\"\"\n        contract = self.get_contract()\n\n        if contract.m_secType == \"FUT\":\n            return futures.get_ib_futures(contract.m_symbol, contract.m_exchange)\n\n        # else...\n        return {\n            \"exchange\": None,\n            \"symbol\": None,\n            \"description\": None,\n            \"class\": None,\n            \"intraday_initial\": None,\n            \"intraday_maintenance\": None,\n            \"overnight_initial\": None,\n            \"overnight_maintenance\": None,\n            \"currency\": None,\n        }"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_max_contracts_allowed(self, overnight=True):\n        timeframe = 'overnight_initial' if overnight else 'intraday_initial'\n        req_margin = self.get_margin_requirement()\n        if req_margin[timeframe] is not None:\n            if 'AvailableFunds' in self.parent.account:\n                return int(math.floor(self.parent.account['AvailableFunds'\n                                                          ] / req_margin[timeframe]))\n\n        return None", "response": "Returns the maximum number of contracts allowed to trade the current account."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nchecking if instrument pnl is within given range", "response": "def pnl_in_range(self, min_pnl, max_pnl):\n        \"\"\" Check if instrument pnl is within given range\n\n        :Parameters:\n            min_pnl : flaot\n                minimum session pnl (in USD / IB currency)\n            max_pnl : flaot\n                maximum session pnl (in USD / IB currency)\n\n        :Retruns:\n            status : bool\n                if pnl is within range\n        \"\"\"\n        portfolio = self.get_portfolio()\n        return -abs(min_pnl) < portfolio['totalPNL'] < abs(max_pnl)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef parse_intf_section(interface):\n    interface = interface.strip()\n    re_protocol = (\n        r\"^(?P<intf_name>\\S+?)\\s+is\\s+(?P<status>.+?)\"\n        r\",\\s+line\\s+protocol\\s+is\\s+(?P<protocol>\\S+).*$\"\n    )\n    re_intf_name_state = r\"^(?P<intf_name>\\S+) is (?P<intf_state>\\S+).*\"\n    re_is_enabled_1 = r\"^admin state is (?P<is_enabled>\\S+)$\"\n    re_is_enabled_2 = r\"^admin state is (?P<is_enabled>\\S+), \"\n    re_is_enabled_3 = r\"^.* is down.*Administratively down.*$\"\n    re_mac = r\"^\\s+Hardware:\\s+(?P<hardware>.*),\\s+address:\\s+(?P<mac_address>\\S+) \"\n    re_speed = (\n        r\"\\s+MTU (?P<mtu>\\S+)\\s+bytes,\\s+BW\\s+(?P<speed>\\S+)\\s+(?P<speed_unit>\\S+).*$\"\n    )\n    re_mtu_nve = r\"\\s+MTU (?P<mtu_nve>\\S+)\\s+bytes.*$\"\n    re_description_1 = r\"^\\s+Description:\\s+(?P<description>.*)  (?:MTU|Internet)\"\n    re_description_2 = r\"^\\s+Description:\\s+(?P<description>.*)$\"\n    re_hardware = r\"^.* Hardware: (?P<hardware>\\S+)$\"\n\n    # Check for 'protocol is ' lines\n    match = re.search(re_protocol, interface, flags=re.M)\n    if match:\n        intf_name = match.group(\"intf_name\")\n        status = match.group(\"status\")\n        protocol = match.group(\"protocol\")\n\n        if \"admin\" in status.lower():\n            is_enabled = False\n        else:\n            is_enabled = True\n        is_up = bool(\"up\" in protocol)\n\n    else:\n        # More standard is up, next line admin state is lines\n        match = re.search(re_intf_name_state, interface)\n        intf_name = match.group(\"intf_name\")\n        intf_state = match.group(\"intf_state\").strip()\n        is_up = True if intf_state == \"up\" else False\n\n        admin_state_present = re.search(\"admin state is\", interface)\n        if admin_state_present:\n            # Parse cases where 'admin state' string exists\n            for x_pattern in [re_is_enabled_1, re_is_enabled_2]:\n                match = re.search(x_pattern, interface, flags=re.M)\n                if match:\n                    is_enabled = match.group(\"is_enabled\").strip()\n                    is_enabled = True if re.search(\"up\", is_enabled) else False\n                    break\n            else:\n                msg = \"Error parsing intf, 'admin state' never detected:\\n\\n{}\".format(\n                    interface\n                )\n                raise ValueError(msg)\n        else:\n            # No 'admin state' should be 'is up' or 'is down' strings\n            # If interface is up; it is enabled\n            is_enabled = True\n            if not is_up:\n                match = re.search(re_is_enabled_3, interface, flags=re.M)\n                if match:\n                    is_enabled = False\n\n    match = re.search(re_mac, interface, flags=re.M)\n    if match:\n        mac_address = match.group(\"mac_address\")\n        mac_address = helpers.mac(mac_address)\n    else:\n        mac_address = \"\"\n\n    match = re.search(re_hardware, interface, flags=re.M)\n    speed_exist = True\n    if match:\n        if match.group(\"hardware\") == \"NVE\":\n            match = re.search(re_mtu_nve, interface, flags=re.M)\n            mtu = int(match.group(\"mtu_nve\"))\n            speed_exist = False\n\n    if speed_exist:\n        match = re.search(re_speed, interface, flags=re.M)\n        speed = int(match.group(\"speed\"))\n        mtu = int(match.group(\"mtu\"))\n        speed_unit = match.group(\"speed_unit\")\n        speed_unit = speed_unit.rstrip(\",\")\n        # This was alway in Kbit (in the data I saw)\n        if speed_unit != \"Kbit\":\n            msg = \"Unexpected speed unit in show interfaces parsing:\\n\\n{}\".format(\n                interface\n            )\n            raise ValueError(msg)\n        speed = int(round(speed / 1000.0))\n    else:\n        speed = -1\n\n    description = \"\"\n    for x_pattern in [re_description_1, re_description_2]:\n        match = re.search(x_pattern, interface, flags=re.M)\n        if match:\n            description = match.group(\"description\")\n            break\n\n    return {\n        intf_name: {\n            \"description\": description,\n            \"is_enabled\": is_enabled,\n            \"is_up\": is_up,\n            \"last_flapped\": -1.0,\n            \"mac_address\": mac_address,\n            \"mtu\": mtu,\n            \"speed\": speed,\n        }\n    }", "response": "Parse a single entry from show interfaces output."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef convert_hhmmss(hhmmss):\n    fields = hhmmss.split(\":\")\n    if len(fields) != 3:\n        raise ValueError(\"Received invalid HH:MM:SS data: {}\".format(hhmmss))\n    fields = [int(x) for x in fields]\n    hours, minutes, seconds = fields\n    return (hours * 3600) + (minutes * 60) + seconds", "response": "Convert hh : mm : ss to seconds."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nnormalizes bgp table data so that the line wrap doesn t exit.", "response": "def bgp_normalize_table_data(bgp_table):\n    \"\"\"The 'show bgp all summary vrf all' table can have entries that wrap multiple lines.\n\n    2001:db8:4:701::2\n                4 65535  163664  163693      145    0    0     3w2d 3\n    2001:db8:e0:dd::1\n                4    10  327491  327278      145    0    0     3w1d 4\n\n    Normalize this so the line wrap doesn't exit.\n    \"\"\"\n    bgp_table = bgp_table.strip()\n    bgp_multiline_pattern = r\"({})\\s*\\n\".format(IPV4_OR_IPV6_REGEX)\n    # Strip out the newline\n    return re.sub(bgp_multiline_pattern, r\"\\1\", bgp_table)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef bgp_summary_parser(bgp_summary):\n\n    bgp_summary_dict = {}\n    # Check for BGP summary information lines that have no data\n    if len(bgp_summary.strip().splitlines()) <= 1:\n        return {}\n\n    allowed_afi = [\"ipv4\", \"ipv6\", \"l2vpn\"]\n    vrf_regex = r\"^BGP summary information for VRF\\s+(?P<vrf>\\S+),\"\n    afi_regex = (\n        r\"^BGP summary information.*address family (?P<afi>\\S+ (?:Unicast|EVPN))\"\n    )\n    local_router_regex = (\n        r\"^BGP router identifier\\s+(?P<router_id>\\S+)\"\n        r\",\\s+local AS number\\s+(?P<local_as>\\S+)\"\n    )\n\n    for pattern in [vrf_regex, afi_regex, local_router_regex]:\n        match = re.search(pattern, bgp_summary, flags=re.M)\n        if match:\n            bgp_summary_dict.update(match.groupdict(1))\n\n    # Some post regex cleanup and validation\n    vrf = bgp_summary_dict[\"vrf\"]\n    if vrf.lower() == \"default\":\n        bgp_summary_dict[\"vrf\"] = \"global\"\n\n    afi = bgp_summary_dict[\"afi\"]\n    afi = afi.split()[0].lower()\n    if afi not in allowed_afi:\n        raise ValueError(\"AFI ({}) is invalid and not supported.\".format(afi))\n    bgp_summary_dict[\"afi\"] = afi\n\n    local_as = bgp_summary_dict[\"local_as\"]\n    local_as = helpers.as_number(local_as)\n\n    match = re.search(IPV4_ADDR_REGEX, bgp_summary_dict[\"router_id\"])\n    if not match:\n        raise ValueError(\n            \"BGP router_id ({}) is not valid\".format(bgp_summary_dict[\"router_id\"])\n        )\n\n    vrf = bgp_summary_dict[\"vrf\"]\n    bgp_return_dict = {vrf: {\"router_id\": bgp_summary_dict[\"router_id\"], \"peers\": {}}}\n\n    # Extract and process the tabular data\n    tabular_divider = r\"^Neighbor\\s+.*PfxRcd$\"\n    tabular_data = re.split(tabular_divider, bgp_summary, flags=re.M)\n    if len(tabular_data) != 2:\n        msg = \"Unexpected data processing BGP summary information:\\n\\n{}\".format(\n            bgp_summary\n        )\n        raise ValueError(msg)\n    tabular_data = tabular_data[1]\n    bgp_table = bgp_normalize_table_data(tabular_data)\n    for bgp_entry in bgp_table_parser(bgp_table):\n        bgp_return_dict[vrf][\"peers\"].update(bgp_entry)\n\n    bgp_new_dict = {}\n    for neighbor, bgp_data in bgp_return_dict[vrf][\"peers\"].items():\n        received_prefixes = bgp_data.pop(\"received_prefixes\")\n        bgp_data[\"address_family\"] = {}\n        prefixes_dict = {\n            \"sent_prefixes\": -1,\n            \"accepted_prefixes\": -1,\n            \"received_prefixes\": received_prefixes,\n        }\n        bgp_data[\"address_family\"][afi] = prefixes_dict\n        bgp_data[\"local_as\"] = local_as\n        # FIX, hard-coding\n        bgp_data[\"remote_id\"] = \"0.0.0.0\"\n        bgp_new_dict[neighbor] = bgp_data\n\n    bgp_return_dict[vrf][\"peers\"] = bgp_new_dict\n\n    return bgp_return_dict", "response": "Parse show bgp all summary vrf output information from NX - OS devices."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _send_command_list(self, commands):\n        output = \"\"\n        for command in commands:\n            output += self.device.send_command(\n                command, strip_prompt=False, strip_command=False\n            )\n        return output", "response": "Wrapper for Netmiko s send_command method for list of commands."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nextract the uptime string from the Cisco IOS Device.", "response": "def parse_uptime(uptime_str):\n        \"\"\"\n        Extract the uptime string from the given Cisco IOS Device.\n        Return the uptime in seconds as an integer\n        \"\"\"\n        # Initialize to zero\n        (years, weeks, days, hours, minutes) = (0, 0, 0, 0, 0)\n\n        uptime_str = uptime_str.strip()\n        time_list = uptime_str.split(\",\")\n        for element in time_list:\n            if re.search(\"year\", element):\n                years = int(element.split()[0])\n            elif re.search(\"week\", element):\n                weeks = int(element.split()[0])\n            elif re.search(\"day\", element):\n                days = int(element.split()[0])\n            elif re.search(\"hour\", element):\n                hours = int(element.split()[0])\n            elif re.search(\"minute\", element):\n                minutes = int(element.split()[0])\n            elif re.search(\"second\", element):\n                seconds = int(element.split()[0])\n\n        uptime_sec = (\n            (years * YEAR_SECONDS)\n            + (weeks * WEEK_SECONDS)\n            + (days * DAY_SECONDS)\n            + (hours * 3600)\n            + (minutes * 60)\n            + seconds\n        )\n        return uptime_sec"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef is_alive(self):\n        null = chr(0)\n        try:\n            if self.device is None:\n                return {\"is_alive\": False}\n            else:\n                # Try sending ASCII null byte to maintain the connection alive\n                self._send_command(null)\n        except (socket.error, EOFError):\n            # If unable to send, we can tell for sure that the connection is unusable,\n            # hence return False.\n            return {\"is_alive\": False}\n        return {\"is_alive\": self.device.remote_conn.transport.is_active()}", "response": "Returns a flag with the state of the SSH connection."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_facts(self):\n        # default values.\n        vendor = \"Cisco\"\n        uptime = -1\n        serial_number, fqdn, os_version, hostname, domain_name, model = (\"\",) * 6\n\n        # obtain output from device\n        show_ver = self._send_command(\"show version\")\n        show_hosts = self._send_command(\"show hosts\")\n        show_int_status = self._send_command(\"show interface status\")\n        show_hostname = self._send_command(\"show hostname\")\n\n        # uptime/serial_number/IOS version\n        for line in show_ver.splitlines():\n            if \" uptime is \" in line:\n                _, uptime_str = line.split(\" uptime is \")\n                uptime = self.parse_uptime(uptime_str)\n\n            if \"Processor Board ID\" in line:\n                _, serial_number = line.split(\"Processor Board ID \")\n                serial_number = serial_number.strip()\n\n            if \"system: \" in line or \"NXOS: \" in line:\n                line = line.strip()\n                os_version = line.split()[2]\n                os_version = os_version.strip()\n\n            if \"cisco\" in line and \"hassis\" in line:\n                match = re.search(r\".cisco (.*) \\(\", line)\n                if match:\n                    model = match.group(1).strip()\n                match = re.search(r\".cisco (.* [cC]hassis)\", line)\n                if match:\n                    model = match.group(1).strip()\n\n        hostname = show_hostname.strip()\n\n        # Determine domain_name and fqdn\n        for line in show_hosts.splitlines():\n            if \"Default domain\" in line:\n                _, domain_name = re.split(r\".*Default domain.*is \", line)\n                domain_name = domain_name.strip()\n                break\n        if hostname.count(\".\") >= 2:\n            fqdn = hostname\n            # Remove domain name from hostname\n            if domain_name:\n                hostname = re.sub(re.escape(domain_name) + \"$\", \"\", hostname)\n                hostname = hostname.strip(\".\")\n        elif domain_name:\n            fqdn = \"{}.{}\".format(hostname, domain_name)\n\n        # interface_list filter\n        interface_list = []\n        show_int_status = show_int_status.strip()\n        # Remove the header information\n        show_int_status = re.sub(\n            r\"(?:^---------+$|^Port .*$|^ .*$)\", \"\", show_int_status, flags=re.M\n        )\n        for line in show_int_status.splitlines():\n            if not line:\n                continue\n            interface = line.split()[0]\n            # Return canonical interface name\n            interface_list.append(helpers.canonical_interface_name(interface))\n\n        return {\n            \"uptime\": int(uptime),\n            \"vendor\": vendor,\n            \"os_version\": py23_compat.text_type(os_version),\n            \"serial_number\": py23_compat.text_type(serial_number),\n            \"model\": py23_compat.text_type(model),\n            \"hostname\": py23_compat.text_type(hostname),\n            \"fqdn\": fqdn,\n            \"interface_list\": interface_list,\n        }", "response": "Return a set of facts from the devices."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nget the list of interfaces.", "response": "def get_interfaces(self):\n        \"\"\"\n        Get interface details.\n\n        last_flapped is not implemented\n\n        Example Output:\n\n        {   u'Vlan1': {   'description': u'',\n                      'is_enabled': True,\n                      'is_up': True,\n                      'last_flapped': -1.0,\n                      'mac_address': u'a493.4cc1.67a7',\n                      'speed': 100},\n        u'Vlan100': {   'description': u'Data Network',\n                        'is_enabled': True,\n                        'is_up': True,\n                        'last_flapped': -1.0,\n                        'mac_address': u'a493.4cc1.67a7',\n                        'speed': 100},\n        u'Vlan200': {   'description': u'Voice Network',\n                        'is_enabled': True,\n                        'is_up': True,\n                        'last_flapped': -1.0,\n                        'mac_address': u'a493.4cc1.67a7',\n                        'speed': 100}}\n        \"\"\"\n        interfaces = {}\n        command = \"show interface\"\n        output = self._send_command(command)\n        if not output:\n            return {}\n\n        # Break output into per-interface sections (note, separator text is retained)\n        separator1 = r\"^\\S+\\s+is \\S+.*\\nadmin state is.*$\"\n        separator2 = r\"^.* is .*, line protocol is .*$\"\n        separator3 = r\"^.* is (?:down|up).*$\"\n        separators = r\"({}|{}|{})\".format(separator1, separator2, separator3)\n        interface_lines = re.split(separators, output, flags=re.M)\n\n        if len(interface_lines) == 1:\n            msg = \"Unexpected output data in '{}':\\n\\n{}\".format(\n                command, interface_lines\n            )\n            raise ValueError(msg)\n\n        # Get rid of the blank data at the beginning\n        interface_lines.pop(0)\n\n        # Must be pairs of data (the separator and section corresponding to it)\n        if len(interface_lines) % 2 != 0:\n            msg = \"Unexpected output data in '{}':\\n\\n{}\".format(\n                command, interface_lines\n            )\n            raise ValueError(msg)\n\n        # Combine the separator and section into one string\n        intf_iter = iter(interface_lines)\n        try:\n            new_interfaces = [line + next(intf_iter, \"\") for line in intf_iter]\n        except TypeError:\n            raise ValueError()\n\n        for entry in new_interfaces:\n            interfaces.update(parse_intf_section(entry))\n\n        return interfaces"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get_bgp_neighbors(self):\n        bgp_dict = {}\n\n        # get summary output from device\n        cmd_bgp_all_sum = \"show bgp all summary vrf all\"\n        bgp_summary_output = self._send_command(cmd_bgp_all_sum).strip()\n\n        section_separator = r\"BGP summary information for \"\n        bgp_summary_sections = re.split(section_separator, bgp_summary_output)\n        if len(bgp_summary_sections):\n            bgp_summary_sections.pop(0)\n\n        for bgp_section in bgp_summary_sections:\n            bgp_section = section_separator + bgp_section\n            bgp_dict.update(bgp_summary_parser(bgp_section))\n\n        # FIX -- look up logical or behavior we did in Cisco IOS bgp parser (make consistent here)\n        # FIX -- need to merge IPv6 and IPv4 AFI for same neighbor\n        return bgp_dict", "response": "Get BGP neighbor information."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nget the environment of the current state of the user.", "response": "def get_environment(self):\n        \"\"\"\n        Get environment facts.\n\n        power and fan are currently not implemented\n        cpu is using 1-minute average\n        \"\"\"\n\n        environment = {}\n        # sys_resources contains cpu and mem output\n        sys_resources = self._send_command(\"show system resources\")\n        temp_cmd = \"show environment temperature\"\n\n        # cpu\n        environment.setdefault(\"cpu\", {})\n        environment[\"cpu\"][\"0\"] = {}\n        environment[\"cpu\"][\"0\"][\"%usage\"] = -1.0\n        system_resources_cpu = helpers.textfsm_extractor(\n            self, \"system_resources\", sys_resources\n        )\n        for cpu in system_resources_cpu:\n            cpu_dict = {\n                cpu.get(\"cpu_id\"): {\n                    \"%usage\": round(100 - float(cpu.get(\"cpu_idle\")), 2)\n                }\n            }\n            environment[\"cpu\"].update(cpu_dict)\n\n        # memory\n        environment.setdefault(\"memory\", {})\n        for line in sys_resources.splitlines():\n            # Memory usage:   16401224K total,   4798280K used,   11602944K free\n            if \"Memory usage:\" in line:\n                proc_total_mem, proc_used_mem, _ = line.split(\",\")\n                proc_used_mem = re.search(r\"\\d+\", proc_used_mem).group(0)\n                proc_total_mem = re.search(r\"\\d+\", proc_total_mem).group(0)\n                break\n        else:\n            raise ValueError(\"Unexpected output from: {}\".format(line))\n        environment[\"memory\"][\"used_ram\"] = int(proc_used_mem)\n        environment[\"memory\"][\"available_ram\"] = int(proc_total_mem)\n\n        # temperature\n        output = self._send_command(temp_cmd)\n        environment.setdefault(\"temperature\", {})\n        for line in output.splitlines():\n            # Module   Sensor        MajorThresh   MinorThres   CurTemp     Status\n            # 1        Intake          70              42          28         Ok\n            if re.match(r\"^[0-9]\", line):\n                module, sensor, is_critical, is_alert, temp, _ = line.split()\n                is_critical = float(is_critical)\n                is_alert = float(is_alert)\n                temp = float(temp)\n                env_value = {\n                    \"is_alert\": temp >= is_alert,\n                    \"is_critical\": temp >= is_critical,\n                    \"temperature\": temp,\n                }\n                location = \"{0}-{1}\".format(sensor, module)\n                environment[\"temperature\"][location] = env_value\n\n        # Initialize 'power' and 'fan' to default values (not implemented)\n        environment.setdefault(\"power\", {})\n        environment[\"power\"][\"invalid\"] = {\n            \"status\": True,\n            \"output\": -1.0,\n            \"capacity\": -1.0,\n        }\n        environment.setdefault(\"fans\", {})\n        environment[\"fans\"][\"invalid\"] = {\"status\": True}\n\n        return environment"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nget the interfaces IP details. Returns a dictionary of dictionaries.", "response": "def get_interfaces_ip(self):\n        \"\"\"\n        Get interface IP details. Returns a dictionary of dictionaries.\n\n        Sample output:\n        {\n            \"Ethernet2/3\": {\n                \"ipv4\": {\n                    \"4.4.4.4\": {\n                        \"prefix_length\": 16\n                    }\n                },\n                \"ipv6\": {\n                    \"2001:db8::1\": {\n                        \"prefix_length\": 10\n                    },\n                    \"fe80::2ec2:60ff:fe4f:feb2\": {\n                        \"prefix_length\": \"128\"\n                    }\n                }\n            },\n            \"Ethernet2/2\": {\n                \"ipv4\": {\n                    \"2.2.2.2\": {\n                        \"prefix_length\": 27\n                    }\n                }\n            }\n        }\n        \"\"\"\n        interfaces_ip = {}\n        ipv4_command = \"show ip interface vrf all\"\n        ipv6_command = \"show ipv6 interface vrf all\"\n        output_v4 = self._send_command(ipv4_command)\n        output_v6 = self._send_command(ipv6_command)\n\n        v4_interfaces = {}\n        for line in output_v4.splitlines():\n            # Ethernet2/2, Interface status: protocol-up/link-up/admin-up, iod: 38,\n            # IP address: 2.2.2.2, IP subnet: 2.2.2.0/27 route-preference: 0, tag: 0\n            # IP address: 3.3.3.3, IP subnet: 3.3.3.0/25 secondary route-preference: 0, tag: 0\n            if \"Interface status\" in line:\n                interface = line.split(\",\")[0]\n                continue\n            if \"IP address\" in line:\n                ip_address = line.split(\",\")[0].split()[2]\n                try:\n                    prefix_len = int(line.split()[5].split(\"/\")[1])\n                except ValueError:\n                    prefix_len = \"N/A\"\n                val = {\"prefix_length\": prefix_len}\n                v4_interfaces.setdefault(interface, {})[ip_address] = val\n\n        v6_interfaces = {}\n        for line in output_v6.splitlines():\n            # Ethernet2/4, Interface status: protocol-up/link-up/admin-up, iod: 40\n            # IPv6 address:\n            #   2001:11:2233::a1/24 [VALID]\n            #   2001:cc11:22bb:0:2ec2:60ff:fe4f:feb2/64 [VALID]\n            # IPv6 subnet:  2001::/24\n            # IPv6 link-local address: fe80::2ec2:60ff:fe4f:feb2 (default) [VALID]\n            # IPv6 address: fe80::a293:51ff:fe5f:5ce9 [VALID]\n            if \"Interface status\" in line:\n                interface = line.split(\",\")[0]\n                continue\n            if \"VALID\" in line:\n                line = line.strip()\n                if \"link-local address\" in line:\n                    # match the following format:\n                    # IPv6 link-local address: fe80::2ec2:60ff:fe4f:feb2 (default) [VALID]\n                    ip_address = line.split()[3]\n                    prefix_len = \"64\"\n                elif \"IPv6 address\" in line:\n                    # match the following format:\n                    # IPv6 address: fe80::a293:51ff:fe5f:5ce9 [VALID]\n                    ip_address = line.split()[2]\n                    prefix_len = \"64\"\n                else:\n                    ip_address, prefix_len = line.split()[0].split(\"/\")\n                prefix_len = int(prefix_len)\n                val = {\"prefix_length\": prefix_len}\n                v6_interfaces.setdefault(interface, {})[ip_address] = val\n\n        # Join data from intermediate dictionaries.\n        for interface, data in v4_interfaces.items():\n            interfaces_ip.setdefault(interface, {\"ipv4\": {}})[\"ipv4\"] = data\n\n        for interface, data in v6_interfaces.items():\n            interfaces_ip.setdefault(interface, {\"ipv6\": {}})[\"ipv6\"] = data\n\n        return interfaces_ip"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _get_bgp_route_attr(self, destination, vrf, next_hop, ip_version=4):\n\n        CMD_SHIBNV = 'show ip bgp neighbors vrf {vrf} | include \"is {neigh}\"'\n\n        search_re_dict = {\n            \"aspath\": {\n                \"re\": r\"AS-Path: ([\\d\\(\\)]([\\d\\(\\) ])*)\",\n                \"group\": 1,\n                \"default\": \"\",\n            },\n            \"bgpnh\": {\n                \"re\": r\"[^|\\\\n][ ]{4}(\" + IP_ADDR_REGEX + r\")\",\n                \"group\": 1,\n                \"default\": \"\",\n            },\n            \"bgpfrom\": {\n                \"re\": r\"from (\" + IP_ADDR_REGEX + r\")\",\n                \"group\": 1,\n                \"default\": \"\",\n            },\n            \"bgpcomm\": {\n                \"re\": r\"  Community: ([\\w\\d\\-\\: ]+)\",\n                \"group\": 1,\n                \"default\": \"\",\n            },\n            \"bgplp\": {\"re\": r\"localpref (\\d+)\", \"group\": 1, \"default\": \"\"},\n            # external, internal, redist\n            \"bgpie\": {\"re\": r\"^: (\\w+),\", \"group\": 1, \"default\": \"\"},\n            \"vrfimp\": {\n                \"re\": r\"Imported from [\\S]+ \\(VRF (\\S+)\\)\",\n                \"group\": 1,\n                \"default\": \"\",\n            },\n        }\n\n        bgp_attr = {}\n        # get BGP AS number\n        outbgp = self._send_command('show bgp process | include \"BGP Protocol Tag\"')\n        matchbgpattr = RE_BGP_PROTO_TAG.match(outbgp)\n        if not matchbgpattr:\n            return bgp_attr\n        bgpas = matchbgpattr.group(1)\n        if ip_version == 4:\n            bgpcmd = \"show ip bgp vrf {vrf} {destination}\".format(\n                vrf=vrf, destination=destination\n            )\n            outbgp = self._send_command(bgpcmd)\n            outbgpsec = outbgp.split(\"Path type\")\n\n            # this should not happen (zero BGP paths)...\n            if len(outbgpsec) == 1:\n                return bgp_attr\n\n            # process all bgp paths\n            for bgppath in outbgpsec[1:]:\n                if \"is best path\" not in bgppath:\n                    # only best path is added to protocol attributes\n                    continue\n                # find BGP attributes\n                for key in search_re_dict:\n                    matchre = re.search(search_re_dict[key][\"re\"], bgppath)\n                    if matchre:\n                        groupnr = int(search_re_dict[key][\"group\"])\n                        search_re_dict[key][\"result\"] = matchre.group(groupnr)\n                    else:\n                        search_re_dict[key][\"result\"] = search_re_dict[key][\"default\"]\n                bgpnh = search_re_dict[\"bgpnh\"][\"result\"]\n\n                # if route is not leaked next hops have to match\n                if (\n                    not (search_re_dict[\"bgpie\"][\"result\"] in [\"redist\", \"local\"])\n                ) and (bgpnh != next_hop):\n                    # this is not the right route\n                    continue\n                # find remote AS nr. of this neighbor\n                bgpcmd = CMD_SHIBNV.format(vrf=vrf, neigh=bgpnh)\n                outbgpnei = self._send_command(bgpcmd)\n                matchbgpras = RE_BGP_REMOTE_AS.search(outbgpnei)\n                if matchbgpras:\n                    bgpras = matchbgpras.group(1)\n                else:\n                    # next-hop is not known in this vrf, route leaked from\n                    #  other vrf or from vpnv4 table?\n                    # get remote AS nr. from as-path if it is ebgp neighbor\n                    # if locally sourced remote AS if undefined\n                    bgpie = search_re_dict[\"bgpie\"][\"result\"]\n                    if bgpie == \"external\":\n                        bgpras = bgpie.split(\" \")[0].replace(\"(\", \"\")\n                    elif bgpie == \"internal\":\n                        bgpras = bgpas\n                    else:  # redist, local\n                        bgpras = \"\"\n                # community\n                bothcomm = []\n                extcomm = []\n                stdcomm = search_re_dict[\"bgpcomm\"][\"result\"].split()\n                commsplit = bgppath.split(\"Extcommunity:\")\n                if len(commsplit) == 2:\n                    for line in commsplit[1].split(\"\\n\")[1:]:\n                        #          RT:65004:22\n                        matchcommun = RE_BGP_COMMUN.match(line)\n                        if matchcommun:\n                            extcomm.append(matchcommun.group(1))\n                        else:\n                            # we've reached end of the extended community section\n                            break\n                bothcomm = stdcomm + extcomm\n                bgp_attr = {\n                    \"as_path\": search_re_dict[\"aspath\"][\"result\"].strip(),\n                    \"remote_address\": search_re_dict[\"bgpfrom\"][\"result\"],\n                    \"local_preference\": int(search_re_dict[\"bgplp\"][\"result\"]),\n                    \"communities\": bothcomm,\n                    \"local_as\": helpers.as_number(bgpas),\n                }\n                if bgpras:\n                    bgp_attr[\"remote_as\"] = helpers.as_number(bgpras)\n                else:\n                    bgp_attr[\"remote_as\"] = 0  # 0? , locally sourced\n        return bgp_attr", "response": "Get BGP protocol attributes for get_route_tp\n Only IPv4 supported"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_route_to(self, destination=\"\", protocol=\"\"):\n        longer_pref = \"\"  # longer_prefixes support, for future use\n        vrf = \"\"\n\n        ip_version = None\n        try:\n            ip_version = IPNetwork(destination).version\n        except AddrFormatError:\n            return \"Please specify a valid destination!\"\n        if ip_version == 4:  # process IPv4 routing table\n            routes = {}\n            if vrf:\n                send_cmd = \"show ip route vrf {vrf} {destination} {longer}\".format(\n                    vrf=vrf, destination=destination, longer=longer_pref\n                ).rstrip()\n            else:\n                send_cmd = \"show ip route vrf all {destination} {longer}\".format(\n                    destination=destination, longer=longer_pref\n                ).rstrip()\n            out_sh_ip_rou = self._send_command(send_cmd)\n            # IP Route Table for VRF \"TEST\"\n            for vrfsec in out_sh_ip_rou.split(\"IP Route Table for \")[1:]:\n                if \"Route not found\" in vrfsec:\n                    continue\n                vrffound = False\n                preffound = False\n                nh_list = []\n                cur_prefix = \"\"\n                for line in vrfsec.split(\"\\n\"):\n                    if not vrffound:\n                        vrfstr = RE_RT_VRF_NAME.match(line)\n                        if vrfstr:\n                            curvrf = vrfstr.group(1)\n                            vrffound = True\n                    else:\n                        # 10.10.56.0/24, ubest/mbest: 2/0\n                        prefstr = RE_RT_IPV4_ROUTE_PREF.match(line)\n                        if prefstr:\n                            if preffound:  # precess previous prefix\n                                if cur_prefix not in routes:\n                                    routes[cur_prefix] = []\n                                for nh in nh_list:\n                                    routes[cur_prefix].append(nh)\n                                nh_list = []\n                            else:\n                                preffound = True\n                            cur_prefix = prefstr.group(1)\n                            continue\n                        #     *via 10.2.49.60, Vlan3013, [0/0], 1y18w, direct\n                        #      via 10.17.205.132, Po77.3602, [110/20], 1y18w, ospf-1000,\n                        #            type-2, tag 2112\n                        #     *via 10.17.207.42, Eth3/7.212, [110/20], 02:19:36, ospf-1000, type-2,\n                        #            tag 2121\n                        #     *via 10.17.207.73, [1/0], 1y18w, static\n                        #     *via 10.17.209.132%vrf2, Po87.3606, [20/20], 1y25w, bgp-65000,\n                        #            external, tag 65000\n                        #     *via Vlan596, [1/0], 1y18w, static\n                        viastr = RE_IP_ROUTE_VIA_REGEX.match(line)\n                        if viastr:\n                            nh_used = viastr.group(\"used\") == \"*\"\n                            nh_ip = viastr.group(\"ip\") or \"\"\n                            # when next hop is leaked from other vrf, for future use\n                            # nh_vrf = viastr.group('vrf')\n                            nh_int = viastr.group(\"int\")\n                            nh_metric = viastr.group(\"metric\")\n                            nh_age = bgp_time_conversion(viastr.group(\"age\"))\n                            nh_source = viastr.group(\"source\")\n                            # for future use\n                            # rest_of_line = viastr.group('rest')\n                            # use only routes from specified protocol\n                            if protocol and protocol != nh_source:\n                                continue\n                            # routing protocol process number, for future use\n                            # nh_source_proc_nr = viastr.group('procnr)\n                            if nh_int:\n                                nh_int_canon = helpers.canonical_interface_name(nh_int)\n                            else:\n                                nh_int_canon = \"\"\n                            route_entry = {\n                                \"protocol\": nh_source,\n                                \"outgoing_interface\": nh_int_canon,\n                                \"age\": nh_age,\n                                \"current_active\": nh_used,\n                                \"routing_table\": curvrf,\n                                \"last_active\": nh_used,\n                                \"next_hop\": nh_ip,\n                                \"selected_next_hop\": nh_used,\n                                \"inactive_reason\": \"\",\n                                \"preference\": int(nh_metric),\n                            }\n                            if nh_source == \"bgp\":\n                                route_entry[\n                                    \"protocol_attributes\"\n                                ] = self._get_bgp_route_attr(cur_prefix, curvrf, nh_ip)\n                            else:\n                                route_entry[\"protocol_attributes\"] = {}\n                            nh_list.append(route_entry)\n                # process last next hop entries\n                if preffound:\n                    if cur_prefix not in routes:\n                        routes[cur_prefix] = []\n                    for nh in nh_list:\n                        routes[cur_prefix].append(nh)\n        return routes", "response": "Get route to destination."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _netmiko_open(self, device_type, netmiko_optional_args=None):\n        if netmiko_optional_args is None:\n            netmiko_optional_args = {}\n        try:\n            self._netmiko_device = ConnectHandler(\n                device_type=device_type,\n                host=self.hostname,\n                username=self.username,\n                password=self.password,\n                timeout=self.timeout,\n                **netmiko_optional_args\n            )\n        except NetMikoTimeoutException:\n            raise ConnectionException(\"Cannot connect to {}\".format(self.hostname))\n\n        # ensure in enable mode\n        self._netmiko_device.enable()\n        return self._netmiko_device", "response": "Standardized method of creating a Netmiko connection using napalm attributes."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef load_template(\n        self, template_name, template_source=None, template_path=None, **template_vars\n    ):\n        \"\"\"\n        Will load a templated configuration on the device.\n\n        :param cls: Instance of the driver class.\n        :param template_name: Identifies the template name.\n        :param template_source (optional): Custom config template rendered and loaded on device\n        :param template_path (optional): Absolute path to directory for the configuration templates\n        :param template_vars: Dictionary with arguments to be used when the template is rendered.\n        :raise DriverTemplateNotImplemented: No template defined for the device type.\n        :raise TemplateNotImplemented: The template specified in template_name does not exist in \\\n        the default path or in the custom path if any specified using parameter `template_path`.\n        :raise TemplateRenderException: The template could not be rendered. Either the template \\\n        source does not have the right format, either the arguments in `template_vars` are not \\\n        properly specified.\n        \"\"\"\n        return napalm.base.helpers.load_template(\n            self,\n            template_name,\n            template_source=template_source,\n            template_path=template_path,\n            **template_vars\n        )", "response": "Load a templated configuration on the device."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nexecute a traceroute on the device and returns the result.", "response": "def traceroute(\n        self,\n        destination,\n        source=c.TRACEROUTE_SOURCE,\n        ttl=c.TRACEROUTE_TTL,\n        timeout=c.TRACEROUTE_TIMEOUT,\n        vrf=c.TRACEROUTE_VRF,\n    ):\n        \"\"\"\n        Executes traceroute on the device and returns a dictionary with the result.\n\n        :param destination: Host or IP Address of the destination\n        :param source (optional): Use a specific IP Address to execute the traceroute\n        :param ttl (optional): Maimum number of hops\n        :param timeout (optional): Number of seconds to wait for response\n\n        Output dictionary has one of the following keys:\n\n            * success\n            * error\n\n        In case of success, the keys of the dictionary represent the hop ID, while values are\n        dictionaries containing the probes results:\n\n            * rtt (float)\n            * ip_address (str)\n            * host_name (str)\n\n        Example::\n\n            {\n                'success': {\n                    1: {\n                        'probes': {\n                            1: {\n                                'rtt': 1.123,\n                                'ip_address': u'206.223.116.21',\n                                'host_name': u'eqixsj-google-gige.google.com'\n                            },\n                            2: {\n                                'rtt': 1.9100000000000001,\n                                'ip_address': u'206.223.116.21',\n                                'host_name': u'eqixsj-google-gige.google.com'\n                            },\n                            3: {\n                                'rtt': 3.347,\n                                'ip_address': u'198.32.176.31',\n                                'host_name': u'core2-1-1-0.pao.net.google.com'}\n                            }\n                        },\n                        2: {\n                            'probes': {\n                                1: {\n                                    'rtt': 1.586,\n                                    'ip_address': u'209.85.241.171',\n                                    'host_name': u'209.85.241.171'\n                                    },\n                                2: {\n                                    'rtt': 1.6300000000000001,\n                                    'ip_address': u'209.85.241.171',\n                                    'host_name': u'209.85.241.171'\n                                },\n                                3: {\n                                    'rtt': 1.6480000000000001,\n                                    'ip_address': u'209.85.241.171',\n                                    'host_name': u'209.85.241.171'}\n                                }\n                            },\n                        3: {\n                            'probes': {\n                                1: {\n                                    'rtt': 2.529,\n                                    'ip_address': u'216.239.49.123',\n                                    'host_name': u'216.239.49.123'},\n                                2: {\n                                    'rtt': 2.474,\n                                    'ip_address': u'209.85.255.255',\n                                    'host_name': u'209.85.255.255'\n                                },\n                                3: {\n                                    'rtt': 7.813,\n                                    'ip_address': u'216.239.58.193',\n                                    'host_name': u'216.239.58.193'}\n                                }\n                            },\n                        4: {\n                            'probes': {\n                                1: {\n                                    'rtt': 1.361,\n                                    'ip_address': u'8.8.8.8',\n                                    'host_name': u'google-public-dns-a.google.com'\n                                },\n                                2: {\n                                    'rtt': 1.605,\n                                    'ip_address': u'8.8.8.8',\n                                    'host_name': u'google-public-dns-a.google.com'\n                                },\n                                3: {\n                                    'rtt': 0.989,\n                                    'ip_address': u'8.8.8.8',\n                                    'host_name': u'google-public-dns-a.google.com'}\n                                }\n                            }\n                        }\n                    }\n\n            OR\n\n            {\n                'error': 'unknown host 8.8.8.8.8'\n            }\n            \"\"\"\n        raise NotImplementedError"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning a compliance report.", "response": "def compliance_report(self, validation_file=None, validation_source=None):\n        \"\"\"\n        Return a compliance report.\n\n        Verify that the device complies with the given validation file and writes a compliance\n        report file. See https://napalm.readthedocs.io/en/latest/validate/index.html.\n\n        :param validation_file: Path to the file containing compliance definition. Default is None.\n        :param validation_source: Dictionary containing compliance rules.\n        :raise ValidationException: File is not valid.\n        :raise NotImplementedError: Method not implemented.\n        \"\"\"\n        return validate.compliance_report(\n            self, validation_file=validation_file, validation_source=validation_source\n        )"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nexpose the helper function within this class.", "response": "def _canonical_int(self, interface):\n        \"\"\"Expose the helper function within this class.\"\"\"\n        if self.use_canonical_interface is True:\n            return napalm.base.helpers.canonical_interface_name(\n                interface, addl_name_map=None\n            )\n        else:\n            return interface"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef open(self):\n        try:\n            self.device.open()\n        except ConnectTimeoutError as cte:\n            raise ConnectionException(cte.msg)\n        self.device.timeout = self.timeout\n        self.device._conn._session.transport.set_keepalive(self.keepalive)\n        if hasattr(self.device, \"cu\"):\n            # make sure to remove the cu attr from previous session\n            # ValueError: requested attribute name cu already exists\n            del self.device.cu\n        self.device.bind(cu=Config)\n        if not self.lock_disable and self.session_config_lock:\n            self._lock()", "response": "Open the connection with the device."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nlocks the config DB.", "response": "def _lock(self):\n        \"\"\"Lock the config DB.\"\"\"\n        if not self.locked:\n            try:\n                self.device.cu.lock()\n                self.locked = True\n            except JnprLockError as jle:\n                raise LockError(py23_compat.text_type(jle))"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _unlock(self):\n        if self.locked:\n            try:\n                self.device.cu.unlock()\n                self.locked = False\n            except JnrpUnlockError as jue:\n                raise UnlockError(jue.messsage)", "response": "Unlock the config DB."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef load_replace_candidate(self, filename=None, config=None):\n        self.config_replace = True\n        self._load_candidate(filename, config, True)", "response": "Open the candidate config and merge."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nopens the candidate config and replace.", "response": "def load_merge_candidate(self, filename=None, config=None):\n        \"\"\"Open the candidate config and replace.\"\"\"\n        self.config_replace = False\n        self._load_candidate(filename, config, False)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ncommitting the current configuration.", "response": "def commit_config(self, message=\"\"):\n        \"\"\"Commit configuration.\"\"\"\n        commit_args = {\"comment\": message} if message else {}\n        self.device.cu.commit(ignore_warning=self.ignore_warning, **commit_args)\n        if not self.lock_disable and not self.session_config_lock:\n            self._unlock()"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ndiscarding changes ( rollback 0 ).", "response": "def discard_config(self):\n        \"\"\"Discard changes (rollback 0).\"\"\"\n        self.device.cu.rollback(rb_id=0)\n        if not self.lock_disable and not self.session_config_lock:\n            self._unlock()"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_interfaces(self):\n        result = {}\n\n        interfaces = junos_views.junos_iface_table(self.device)\n        interfaces.get()\n        interfaces_logical = junos_views.junos_logical_iface_table(self.device)\n        interfaces_logical.get()\n\n        # convert all the tuples to our pre-defined dict structure\n        def _convert_to_dict(interfaces):\n            # calling .items() here wont work.\n            # The dictionary values will end up being tuples instead of dictionaries\n            interfaces = dict(interfaces)\n            for iface, iface_data in interfaces.items():\n                result[iface] = {\n                    \"is_up\": iface_data[\"is_up\"],\n                    # For physical interfaces <admin-status> will always be there, so just\n                    # return the value interfaces[iface]['is_enabled']\n                    # For logical interfaces if <iff-down> is present interface is disabled,\n                    # otherwise interface is enabled\n                    \"is_enabled\": (\n                        True\n                        if iface_data[\"is_enabled\"] is None\n                        else iface_data[\"is_enabled\"]\n                    ),\n                    \"description\": (iface_data[\"description\"] or \"\"),\n                    \"last_flapped\": float((iface_data[\"last_flapped\"] or -1)),\n                    \"mac_address\": napalm.base.helpers.convert(\n                        napalm.base.helpers.mac,\n                        iface_data[\"mac_address\"],\n                        py23_compat.text_type(iface_data[\"mac_address\"]),\n                    ),\n                    \"speed\": -1,\n                    \"mtu\": 0,\n                }\n                # result[iface]['last_flapped'] = float(result[iface]['last_flapped'])\n\n                match_mtu = re.search(r\"(\\w+)\", str(iface_data[\"mtu\"]) or \"\")\n                mtu = napalm.base.helpers.convert(int, match_mtu.group(0), 0)\n                result[iface][\"mtu\"] = mtu\n                match = re.search(r\"(\\d+|[Aa]uto)(\\w*)\", iface_data[\"speed\"] or \"\")\n                if match and match.group(1).lower() == \"auto\":\n                    match = re.search(\n                        r\"(\\d+)(\\w*)\", iface_data[\"negotiated_speed\"] or \"\"\n                    )\n                if match is None:\n                    continue\n                speed_value = napalm.base.helpers.convert(int, match.group(1), -1)\n                if speed_value == -1:\n                    continue\n                speed_unit = match.group(2)\n                if speed_unit.lower() == \"gbps\":\n                    speed_value *= 1000\n                result[iface][\"speed\"] = speed_value\n\n            return result\n\n        result = _convert_to_dict(interfaces)\n        result.update(_convert_to_dict(interfaces_logical))\n        return result", "response": "Return the details of all interfaces on the device."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get_environment(self):\n        environment = junos_views.junos_environment_table(self.device)\n        routing_engine = junos_views.junos_routing_engine_table(self.device)\n        temperature_thresholds = junos_views.junos_temperature_thresholds(self.device)\n        power_supplies = junos_views.junos_pem_table(self.device)\n        environment.get()\n        routing_engine.get()\n        temperature_thresholds.get()\n        environment_data = {}\n        current_class = None\n\n        for sensor_object, object_data in environment.items():\n            structured_object_data = {k: v for k, v in object_data}\n\n            if structured_object_data[\"class\"]:\n                # If current object has a 'class' defined, store it for use\n                # on subsequent unlabeled lines.\n                current_class = structured_object_data[\"class\"]\n            else:\n                # Juniper doesn't label the 2nd+ lines of a given class with a\n                # class name.  In that case, we use the most recent class seen.\n                structured_object_data[\"class\"] = current_class\n\n            if structured_object_data[\"class\"] == \"Power\":\n                # Create a dict for the 'power' key\n                try:\n                    environment_data[\"power\"][sensor_object] = {}\n                except KeyError:\n                    environment_data[\"power\"] = {}\n                    environment_data[\"power\"][sensor_object] = {}\n\n                environment_data[\"power\"][sensor_object][\"capacity\"] = -1.0\n                environment_data[\"power\"][sensor_object][\"output\"] = -1.0\n\n            if structured_object_data[\"class\"] == \"Fans\":\n                # Create a dict for the 'fans' key\n                try:\n                    environment_data[\"fans\"][sensor_object] = {}\n                except KeyError:\n                    environment_data[\"fans\"] = {}\n                    environment_data[\"fans\"][sensor_object] = {}\n\n            status = structured_object_data[\"status\"]\n            env_class = structured_object_data[\"class\"]\n            if status == \"OK\" and env_class == \"Power\":\n                # If status is Failed, Absent or Testing, set status to False.\n                environment_data[\"power\"][sensor_object][\"status\"] = True\n\n            elif status != \"OK\" and env_class == \"Power\":\n                environment_data[\"power\"][sensor_object][\"status\"] = False\n\n            elif status == \"OK\" and env_class == \"Fans\":\n                # If status is Failed, Absent or Testing, set status to False.\n                environment_data[\"fans\"][sensor_object][\"status\"] = True\n\n            elif status != \"OK\" and env_class == \"Fans\":\n                environment_data[\"fans\"][sensor_object][\"status\"] = False\n\n            for temperature_object, temperature_data in temperature_thresholds.items():\n                structured_temperature_data = {k: v for k, v in temperature_data}\n                if structured_object_data[\"class\"] == \"Temp\":\n                    # Create a dict for the 'temperature' key\n                    try:\n                        environment_data[\"temperature\"][sensor_object] = {}\n                    except KeyError:\n                        environment_data[\"temperature\"] = {}\n                        environment_data[\"temperature\"][sensor_object] = {}\n                    # Check we have a temperature field in this class (See #66)\n                    if structured_object_data[\"temperature\"]:\n                        environment_data[\"temperature\"][sensor_object][\n                            \"temperature\"\n                        ] = float(structured_object_data[\"temperature\"])\n                    # Set a default value (False) to the key is_critical and is_alert\n                    environment_data[\"temperature\"][sensor_object][\"is_alert\"] = False\n                    environment_data[\"temperature\"][sensor_object][\n                        \"is_critical\"\n                    ] = False\n                    # Check if the working temperature is equal to or higher than alerting threshold\n                    temp = structured_object_data[\"temperature\"]\n                    if temp is not None:\n                        if structured_temperature_data[\"red-alarm\"] <= temp:\n                            environment_data[\"temperature\"][sensor_object][\n                                \"is_critical\"\n                            ] = True\n                            environment_data[\"temperature\"][sensor_object][\n                                \"is_alert\"\n                            ] = True\n                        elif structured_temperature_data[\"yellow-alarm\"] <= temp:\n                            environment_data[\"temperature\"][sensor_object][\n                                \"is_alert\"\n                            ] = True\n                    else:\n                        environment_data[\"temperature\"][sensor_object][\n                            \"temperature\"\n                        ] = 0.0\n\n        # Try to correct Power Supply information\n        pem_table = dict()\n        try:\n            power_supplies.get()\n        except RpcError:\n            # Not all platforms have support for this\n            pass\n        else:\n            # Format PEM information and correct capacity and output values\n            if \"power\" not in environment_data.keys():\n                # Power supplies were not included from the environment table above\n                # Need to initialize data\n                environment_data[\"power\"] = {}\n                for pem in power_supplies.items():\n                    pem_name = pem[0].replace(\"PEM\", \"Power Supply\")\n                    environment_data[\"power\"][pem_name] = {}\n                    environment_data[\"power\"][pem_name][\"output\"] = -1.0\n                    environment_data[\"power\"][pem_name][\"capacity\"] = -1.0\n                    environment_data[\"power\"][pem_name][\"status\"] = False\n            for pem in power_supplies.items():\n                pem_name = pem[0].replace(\"PEM\", \"Power Supply\")\n                pem_table[pem_name] = dict(pem[1])\n                if pem_table[pem_name][\"capacity\"] is not None:\n                    environment_data[\"power\"][pem_name][\"capacity\"] = pem_table[\n                        pem_name\n                    ][\"capacity\"]\n                if pem_table[pem_name][\"output\"] is not None:\n                    environment_data[\"power\"][pem_name][\"output\"] = pem_table[pem_name][\n                        \"output\"\n                    ]\n                environment_data[\"power\"][pem_name][\"status\"] = pem_table[pem_name][\n                    \"status\"\n                ]\n\n        for routing_engine_object, routing_engine_data in routing_engine.items():\n            structured_routing_engine_data = {k: v for k, v in routing_engine_data}\n            # Create dicts for 'cpu' and 'memory'.\n            try:\n                environment_data[\"cpu\"][routing_engine_object] = {}\n                environment_data[\"memory\"] = {}\n            except KeyError:\n                environment_data[\"cpu\"] = {}\n                environment_data[\"cpu\"][routing_engine_object] = {}\n                environment_data[\"memory\"] = {}\n            # Calculate the CPU usage by using the CPU idle value.\n            environment_data[\"cpu\"][routing_engine_object][\"%usage\"] = (\n                100.0 - structured_routing_engine_data[\"cpu-idle\"]\n            )\n            try:\n                environment_data[\"memory\"][\"available_ram\"] = int(\n                    structured_routing_engine_data[\"memory-dram-size\"]\n                )\n            except ValueError:\n                environment_data[\"memory\"][\"available_ram\"] = int(\n                    \"\".join(\n                        i\n                        for i in structured_routing_engine_data[\"memory-dram-size\"]\n                        if i.isdigit()\n                    )\n                )\n            # Junos gives us RAM in %, so calculation has to be made.\n            # Sadly, bacause of this, results are not 100% accurate to the truth.\n            environment_data[\"memory\"][\"used_ram\"] = int(\n                round(\n                    environment_data[\"memory\"][\"available_ram\"]\n                    / 100.0\n                    * structured_routing_engine_data[\"memory-buffer-utilization\"]\n                )\n            )\n\n        return environment_data", "response": "Return the environment details."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _get_address_family(table, instance):\n        address_family_mapping = {\"inet\": \"ipv4\", \"inet6\": \"ipv6\", \"inetflow\": \"flow\"}\n        if instance == \"master\":\n            family = table.rsplit(\".\", 1)[-2]\n        else:\n            family = table.split(\".\")[-2]\n        try:\n            address_family = address_family_mapping[family]\n        except KeyError:\n            address_family = None\n        return address_family", "response": "Function to derive the address family from a junos routing table name."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef get_lldp_neighbors_detail(self, interface=\"\"):\n        lldp_neighbors = defaultdict(list)\n        lldp_table = junos_views.junos_lldp_neighbors_detail_table(self.device)\n        if not interface:\n            try:\n                lldp_table.get()\n            except RpcError as rpcerr:\n                # this assumes the library runs in an environment\n                # able to handle logs\n                # otherwise, the user just won't see this happening\n                log.error(\"Unable to retrieve the LLDP neighbors information:\")\n                log.error(py23_compat.text_type(rpcerr))\n                return {}\n            interfaces = lldp_table.get().keys()\n        else:\n            interfaces = [interface]\n\n        if self.device.facts.get(\"switch_style\") == \"VLAN\":\n            lldp_table.GET_RPC = \"get-lldp-interface-neighbors-information\"\n            interface_variable = \"interface_name\"\n            alt_rpc = \"get-lldp-interface-neighbors\"\n            alt_interface_variable = \"interface_device\"\n        else:\n            lldp_table.GET_RPC = \"get-lldp-interface-neighbors\"\n            interface_variable = \"interface_device\"\n            alt_rpc = \"get-lldp-interface-neighbors-information\"\n            alt_interface_variable = \"interface_name\"\n\n        for interface in interfaces:\n            try:\n                interface_args = {interface_variable: interface}\n                lldp_table.get(**interface_args)\n            except RpcError as e:\n                if \"syntax error\" in e.message:\n                    # Looks like we need to call a different RPC on this device\n                    # Switch to the alternate style\n                    lldp_table.GET_RPC = alt_rpc\n                    interface_variable = alt_interface_variable\n                    # Retry\n                    interface_args = {interface_variable: interface}\n                    lldp_table.get(**interface_args)\n\n            for item in lldp_table:\n                lldp_neighbors[interface].append(\n                    {\n                        \"parent_interface\": item.parent_interface,\n                        \"remote_port\": item.remote_port or \"\",\n                        \"remote_chassis_id\": napalm.base.helpers.convert(\n                            napalm.base.helpers.mac,\n                            item.remote_chassis_id,\n                            item.remote_chassis_id,\n                        ),\n                        \"remote_port_description\": napalm.base.helpers.convert(\n                            py23_compat.text_type, item.remote_port_description\n                        ),\n                        \"remote_system_name\": item.remote_system_name,\n                        \"remote_system_description\": item.remote_system_description,\n                        \"remote_system_capab\": self._transform_lldp_capab(\n                            item.remote_system_capab\n                        ),\n                        \"remote_system_enable_capab\": self._transform_lldp_capab(\n                            item.remote_system_enable_capab\n                        ),\n                    }\n                )\n\n        return lldp_neighbors", "response": "Detailed view of the LLDP neighbors."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nexecuting raw CLI commands and returns their output.", "response": "def cli(self, commands):\n        \"\"\"Execute raw CLI commands and returns their output.\"\"\"\n        cli_output = {}\n\n        def _count(txt, none):  # Second arg for consistency only. noqa\n            \"\"\"\n            Return the exact output, as Junos displays\n            e.g.:\n            > show system processes extensive | match root | count\n            Count: 113 lines\n            \"\"\"\n            count = len(txt.splitlines())\n            return \"Count: {count} lines\".format(count=count)\n\n        def _trim(txt, length):\n            \"\"\"\n            Trim specified number of columns from start of line.\n            \"\"\"\n            try:\n                newlines = []\n                for line in txt.splitlines():\n                    newlines.append(line[int(length) :])\n                return \"\\n\".join(newlines)\n            except ValueError:\n                return txt\n\n        def _except(txt, pattern):\n            \"\"\"\n            Show only text that does not match a pattern.\n            \"\"\"\n            rgx = \"^.*({pattern}).*$\".format(pattern=pattern)\n            unmatched = [\n                line for line in txt.splitlines() if not re.search(rgx, line, re.I)\n            ]\n            return \"\\n\".join(unmatched)\n\n        def _last(txt, length):\n            \"\"\"\n            Display end of output only.\n            \"\"\"\n            try:\n                return \"\\n\".join(txt.splitlines()[(-1) * int(length) :])\n            except ValueError:\n                return txt\n\n        def _match(txt, pattern):\n            \"\"\"\n            Show only text that matches a pattern.\n            \"\"\"\n            rgx = \"^.*({pattern}).*$\".format(pattern=pattern)\n            matched = [line for line in txt.splitlines() if re.search(rgx, line, re.I)]\n            return \"\\n\".join(matched)\n\n        def _find(txt, pattern):\n            \"\"\"\n            Search for first occurrence of pattern.\n            \"\"\"\n            rgx = \"^.*({pattern})(.*)$\".format(pattern=pattern)\n            match = re.search(rgx, txt, re.I | re.M | re.DOTALL)\n            if match:\n                return \"{pattern}{rest}\".format(pattern=pattern, rest=match.group(2))\n            else:\n                return \"\\nPattern not found\"\n\n        def _process_pipe(cmd, txt):\n            \"\"\"\n            Process CLI output from Juniper device that\n            doesn't allow piping the output.\n            \"\"\"\n            if txt is None:\n                return txt\n            _OF_MAP = OrderedDict()\n            _OF_MAP[\"except\"] = _except\n            _OF_MAP[\"match\"] = _match\n            _OF_MAP[\"last\"] = _last\n            _OF_MAP[\"trim\"] = _trim\n            _OF_MAP[\"count\"] = _count\n            _OF_MAP[\"find\"] = _find\n            # the operations order matter in this case!\n            exploded_cmd = cmd.split(\"|\")\n            pipe_oper_args = {}\n            for pipe in exploded_cmd[1:]:\n                exploded_pipe = pipe.split()\n                pipe_oper = exploded_pipe[0]  # always there\n                pipe_args = \"\".join(exploded_pipe[1:2])\n                # will not throw error when there's no arg\n                pipe_oper_args[pipe_oper] = pipe_args\n            for oper in _OF_MAP.keys():\n                # to make sure the operation sequence is correct\n                if oper not in pipe_oper_args.keys():\n                    continue\n                txt = _OF_MAP[oper](txt, pipe_oper_args[oper])\n            return txt\n\n        if not isinstance(commands, list):\n            raise TypeError(\"Please enter a valid list of commands!\")\n        _PIPE_BLACKLIST = [\"save\"]\n        # Preprocessing to avoid forbidden commands\n        for command in commands:\n            exploded_cmd = command.split(\"|\")\n            command_safe_parts = []\n            for pipe in exploded_cmd[1:]:\n                exploded_pipe = pipe.split()\n                pipe_oper = exploded_pipe[0]  # always there\n                if pipe_oper in _PIPE_BLACKLIST:\n                    continue\n                pipe_args = \"\".join(exploded_pipe[1:2])\n                safe_pipe = (\n                    pipe_oper\n                    if not pipe_args\n                    else \"{fun} {args}\".format(fun=pipe_oper, args=pipe_args)\n                )\n                command_safe_parts.append(safe_pipe)\n            safe_command = (\n                exploded_cmd[0]\n                if not command_safe_parts\n                else \"{base} | {pipes}\".format(\n                    base=exploded_cmd[0], pipes=\" | \".join(command_safe_parts)\n                )\n            )\n            raw_txt = self.device.cli(safe_command, warning=False)\n            cli_output[py23_compat.text_type(command)] = py23_compat.text_type(\n                _process_pipe(command, raw_txt)\n            )\n        return cli_output"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ndetailing view of the BGP neighbors operational data.", "response": "def get_bgp_neighbors_detail(self, neighbor_address=\"\"):\n        \"\"\"Detailed view of the BGP neighbors operational data.\"\"\"\n        bgp_neighbors = {}\n        default_neighbor_details = {\n            \"up\": False,\n            \"local_as\": 0,\n            \"remote_as\": 0,\n            \"router_id\": \"\",\n            \"local_address\": \"\",\n            \"routing_table\": \"\",\n            \"local_address_configured\": False,\n            \"local_port\": 0,\n            \"remote_address\": \"\",\n            \"remote_port\": 0,\n            \"multihop\": False,\n            \"multipath\": False,\n            \"remove_private_as\": False,\n            \"import_policy\": \"\",\n            \"export_policy\": \"\",\n            \"input_messages\": -1,\n            \"output_messages\": -1,\n            \"input_updates\": -1,\n            \"output_updates\": -1,\n            \"messages_queued_out\": -1,\n            \"connection_state\": \"\",\n            \"previous_connection_state\": \"\",\n            \"last_event\": \"\",\n            \"suppress_4byte_as\": False,\n            \"local_as_prepend\": False,\n            \"holdtime\": 0,\n            \"configured_holdtime\": 0,\n            \"keepalive\": 0,\n            \"configured_keepalive\": 0,\n            \"active_prefix_count\": -1,\n            \"received_prefix_count\": -1,\n            \"accepted_prefix_count\": -1,\n            \"suppressed_prefix_count\": -1,\n            \"advertised_prefix_count\": -1,\n            \"flap_count\": 0,\n        }\n        OPTION_KEY_MAP = {\n            \"RemovePrivateAS\": \"remove_private_as\",\n            \"Multipath\": \"multipath\",\n            \"Multihop\": \"multihop\",\n            \"AddressFamily\": \"local_address_configured\"\n            # 'AuthKey'        : 'authentication_key_set'\n            # but other vendors do not specify if auth key is set\n            # other options:\n            # Preference, HoldTime, Ttl, LogUpDown, Refresh\n        }\n\n        def _bgp_iter_core(neighbor_data, instance=None):\n            \"\"\"\n            Iterate over a list of neighbors.\n            For older junos, the routing instance is not specified inside the\n            BGP neighbors XML, therefore we need to use a super sub-optimal structure\n            as in get_bgp_neighbors: iterate through the list of network instances\n            then execute one request for each and every routing instance.\n            For newer junos, this is not necessary as the routing instance is available\n            and we can get everything solve in a single request.\n            \"\"\"\n            for bgp_neighbor in neighbor_data:\n                remote_as = int(bgp_neighbor[0])\n                neighbor_details = deepcopy(default_neighbor_details)\n                neighbor_details.update(\n                    {\n                        elem[0]: elem[1]\n                        for elem in bgp_neighbor[1]\n                        if elem[1] is not None\n                    }\n                )\n                if not instance:\n                    peer_fwd_rti = neighbor_details.pop(\"peer_fwd_rti\")\n                    instance = peer_fwd_rti\n                else:\n                    peer_fwd_rti = neighbor_details.pop(\"peer_fwd_rti\", \"\")\n                instance_name = \"global\" if instance == \"master\" else instance\n                options = neighbor_details.pop(\"options\", \"\")\n                if isinstance(options, str):\n                    options_list = options.split()\n                    for option in options_list:\n                        key = OPTION_KEY_MAP.get(option)\n                        if key is not None:\n                            neighbor_details[key] = True\n                four_byte_as = neighbor_details.pop(\"4byte_as\", 0)\n                local_address = neighbor_details.pop(\"local_address\", \"\")\n                local_details = local_address.split(\"+\")\n                neighbor_details[\"local_address\"] = napalm.base.helpers.convert(\n                    napalm.base.helpers.ip, local_details[0], local_details[0]\n                )\n                if len(local_details) == 2:\n                    neighbor_details[\"local_port\"] = int(local_details[1])\n                else:\n                    neighbor_details[\"local_port\"] = 179\n                neighbor_details[\"suppress_4byte_as\"] = remote_as != four_byte_as\n                peer_address = neighbor_details.pop(\"peer_address\", \"\")\n                remote_details = peer_address.split(\"+\")\n                neighbor_details[\"remote_address\"] = napalm.base.helpers.convert(\n                    napalm.base.helpers.ip, remote_details[0], remote_details[0]\n                )\n                if len(remote_details) == 2:\n                    neighbor_details[\"remote_port\"] = int(remote_details[1])\n                else:\n                    neighbor_details[\"remote_port\"] = 179\n                neighbor_details[\"routing_table\"] = instance_name\n                neighbor_details[\"local_as\"] = napalm.base.helpers.as_number(\n                    neighbor_details[\"local_as\"]\n                )\n                neighbor_details[\"remote_as\"] = napalm.base.helpers.as_number(\n                    neighbor_details[\"remote_as\"]\n                )\n                neighbors_rib = neighbor_details.pop(\"rib\")\n                neighbors_queue = neighbor_details.pop(\"queue\")\n                messages_queued_out = 0\n                for queue_entry in neighbors_queue.items():\n                    messages_queued_out += queue_entry[1][0][1]\n                neighbor_details[\"messages_queued_out\"] = messages_queued_out\n                if instance_name not in bgp_neighbors.keys():\n                    bgp_neighbors[instance_name] = {}\n                if remote_as not in bgp_neighbors[instance_name].keys():\n                    bgp_neighbors[instance_name][remote_as] = []\n                neighbor_rib_stats = neighbors_rib.items()\n                if not neighbor_rib_stats:\n                    bgp_neighbors[instance_name][remote_as].append(neighbor_details)\n                    continue  # no RIBs available, pass default details\n                neighbor_rib_details = {\n                    \"active_prefix_count\": 0,\n                    \"received_prefix_count\": 0,\n                    \"accepted_prefix_count\": 0,\n                    \"suppressed_prefix_count\": 0,\n                    \"advertised_prefix_count\": 0,\n                }\n                for rib_entry in neighbor_rib_stats:\n                    for elem in rib_entry[1]:\n                        if elem[1] is None:\n                            neighbor_rib_details[elem[0]] += 0\n                        else:\n                            neighbor_rib_details[elem[0]] += elem[1]\n                neighbor_details.update(neighbor_rib_details)\n                bgp_neighbors[instance_name][remote_as].append(neighbor_details)\n\n        # old_junos = napalm.base.helpers.convert(\n        #     int, self.device.facts.get('version', '0.0').split('.')[0], 0) < 15\n        bgp_neighbors_table = junos_views.junos_bgp_neighbors_table(self.device)\n\n        # if old_junos:\n        instances = junos_views.junos_route_instance_table(self.device)\n        for instance, instance_data in instances.get().items():\n            if instance.startswith(\"__\"):\n                # junos internal instances\n                continue\n            neighbor_data = bgp_neighbors_table.get(\n                instance=instance, neighbor_address=str(neighbor_address)\n            ).items()\n            _bgp_iter_core(neighbor_data, instance=instance)\n        # else:\n        #     bgp_neighbors_table = junos_views.junos_bgp_neighbors_table(self.device)\n        #     neighbor_data = bgp_neighbors_table.get(neighbor_address=neighbor_address).items()\n        #     _bgp_iter_core(neighbor_data)\n        return bgp_neighbors"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_arp_table(self, vrf=\"\"):\n        # could use ArpTable\n        # from jnpr.junos.op.phyport import ArpTable\n        # and simply use it\n        # but\n        # we need:\n        #   - filters\n        #   - group by VLAN ID\n        #   - hostname & TTE fields as well\n        if vrf:\n            msg = \"VRF support has not been added for this getter on this platform.\"\n            raise NotImplementedError(msg)\n\n        arp_table = []\n\n        arp_table_raw = junos_views.junos_arp_table(self.device)\n        arp_table_raw.get()\n        arp_table_items = arp_table_raw.items()\n\n        for arp_table_entry in arp_table_items:\n            arp_entry = {elem[0]: elem[1] for elem in arp_table_entry[1]}\n            arp_entry[\"mac\"] = napalm.base.helpers.mac(arp_entry.get(\"mac\"))\n            arp_entry[\"ip\"] = napalm.base.helpers.ip(arp_entry.get(\"ip\"))\n            arp_table.append(arp_entry)\n\n        return arp_table", "response": "Return the ARP table."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns the IPv6 neighbors table.", "response": "def get_ipv6_neighbors_table(self):\n        \"\"\"Return the IPv6 neighbors table.\"\"\"\n        ipv6_neighbors_table = []\n\n        ipv6_neighbors_table_raw = junos_views.junos_ipv6_neighbors_table(self.device)\n        ipv6_neighbors_table_raw.get()\n        ipv6_neighbors_table_items = ipv6_neighbors_table_raw.items()\n\n        for ipv6_table_entry in ipv6_neighbors_table_items:\n            ipv6_entry = {elem[0]: elem[1] for elem in ipv6_table_entry[1]}\n            ipv6_entry[\"mac\"] = napalm.base.helpers.mac(ipv6_entry.get(\"mac\"))\n            ipv6_entry[\"ip\"] = napalm.base.helpers.ip(ipv6_entry.get(\"ip\"))\n            ipv6_neighbors_table.append(ipv6_entry)\n\n        return ipv6_neighbors_table"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns NTP stats (associations).", "response": "def get_ntp_stats(self):\n        \"\"\"Return NTP stats (associations).\"\"\"\n        # NTP Peers does not have XML RPC defined\n        # thus we need to retrieve raw text and parse...\n        # :(\n\n        ntp_stats = []\n\n        REGEX = (\n            r\"^\\s?(\\+|\\*|x|-)?([a-zA-Z0-9\\.+-:]+)\"\n            r\"\\s+([a-zA-Z0-9\\.]+)\\s+([0-9]{1,2})\"\n            r\"\\s+(-|u)\\s+([0-9h-]+)\\s+([0-9]+)\"\n            r\"\\s+([0-9]+)\\s+([0-9\\.]+)\\s+([0-9\\.-]+)\"\n            r\"\\s+([0-9\\.]+)\\s?$\"\n        )\n\n        ntp_assoc_output = self.device.cli(\"show ntp associations no-resolve\")\n        ntp_assoc_output_lines = ntp_assoc_output.splitlines()\n\n        for ntp_assoc_output_line in ntp_assoc_output_lines[3:]:  # except last line\n            line_search = re.search(REGEX, ntp_assoc_output_line, re.I)\n            if not line_search:\n                continue  # pattern not found\n            line_groups = line_search.groups()\n            try:\n                ntp_stats.append(\n                    {\n                        \"remote\": napalm.base.helpers.ip(line_groups[1]),\n                        \"synchronized\": (line_groups[0] == \"*\"),\n                        \"referenceid\": py23_compat.text_type(line_groups[2]),\n                        \"stratum\": int(line_groups[3]),\n                        \"type\": py23_compat.text_type(line_groups[4]),\n                        \"when\": py23_compat.text_type(line_groups[5]),\n                        \"hostpoll\": int(line_groups[6]),\n                        \"reachability\": int(line_groups[7]),\n                        \"delay\": float(line_groups[8]),\n                        \"offset\": float(line_groups[9]),\n                        \"jitter\": float(line_groups[10]),\n                    }\n                )\n            except Exception:\n                continue  # jump to next line\n\n        return ntp_stats"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_interfaces_ip(self):\n        interfaces_ip = {}\n\n        interface_table = junos_views.junos_ip_interfaces_table(self.device)\n        interface_table.get()\n        interface_table_items = interface_table.items()\n\n        _FAMILY_VMAP_ = {\n            \"inet\": \"ipv4\",\n            \"inet6\": \"ipv6\"\n            # can add more mappings\n        }\n        _FAMILY_MAX_PREFIXLEN = {\"inet\": 32, \"inet6\": 128}\n\n        for interface_details in interface_table_items:\n            ip_network = interface_details[0]\n            ip_address = ip_network.split(\"/\")[0]\n            address = napalm.base.helpers.convert(\n                napalm.base.helpers.ip, ip_address, ip_address\n            )\n            try:\n                interface_details_dict = dict(interface_details[1])\n                family_raw = interface_details_dict.get(\"family\")\n                interface = py23_compat.text_type(\n                    interface_details_dict.get(\"interface\")\n                )\n            except ValueError:\n                continue\n            prefix = napalm.base.helpers.convert(\n                int, ip_network.split(\"/\")[-1], _FAMILY_MAX_PREFIXLEN.get(family_raw)\n            )\n            family = _FAMILY_VMAP_.get(family_raw)\n            if not family or not interface:\n                continue\n            if interface not in interfaces_ip.keys():\n                interfaces_ip[interface] = {}\n            if family not in interfaces_ip[interface].keys():\n                interfaces_ip[interface][family] = {}\n            if address not in interfaces_ip[interface][family].keys():\n                interfaces_ip[interface][family][address] = {}\n            interfaces_ip[interface][family][address][\"prefix_length\"] = prefix\n\n        return interfaces_ip", "response": "Return the configured IP addresses."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning the MAC address table.", "response": "def get_mac_address_table(self):\n        \"\"\"Return the MAC address table.\"\"\"\n        mac_address_table = []\n\n        switch_style = self.device.facts.get(\"switch_style\", \"\")\n        if switch_style == \"VLAN_L2NG\":\n            mac_table = junos_views.junos_mac_address_table_switch_l2ng(self.device)\n        elif switch_style == \"BRIDGE_DOMAIN\":\n            mac_table = junos_views.junos_mac_address_table(self.device)\n        else:  # switch_style == \"VLAN\"\n            mac_table = junos_views.junos_mac_address_table_switch(self.device)\n\n        try:\n            mac_table.get()\n        except RpcError as e:\n            # Device hasn't got it's l2 subsystem running\n            # Don't error but just return an empty result\n            if \"l2-learning subsystem\" in e.message:\n                return []\n            else:\n                raise\n\n        mac_table_items = mac_table.items()\n\n        default_values = {\n            \"mac\": \"\",\n            \"interface\": \"\",\n            \"vlan\": 0,\n            \"static\": False,\n            \"active\": True,\n            \"moves\": 0,\n            \"last_move\": 0.0,\n        }\n\n        for mac_table_entry in mac_table_items:\n            mac_entry = default_values.copy()\n            mac_entry.update({elem[0]: elem[1] for elem in mac_table_entry[1]})\n            mac = mac_entry.get(\"mac\")\n\n            # JUNOS returns '*' for Type = Flood\n            if mac == \"*\":\n                continue\n\n            mac_entry[\"mac\"] = napalm.base.helpers.mac(mac)\n            mac_address_table.append(mac_entry)\n\n        return mac_address_table"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ngets route details to a specific destination learned from a certain protocol.", "response": "def get_route_to(self, destination=\"\", protocol=\"\"):\n        \"\"\"Return route details to a specific destination, learned from a certain protocol.\"\"\"\n        routes = {}\n\n        if not isinstance(destination, py23_compat.string_types):\n            raise TypeError(\"Please specify a valid destination!\")\n\n        if protocol and isinstance(destination, py23_compat.string_types):\n            protocol = protocol.lower()\n\n        if protocol == \"connected\":\n            protocol = \"direct\"  # this is how is called on JunOS\n\n        _COMMON_PROTOCOL_FIELDS_ = [\n            \"destination\",\n            \"prefix_length\",\n            \"protocol\",\n            \"current_active\",\n            \"last_active\",\n            \"age\",\n            \"next_hop\",\n            \"outgoing_interface\",\n            \"selected_next_hop\",\n            \"preference\",\n            \"inactive_reason\",\n            \"routing_table\",\n        ]  # identifies the list of fileds common for all protocols\n\n        _BOOLEAN_FIELDS_ = [\n            \"current_active\",\n            \"selected_next_hop\",\n            \"last_active\",\n        ]  # fields expected to have boolean values\n\n        _PROTOCOL_SPECIFIC_FIELDS_ = {\n            \"bgp\": [\n                \"local_as\",\n                \"remote_as\",\n                \"as_path\",\n                \"communities\",\n                \"local_preference\",\n                \"preference2\",\n                \"remote_address\",\n                \"metric\",\n                \"metric2\",\n            ],\n            \"isis\": [\"level\", \"metric\", \"local_as\"],\n        }\n\n        routes_table = junos_views.junos_protocol_route_table(self.device)\n\n        rt_kargs = {\"destination\": destination}\n        if protocol and isinstance(destination, py23_compat.string_types):\n            rt_kargs[\"protocol\"] = protocol\n\n        try:\n            routes_table.get(**rt_kargs)\n        except RpcTimeoutError:\n            # on devices with milions of routes\n            # in case the destination is too generic (e.g.: 10/8)\n            # will take very very long to determine all routes and\n            # moreover will return a huge list\n            raise CommandTimeoutException(\n                \"Too many routes returned! Please try with a longer prefix or a specific protocol!\"\n            )\n        except RpcError as rpce:\n            if len(rpce.errs) > 0 and \"bad_element\" in rpce.errs[0]:\n                raise CommandErrorException(\n                    \"Unknown protocol: {proto}\".format(\n                        proto=rpce.errs[0][\"bad_element\"]\n                    )\n                )\n            raise CommandErrorException(rpce)\n        except Exception as err:\n            raise CommandErrorException(\n                \"Cannot retrieve routes! Reason: {err}\".format(err=err)\n            )\n\n        routes_items = routes_table.items()\n\n        for route in routes_items:\n            d = {}\n            # next_hop = route[0]\n            d = {elem[0]: elem[1] for elem in route[1]}\n            destination = d.pop(\"destination\", \"\")\n            prefix_length = d.pop(\"prefix_length\", 32)\n            destination = \"{d}/{p}\".format(d=destination, p=prefix_length)\n            d.update({key: False for key in _BOOLEAN_FIELDS_ if d.get(key) is None})\n            as_path = d.get(\"as_path\")\n            if as_path is not None:\n                d[\"as_path\"] = (\n                    as_path.split(\" I \")[0]\n                    .replace(\"AS path:\", \"\")\n                    .replace(\"I\", \"\")\n                    .strip()\n                )\n                # to be sure that contains only AS Numbers\n            if d.get(\"inactive_reason\") is None:\n                d[\"inactive_reason\"] = \"\"\n            route_protocol = d.get(\"protocol\").lower()\n            if protocol and protocol != route_protocol:\n                continue\n            communities = d.get(\"communities\")\n            if communities is not None and type(communities) is not list:\n                d[\"communities\"] = [communities]\n            d_keys = list(d.keys())\n            # fields that are not in _COMMON_PROTOCOL_FIELDS_ are supposed to be protocol specific\n            all_protocol_attributes = {\n                key: d.pop(key) for key in d_keys if key not in _COMMON_PROTOCOL_FIELDS_\n            }\n            protocol_attributes = {\n                key: value\n                for key, value in all_protocol_attributes.items()\n                if key in _PROTOCOL_SPECIFIC_FIELDS_.get(route_protocol, [])\n            }\n            d[\"protocol_attributes\"] = protocol_attributes\n            if destination not in routes.keys():\n                routes[destination] = []\n            routes[destination].append(d)\n\n        return routes"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns the SNMP configuration.", "response": "def get_snmp_information(self):\n        \"\"\"Return the SNMP configuration.\"\"\"\n        snmp_information = {}\n\n        snmp_config = junos_views.junos_snmp_config_table(self.device)\n        snmp_config.get()\n        snmp_items = snmp_config.items()\n\n        if not snmp_items:\n            return snmp_information\n\n        snmp_information = {\n            py23_compat.text_type(ele[0]): ele[1] if ele[1] else \"\"\n            for ele in snmp_items[0][1]\n        }\n\n        snmp_information[\"community\"] = {}\n        communities_table = snmp_information.pop(\"communities_table\")\n        if not communities_table:\n            return snmp_information\n\n        for community in communities_table.items():\n            community_name = py23_compat.text_type(community[0])\n            community_details = {\"acl\": \"\"}\n            community_details.update(\n                {\n                    py23_compat.text_type(ele[0]): py23_compat.text_type(\n                        ele[1]\n                        if ele[0] != \"mode\"\n                        else C.SNMP_AUTHORIZATION_MODE_MAP.get(ele[1])\n                    )\n                    for ele in community[1]\n                }\n            )\n            snmp_information[\"community\"][community_name] = community_details\n\n        return snmp_information"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _get_root(self):\n        _DEFAULT_USER_DETAILS = {\"level\": 20, \"password\": \"\", \"sshkeys\": []}\n        root = {}\n        root_table = junos_views.junos_root_table(self.device)\n        root_table.get()\n        root_items = root_table.items()\n        for user_entry in root_items:\n            username = \"root\"\n            user_details = _DEFAULT_USER_DETAILS.copy()\n            user_details.update({d[0]: d[1] for d in user_entry[1] if d[1]})\n            user_details = {\n                key: py23_compat.text_type(user_details[key])\n                for key in user_details.keys()\n            }\n            user_details[\"level\"] = int(user_details[\"level\"])\n            user_details[\"sshkeys\"] = [\n                user_details.pop(key)\n                for key in [\"ssh_rsa\", \"ssh_dsa\", \"ssh_ecdsa\"]\n                if user_details.get(key, \"\")\n            ]\n            root[username] = user_details\n        return root", "response": "get root user password"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_users(self):\n        users = {}\n\n        _JUNOS_CLASS_CISCO_PRIVILEGE_LEVEL_MAP = {\n            \"super-user\": 15,\n            \"superuser\": 15,\n            \"operator\": 5,\n            \"read-only\": 1,\n            \"unauthorized\": 0,\n        }\n\n        _DEFAULT_USER_DETAILS = {\"level\": 0, \"password\": \"\", \"sshkeys\": []}\n\n        users_table = junos_views.junos_users_table(self.device)\n        users_table.get()\n        users_items = users_table.items()\n        root_user = self._get_root()\n\n        for user_entry in users_items:\n            username = user_entry[0]\n            user_details = _DEFAULT_USER_DETAILS.copy()\n            user_details.update({d[0]: d[1] for d in user_entry[1] if d[1]})\n            user_class = user_details.pop(\"class\", \"\")\n            user_details = {\n                key: py23_compat.text_type(user_details[key])\n                for key in user_details.keys()\n            }\n            level = _JUNOS_CLASS_CISCO_PRIVILEGE_LEVEL_MAP.get(user_class, 0)\n            user_details.update({\"level\": level})\n            user_details[\"sshkeys\"] = [\n                user_details.pop(key)\n                for key in [\"ssh_rsa\", \"ssh_dsa\", \"ssh_ecdsa\"]\n                if user_details.get(key, \"\")\n            ]\n            users[username] = user_details\n        users.update(root_user)\n        return users", "response": "Return the configuration of the users."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef cisco_conf_parse_parents(parent, child, config):\n    if type(config) == str:\n        config = config.splitlines()\n    parse = CiscoConfParse(config)\n    cfg_obj = parse.find_parents_w_child(parent, child)\n    return cfg_obj", "response": "Use CiscoConfParse to find parents that contain a specific child line."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef cisco_conf_parse_objects(cfg_section, config):\n    return_config = []\n    if type(config) is str:\n        config = config.splitlines()\n    parse = CiscoConfParse(config)\n    cfg_obj = parse.find_objects(cfg_section)\n    for parent in cfg_obj:\n        return_config.append(parent.text)\n        for child in parent.all_children:\n            return_config.append(child.text)\n    return return_config", "response": "Use CiscoConfParse to find and return a section of Cisco IOS config."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef regex_find_txt(pattern, text, default=\"\"):\n    text = str(text)\n    value = re.findall(pattern, text)\n    try:\n        if not value:\n            raise Exception\n        if not isinstance(value, type(default)):\n            if isinstance(value, list) and len(value) == 1:\n                value = value[0]\n            value = type(default)(value)\n    except Exception:  # in case of any exception, returns default\n        value = default\n    return value", "response": "RegEx search for pattern in text and return the value of the default value."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\napply a TextFSM template over a raw text and return the matching table. Main usage of this method will be to extract data form a non-structured output from a network device and return the values in a table format. :param cls: Instance of the driver class :param template_name: Specifies the name of the template to be used :param raw_text: Text output as the devices prompts on the CLI :return: table-like list of entries", "response": "def textfsm_extractor(cls, template_name, raw_text):\n    \"\"\"\n    Applies a TextFSM template over a raw text and return the matching table.\n\n    Main usage of this method will be to extract data form a non-structured output\n    from a network device and return the values in a table format.\n\n    :param cls: Instance of the driver class\n    :param template_name: Specifies the name of the template to be used\n    :param raw_text: Text output as the devices prompts on the CLI\n    :return: table-like list of entries\n    \"\"\"\n    textfsm_data = list()\n    fsm_handler = None\n    for c in cls.__class__.mro():\n        if c is object:\n            continue\n        current_dir = os.path.dirname(\n            os.path.abspath(sys.modules[c.__module__].__file__)\n        )\n        template_dir_path = \"{current_dir}/utils/textfsm_templates\".format(\n            current_dir=current_dir\n        )\n        template_path = \"{template_dir_path}/{template_name}.tpl\".format(\n            template_dir_path=template_dir_path, template_name=template_name\n        )\n\n        try:\n            with open(template_path) as f:\n                fsm_handler = textfsm.TextFSM(f)\n\n                for obj in fsm_handler.ParseText(raw_text):\n                    entry = {}\n                    for index, entry_value in enumerate(obj):\n                        entry[fsm_handler.header[index].lower()] = entry_value\n                    textfsm_data.append(entry)\n\n                return textfsm_data\n        except IOError:  # Template not present in this class\n            continue  # Continue up the MRO\n        except textfsm.TextFSMTemplateError as tfte:\n            raise napalm.base.exceptions.TemplateRenderException(\n                \"Wrong format of TextFSM template {template_name}: {error}\".format(\n                    template_name=template_name, error=py23_compat.text_type(tfte)\n                )\n            )\n\n    raise napalm.base.exceptions.TemplateNotImplemented(\n        \"TextFSM template {template_name}.tpl is not defined under {path}\".format(\n            template_name=template_name, path=template_dir_path\n        )\n    )"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nconvert a raw string to a valid IP address.", "response": "def ip(addr, version=None):\n    \"\"\"\n    Converts a raw string to a valid IP address. Optional version argument will detect that \\\n    object matches specified version.\n\n    Motivation: the groups of the IP addreses may contain leading zeros. IPv6 addresses can \\\n    contain sometimes uppercase characters. E.g.: 2001:0dB8:85a3:0000:0000:8A2e:0370:7334 has \\\n    the same logical value as 2001:db8:85a3::8a2e:370:7334. However, their values as strings are \\\n    not the same.\n\n    :param raw: the raw string containing the value of the IP Address\n    :param version: (optional) insist on a specific IP address version.\n    :type version: int.\n    :return: a string containing the IP Address in a standard format (no leading zeros, \\\n    zeros-grouping, lowercase)\n\n    Example:\n\n    .. code-block:: python\n\n        >>> ip('2001:0dB8:85a3:0000:0000:8A2e:0370:7334')\n        u'2001:db8:85a3::8a2e:370:7334'\n    \"\"\"\n    addr_obj = IPAddress(addr)\n    if version and addr_obj.version != version:\n        raise ValueError(\"{} is not an ipv{} address\".format(addr, version))\n    return py23_compat.text_type(addr_obj)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef as_number(as_number_val):\n    as_number_str = py23_compat.text_type(as_number_val)\n    if \".\" in as_number_str:\n        big, little = as_number_str.split(\".\")\n        return (int(big) << 16) + int(little)\n    else:\n        return int(as_number_str)", "response": "Convert AS Number to standardized asplain notation as an integer."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef split_interface(intf_name):\n    head = intf_name.rstrip(r\"/\\0123456789. \")\n    tail = intf_name[len(head) :].lstrip()\n    return (head, tail)", "response": "Split an interface name based on first digit slash or space match."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef canonical_interface_name(interface, addl_name_map=None):\n\n    name_map = {}\n    name_map.update(base_interfaces)\n    interface_type, interface_number = split_interface(interface)\n\n    if isinstance(addl_name_map, dict):\n        name_map.update(addl_name_map)\n    # check in dict for mapping\n    if name_map.get(interface_type):\n        long_int = name_map.get(interface_type)\n        return long_int + py23_compat.text_type(interface_number)\n    # if nothing matched, return the original name\n    else:\n        return interface", "response": "Function to return an interface s canonical name."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef abbreviated_interface_name(interface, addl_name_map=None, addl_reverse_map=None):\n\n    name_map = {}\n    name_map.update(base_interfaces)\n    interface_type, interface_number = split_interface(interface)\n\n    if isinstance(addl_name_map, dict):\n        name_map.update(addl_name_map)\n\n    rev_name_map = {}\n    rev_name_map.update(reverse_mapping)\n\n    if isinstance(addl_reverse_map, dict):\n        rev_name_map.update(addl_reverse_map)\n\n    # Try to ensure canonical type.\n    if name_map.get(interface_type):\n        canonical_type = name_map.get(interface_type)\n    else:\n        canonical_type = interface_type\n\n    try:\n        abbreviated_name = rev_name_map[canonical_type] + py23_compat.text_type(\n            interface_number\n        )\n        return abbreviated_name\n    except KeyError:\n        pass\n\n    # If abbreviated name lookup fails, return original name\n    return interface", "response": "Function to return an abbreviated representation of the interface name."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ncloses the connection to the device and do the necessary cleanup.", "response": "def close(self):\n        \"\"\"Close the connection to the device and do the necessary cleanup.\"\"\"\n\n        # Return file prompt quiet to the original state\n        if self.auto_file_prompt and self.prompt_quiet_changed is True:\n            self.device.send_config_set([\"no file prompt quiet\"])\n            self.prompt_quiet_changed = False\n            self.prompt_quiet_configured = False\n        self._netmiko_close()"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _send_command(self, command):\n        try:\n            if isinstance(command, list):\n                for cmd in command:\n                    output = self.device.send_command(cmd)\n                    if \"% Invalid\" not in output:\n                        break\n            else:\n                output = self.device.send_command(command)\n            return self._send_command_postprocess(output)\n        except (socket.error, EOFError) as e:\n            raise ConnectionClosedException(str(e))", "response": "Wrapper for self. device. send. command."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn a flag with the state of the connection.", "response": "def is_alive(self):\n        \"\"\"Returns a flag with the state of the connection.\"\"\"\n        null = chr(0)\n        if self.device is None:\n            return {\"is_alive\": False}\n        if self.transport == \"telnet\":\n            try:\n                # Try sending IAC + NOP (IAC is telnet way of sending command\n                # IAC = Interpret as Command (it comes before the NOP)\n                self.device.write_channel(telnetlib.IAC + telnetlib.NOP)\n                return {\"is_alive\": True}\n            except UnicodeDecodeError:\n                # Netmiko logging bug (remove after Netmiko >= 1.4.3)\n                return {\"is_alive\": True}\n            except AttributeError:\n                return {\"is_alive\": False}\n        else:\n            # SSH\n            try:\n                # Try sending ASCII null byte to maintain the connection alive\n                self.device.write_channel(null)\n                return {\"is_alive\": self.device.remote_conn.transport.is_active()}\n            except (socket.error, EOFError):\n                # If unable to send, we can tell for sure that the connection is unusable\n                return {\"is_alive\": False}\n        return {\"is_alive\": False}"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _normalize_compare_config(self, diff):\n        ignore_strings = [\n            \"Contextual Config Diffs\",\n            \"No changes were found\",\n            \"ntp clock-period\",\n        ]\n        if self.auto_file_prompt:\n            ignore_strings.append(\"file prompt quiet\")\n\n        new_list = []\n        for line in diff.splitlines():\n            for ignore in ignore_strings:\n                if ignore in line:\n                    break\n            else:  # nobreak\n                new_list.append(line)\n        return \"\\n\".join(new_list)", "response": "Filter out strings that should not show up in the diff."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _normalize_merge_diff_incr(diff):\n        new_diff = []\n\n        changes_found = False\n        for line in diff.splitlines():\n            if re.search(r\"order-dependent line.*re-ordered\", line):\n                changes_found = True\n            elif \"No changes were found\" in line:\n                # IOS in the re-order case still claims \"No changes were found\"\n                if not changes_found:\n                    return \"\"\n                else:\n                    continue\n\n            if line.strip() == \"end\":\n                continue\n            elif \"List of Commands\" in line:\n                continue\n            # Filter blank lines and prepend +sign\n            elif line.strip():\n                if re.search(r\"^no\\s+\", line.strip()):\n                    new_diff.append(\"-\" + line)\n                else:\n                    new_diff.append(\"+\" + line)\n        return \"\\n\".join(new_diff)", "response": "Make the compare config output look better."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _normalize_merge_diff(diff):\n        new_diff = []\n        for line in diff.splitlines():\n            # Filter blank lines and prepend +sign\n            if line.strip():\n                new_diff.append(\"+\" + line)\n        if new_diff:\n            new_diff.insert(\n                0, \"! incremental-diff failed; falling back to echo of merge file\"\n            )\n        else:\n            new_diff.append(\"! No changes specified in merge file.\")\n        return \"\\n\".join(new_diff)", "response": "Make compare_config() for merge look similar to replace config diff."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncomparing the archive config with the current configuration.", "response": "def compare_config(self):\n        \"\"\"\n        show archive config differences <base_file> <new_file>.\n\n        Default operation is to compare system:running-config to self.candidate_cfg\n        \"\"\"\n        # Set defaults\n        base_file = \"running-config\"\n        base_file_system = \"system:\"\n        if self.config_replace:\n            new_file = self.candidate_cfg\n        else:\n            new_file = self.merge_cfg\n        new_file_system = self.dest_file_system\n\n        base_file_full = self._gen_full_path(\n            filename=base_file, file_system=base_file_system\n        )\n        new_file_full = self._gen_full_path(\n            filename=new_file, file_system=new_file_system\n        )\n\n        if self.config_replace:\n            cmd = \"show archive config differences {} {}\".format(\n                base_file_full, new_file_full\n            )\n            diff = self.device.send_command_expect(cmd)\n            diff = self._normalize_compare_config(diff)\n        else:\n            # merge\n            cmd = \"show archive config incremental-diffs {} ignorecase\".format(\n                new_file_full\n            )\n            diff = self.device.send_command_expect(cmd)\n            if \"error code 5\" in diff or \"returned error 5\" in diff:\n                diff = (\n                    \"You have encountered the obscure 'error 5' message. This generally \"\n                    \"means you need to add an 'end' statement to the end of your merge changes.\"\n                )\n            elif \"% Invalid\" not in diff:\n                diff = self._normalize_merge_diff_incr(diff)\n            else:\n                cmd = \"more {}\".format(new_file_full)\n                diff = self.device.send_command_expect(cmd)\n                diff = self._normalize_merge_diff(diff)\n\n        return diff.strip()"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _commit_handler(self, cmd):\n        current_prompt = self.device.find_prompt().strip()\n        terminating_char = current_prompt[-1]\n        # Look for trailing pattern that includes '#' and '>'\n        pattern1 = r\"[>#{}]\\s*$\".format(terminating_char)\n        # Handle special username removal pattern\n        pattern2 = r\".*all username.*confirm\"\n        patterns = r\"(?:{}|{})\".format(pattern1, pattern2)\n        output = self.device.send_command_expect(cmd, expect_string=patterns)\n        loop_count = 50\n        new_output = output\n        for i in range(loop_count):\n            if re.search(pattern2, new_output):\n                # Send confirmation if username removal\n                new_output = self.device.send_command_timing(\n                    \"\\n\", strip_prompt=False, strip_command=False\n                )\n                output += new_output\n            else:\n                break\n        # Reset base prompt in case hostname changed\n        self.device.set_base_prompt()\n        return output", "response": "Handles hostname change on commit operation."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef commit_config(self, message=\"\"):\n        if message:\n            raise NotImplementedError(\n                \"Commit message not implemented for this platform\"\n            )\n        # Always generate a rollback config on commit\n        self._gen_rollback_cfg()\n\n        if self.config_replace:\n            # Replace operation\n            filename = self.candidate_cfg\n            cfg_file = self._gen_full_path(filename)\n            if not self._check_file_exists(cfg_file):\n                raise ReplaceConfigException(\"Candidate config file does not exist\")\n            if self.auto_rollback_on_error:\n                cmd = \"configure replace {} force revert trigger error\".format(cfg_file)\n            else:\n                cmd = \"configure replace {} force\".format(cfg_file)\n            output = self._commit_handler(cmd)\n            if (\n                (\"original configuration has been successfully restored\" in output)\n                or (\"error\" in output.lower())\n                or (\"not a valid config file\" in output.lower())\n                or (\"failed\" in output.lower())\n            ):\n                msg = \"Candidate config could not be applied\\n{}\".format(output)\n                raise ReplaceConfigException(msg)\n            elif \"%Please turn config archive on\" in output:\n                msg = \"napalm-ios replace() requires Cisco 'archive' feature to be enabled.\"\n                raise ReplaceConfigException(msg)\n        else:\n            # Merge operation\n            filename = self.merge_cfg\n            cfg_file = self._gen_full_path(filename)\n            if not self._check_file_exists(cfg_file):\n                raise MergeConfigException(\"Merge source config file does not exist\")\n            cmd = \"copy {} running-config\".format(cfg_file)\n            output = self._commit_handler(cmd)\n            if \"Invalid input detected\" in output:\n                self.rollback()\n                err_header = \"Configuration merge failed; automatic rollback attempted\"\n                merge_error = \"{0}:\\n{1}\".format(err_header, output)\n                raise MergeConfigException(merge_error)\n\n        # After a commit - we no longer know whether this is configured or not.\n        self.prompt_quiet_configured = None\n\n        # Save config to startup (both replace and merge)\n        output += self.device.save_config()", "response": "Perform configuration changes on the current running - config."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _discard_config(self):\n        discard_candidate = \"copy running-config {}\".format(\n            self._gen_full_path(self.candidate_cfg)\n        )\n        discard_merge = \"copy null: {}\".format(self._gen_full_path(self.merge_cfg))\n        self.device.send_command_expect(discard_candidate)\n        self.device.send_command_expect(discard_merge)", "response": "Set candidate_cfg to current running - config. Erase the merge_cfg file."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef rollback(self):\n        filename = self.rollback_cfg\n        cfg_file = self._gen_full_path(filename)\n        if not self._check_file_exists(cfg_file):\n            raise ReplaceConfigException(\"Rollback config file does not exist\")\n        cmd = \"configure replace {} force\".format(cfg_file)\n        self.device.send_command_expect(cmd)\n\n        # After a rollback - we no longer know whether this is configured or not.\n        self.prompt_quiet_configured = None\n\n        # Save config to startup\n        self.device.save_config()", "response": "Rollback configuration to filename or to self. rollback_cfg file."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _inline_tcl_xfer(\n        self, source_file=None, source_config=None, dest_file=None, file_system=None\n    ):\n        \"\"\"\n        Use Netmiko InlineFileTransfer (TCL) to transfer file or config to remote device.\n\n        Return (status, msg)\n        status = boolean\n        msg = details on what happened\n        \"\"\"\n        if source_file:\n            return self._xfer_file(\n                source_file=source_file,\n                dest_file=dest_file,\n                file_system=file_system,\n                TransferClass=InLineTransfer,\n            )\n        if source_config:\n            return self._xfer_file(\n                source_config=source_config,\n                dest_file=dest_file,\n                file_system=file_system,\n                TransferClass=InLineTransfer,\n            )\n        raise ValueError(\"File source not specified for transfer.\")", "response": "Use Netmiko InlineFileTransfer (TCL) to transfer file or config to remote device.\n\n        Return (status, msg)\n        status = boolean\n        msg = details on what happened"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _scp_file(self, source_file, dest_file, file_system):\n        return self._xfer_file(\n            source_file=source_file,\n            dest_file=dest_file,\n            file_system=file_system,\n            TransferClass=FileTransfer,\n        )", "response": "SCP file to remote device."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ntransferring file to remote device.", "response": "def _xfer_file(\n        self,\n        source_file=None,\n        source_config=None,\n        dest_file=None,\n        file_system=None,\n        TransferClass=FileTransfer,\n    ):\n        \"\"\"Transfer file to remote device.\n\n        By default, this will use Secure Copy if self.inline_transfer is set, then will use\n        Netmiko InlineTransfer method to transfer inline using either SSH or telnet (plus TCL\n        onbox).\n\n        Return (status, msg)\n        status = boolean\n        msg = details on what happened\n        \"\"\"\n        if not source_file and not source_config:\n            raise ValueError(\"File source not specified for transfer.\")\n        if not dest_file or not file_system:\n            raise ValueError(\"Destination file or file system not specified.\")\n\n        if source_file:\n            kwargs = dict(\n                ssh_conn=self.device,\n                source_file=source_file,\n                dest_file=dest_file,\n                direction=\"put\",\n                file_system=file_system,\n            )\n        elif source_config:\n            kwargs = dict(\n                ssh_conn=self.device,\n                source_config=source_config,\n                dest_file=dest_file,\n                direction=\"put\",\n                file_system=file_system,\n            )\n        use_scp = True\n        if self.inline_transfer:\n            use_scp = False\n\n        with TransferClass(**kwargs) as transfer:\n\n            # Check if file already exists and has correct MD5\n            if transfer.check_file_exists() and transfer.compare_md5():\n                msg = \"File already exists and has correct MD5: no SCP needed\"\n                return (True, msg)\n            if not transfer.verify_space_available():\n                msg = \"Insufficient space available on remote device\"\n                return (False, msg)\n\n            if use_scp:\n                cmd = \"ip scp server enable\"\n                show_cmd = \"show running-config | inc {}\".format(cmd)\n                output = self.device.send_command_expect(show_cmd)\n                if cmd not in output:\n                    msg = (\n                        \"SCP file transfers are not enabled. \"\n                        \"Configure 'ip scp server enable' on the device.\"\n                    )\n                    raise CommandErrorException(msg)\n\n            # Transfer file\n            transfer.transfer_file()\n\n            # Compares MD5 between local-remote files\n            if transfer.verify_file():\n                msg = \"File successfully transferred to remote device\"\n                return (True, msg)\n            else:\n                msg = \"File transfer to remote device failed\"\n                return (False, msg)\n            return (False, \"\")"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _send_command_postprocess(output):\n        output = re.sub(r\"^Load for five secs.*$\", \"\", output, flags=re.M)\n        output = re.sub(r\"^Time source is .*$\", \"\", output, flags=re.M)\n        return output.strip()", "response": "Cleanup actions on send_command for NAPALM getters."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_interfaces_ip(self):\n        interfaces = {}\n\n        command = \"show ip interface\"\n        show_ip_interface = self._send_command(command)\n        command = \"show ipv6 interface\"\n        show_ipv6_interface = self._send_command(command)\n\n        INTERNET_ADDRESS = r\"\\s+(?:Internet address is|Secondary address)\"\n        INTERNET_ADDRESS += r\" (?P<ip>{})/(?P<prefix>\\d+)\".format(IPV4_ADDR_REGEX)\n        LINK_LOCAL_ADDRESS = (\n            r\"\\s+IPv6 is enabled, link-local address is (?P<ip>[a-fA-F0-9:]+)\"\n        )\n        GLOBAL_ADDRESS = (\n            r\"\\s+(?P<ip>[a-fA-F0-9:]+), subnet is (?:[a-fA-F0-9:]+)/(?P<prefix>\\d+)\"\n        )\n\n        interfaces = {}\n        for line in show_ip_interface.splitlines():\n            if len(line.strip()) == 0:\n                continue\n            if line[0] != \" \":\n                ipv4 = {}\n                interface_name = line.split()[0]\n            m = re.match(INTERNET_ADDRESS, line)\n            if m:\n                ip, prefix = m.groups()\n                ipv4.update({ip: {\"prefix_length\": int(prefix)}})\n                interfaces[interface_name] = {\"ipv4\": ipv4}\n\n        if \"% Invalid input detected at\" not in show_ipv6_interface:\n            for line in show_ipv6_interface.splitlines():\n                if len(line.strip()) == 0:\n                    continue\n                if line[0] != \" \":\n                    ifname = line.split()[0]\n                    ipv6 = {}\n                    if ifname not in interfaces:\n                        interfaces[ifname] = {\"ipv6\": ipv6}\n                    else:\n                        interfaces[ifname].update({\"ipv6\": ipv6})\n                m = re.match(LINK_LOCAL_ADDRESS, line)\n                if m:\n                    ip = m.group(1)\n                    ipv6.update({ip: {\"prefix_length\": 10}})\n                m = re.match(GLOBAL_ADDRESS, line)\n                if m:\n                    ip, prefix = m.groups()\n                    ipv6.update({ip: {\"prefix_length\": int(prefix)}})\n\n        # Interface without ipv6 doesn't appears in show ipv6 interface\n        return interfaces", "response": "Get interface ip details."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nparse the BGP config and return a dict of all the information that can be used to configure the BGP routing table.", "response": "def get_bgp_config(self, group=\"\", neighbor=\"\"):\n        \"\"\"\n        Parse BGP config params into a dict\n            :param group='':\n            :param neighbor='':\n        \"\"\"\n\n        bgp_config = {}\n\n        def build_prefix_limit(af_table, limit, prefix_percent, prefix_timeout):\n            prefix_limit = {}\n            inet = False\n            inet6 = False\n            preifx_type = \"inet\"\n            if isinstance(af_table, list):\n                af_table = str(af_table)\n            if \"ipv4\" in af_table.lower():\n                inet = True\n            if \"ipv6\" in af_table.lower():\n                inet6 = True\n                preifx_type = \"inet6\"\n            if len(af_table.split()) == 2:\n                safi = \"unicast\"\n            else:\n                safi = af_table.split()[-1]\n            if inet or inet6:\n                prefix_limit = {\n                    preifx_type: {\n                        safi: {\n                            \"limit\": limit,\n                            \"teardown\": {\n                                \"threshold\": prefix_percent,\n                                \"timeout\": prefix_timeout,\n                            },\n                        }\n                    }\n                }\n            return prefix_limit\n\n        # Get BGP config using ciscoconfparse because some old devices dont support \"| sec bgp\"\n        cfg = self.get_config(retrieve=\"running\")\n        cfg = cfg[\"running\"].splitlines()\n        bgp_config_text = napalm.base.helpers.cisco_conf_parse_objects(\n            \"router bgp\", cfg\n        )\n        bgp_asn = napalm.base.helpers.regex_find_txt(\n            r\"router bgp (\\d+)\", bgp_config_text, default=0\n        )\n        # Get a list of all neighbors and groups in the config\n        all_neighbors = set()\n        all_groups = set()\n        bgp_group_neighbors = {}\n        all_groups.add(\"_\")\n        for line in bgp_config_text:\n            if \" neighbor \" in line:\n                if re.search(IP_ADDR_REGEX, line) is not None:\n                    all_neighbors.add(re.search(IP_ADDR_REGEX, line).group())\n                elif re.search(IPV6_ADDR_REGEX_2, line) is not None:\n                    all_neighbors.add(re.search(IPV6_ADDR_REGEX_2, line).group())\n                else:\n                    bgp_group = re.search(r\" neighbor [^\\s]+\", line).group()\n                    bgp_group = bgp_group.split()[1]\n                    all_groups.add(bgp_group)\n\n        # Get the neighrbor level config for each neighbor\n        for bgp_neighbor in all_neighbors:\n            # If neighbor_filter is passed in, only continue for that neighbor\n            if neighbor:\n                if bgp_neighbor != neighbor:\n                    continue\n            afi_list = napalm.base.helpers.cisco_conf_parse_parents(\n                r\"\\s+address-family.*\", bgp_neighbor, bgp_config_text\n            )\n            afi = afi_list[0]\n            # Skipping neighbors in VRFs for now\n            if \"vrf\" in str(afi_list):\n                continue\n            else:\n                neighbor_config = napalm.base.helpers.cisco_conf_parse_objects(\n                    bgp_neighbor, bgp_config_text\n                )\n            # For group_name- use peer-group name, else VRF name, else \"_\" for no group\n            group_name = napalm.base.helpers.regex_find_txt(\n                \" peer-group ([^']+)\", neighbor_config, default=\"_\"\n            )\n            # Start finding attributes for the neighbor config\n            description = napalm.base.helpers.regex_find_txt(\n                r\" description ([^\\']+)\\'\", neighbor_config\n            )\n            peer_as = napalm.base.helpers.regex_find_txt(\n                r\" remote-as (\\d+)\", neighbor_config, default=0\n            )\n            prefix_limit = napalm.base.helpers.regex_find_txt(\n                r\"maximum-prefix (\\d+) \\d+ \\w+ \\d+\", neighbor_config, default=0\n            )\n            prefix_percent = napalm.base.helpers.regex_find_txt(\n                r\"maximum-prefix \\d+ (\\d+) \\w+ \\d+\", neighbor_config, default=0\n            )\n            prefix_timeout = napalm.base.helpers.regex_find_txt(\n                r\"maximum-prefix \\d+ \\d+ \\w+ (\\d+)\", neighbor_config, default=0\n            )\n            export_policy = napalm.base.helpers.regex_find_txt(\n                r\"route-map ([^\\s]+) out\", neighbor_config\n            )\n            import_policy = napalm.base.helpers.regex_find_txt(\n                r\"route-map ([^\\s]+) in\", neighbor_config\n            )\n            local_address = napalm.base.helpers.regex_find_txt(\n                r\" update-source (\\w+)\", neighbor_config\n            )\n            local_as = napalm.base.helpers.regex_find_txt(\n                r\"local-as (\\d+)\", neighbor_config, default=0\n            )\n            password = napalm.base.helpers.regex_find_txt(\n                r\"password (?:[0-9] )?([^\\']+\\')\", neighbor_config\n            )\n            nhs = bool(\n                napalm.base.helpers.regex_find_txt(r\" next-hop-self\", neighbor_config)\n            )\n            route_reflector_client = bool(\n                napalm.base.helpers.regex_find_txt(\n                    r\"route-reflector-client\", neighbor_config\n                )\n            )\n\n            # Add the group name to bgp_group_neighbors if its not there already\n            if group_name not in bgp_group_neighbors.keys():\n                bgp_group_neighbors[group_name] = {}\n\n            # Build the neighbor dict of attributes\n            bgp_group_neighbors[group_name][bgp_neighbor] = {\n                \"description\": description,\n                \"remote_as\": peer_as,\n                \"prefix_limit\": build_prefix_limit(\n                    afi, prefix_limit, prefix_percent, prefix_timeout\n                ),\n                \"export_policy\": export_policy,\n                \"import_policy\": import_policy,\n                \"local_address\": local_address,\n                \"local_as\": local_as,\n                \"authentication_key\": password,\n                \"nhs\": nhs,\n                \"route_reflector_client\": route_reflector_client,\n            }\n\n        # Get the peer-group level config for each group\n        for group_name in bgp_group_neighbors.keys():\n            # If a group is passed in params, only continue on that group\n            if group:\n                if group_name != group:\n                    continue\n            # Default no group\n            if group_name == \"_\":\n                bgp_config[\"_\"] = {\n                    \"apply_groups\": [],\n                    \"description\": \"\",\n                    \"local_as\": 0,\n                    \"type\": \"\",\n                    \"import_policy\": \"\",\n                    \"export_policy\": \"\",\n                    \"local_address\": \"\",\n                    \"multipath\": False,\n                    \"multihop_ttl\": 0,\n                    \"remote_as\": 0,\n                    \"remove_private_as\": False,\n                    \"prefix_limit\": {},\n                    \"neighbors\": bgp_group_neighbors.get(\"_\", {}),\n                }\n                continue\n            neighbor_config = napalm.base.helpers.cisco_conf_parse_objects(\n                group_name, bgp_config_text\n            )\n            multipath = False\n            afi_list = napalm.base.helpers.cisco_conf_parse_parents(\n                r\"\\s+address-family.*\", group_name, neighbor_config\n            )\n            for afi in afi_list:\n                afi_config = napalm.base.helpers.cisco_conf_parse_objects(\n                    afi, bgp_config_text\n                )\n                multipath = bool(\n                    napalm.base.helpers.regex_find_txt(r\" multipath\", str(afi_config))\n                )\n                if multipath:\n                    break\n            description = napalm.base.helpers.regex_find_txt(\n                r\" description ([^\\']+)\\'\", neighbor_config\n            )\n            local_as = napalm.base.helpers.regex_find_txt(\n                r\"local-as (\\d+)\", neighbor_config, default=0\n            )\n            import_policy = napalm.base.helpers.regex_find_txt(\n                r\"route-map ([^\\s]+) in\", neighbor_config\n            )\n            export_policy = napalm.base.helpers.regex_find_txt(\n                r\"route-map ([^\\s]+) out\", neighbor_config\n            )\n            local_address = napalm.base.helpers.regex_find_txt(\n                r\" update-source (\\w+)\", neighbor_config\n            )\n            multihop_ttl = napalm.base.helpers.regex_find_txt(\n                r\"ebgp-multihop {\\d+}\", neighbor_config, default=0\n            )\n            peer_as = napalm.base.helpers.regex_find_txt(\n                r\" remote-as (\\d+)\", neighbor_config, default=0\n            )\n            remove_private_as = bool(\n                napalm.base.helpers.regex_find_txt(\n                    r\"remove-private-as\", neighbor_config\n                )\n            )\n            prefix_limit = napalm.base.helpers.regex_find_txt(\n                r\"maximum-prefix (\\d+) \\d+ \\w+ \\d+\", neighbor_config, default=0\n            )\n            prefix_percent = napalm.base.helpers.regex_find_txt(\n                r\"maximum-prefix \\d+ (\\d+) \\w+ \\d+\", neighbor_config, default=0\n            )\n            prefix_timeout = napalm.base.helpers.regex_find_txt(\n                r\"maximum-prefix \\d+ \\d+ \\w+ (\\d+)\", neighbor_config, default=0\n            )\n            bgp_type = \"external\"\n            if local_as:\n                if local_as == peer_as:\n                    bgp_type = \"internal\"\n            elif bgp_asn == peer_as:\n                bgp_type = \"internal\"\n\n            bgp_config[group_name] = {\n                \"apply_groups\": [],  # on IOS will always be empty list!\n                \"description\": description,\n                \"local_as\": local_as,\n                \"type\": bgp_type,\n                \"import_policy\": import_policy,\n                \"export_policy\": export_policy,\n                \"local_address\": local_address,\n                \"multipath\": multipath,\n                \"multihop_ttl\": multihop_ttl,\n                \"remote_as\": peer_as,\n                \"remove_private_as\": remove_private_as,\n                \"prefix_limit\": build_prefix_limit(\n                    afi, prefix_limit, prefix_percent, prefix_timeout\n                ),\n                \"neighbors\": bgp_group_neighbors.get(group_name, {}),\n            }\n        return bgp_config"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ngets BGP neighbor information.", "response": "def get_bgp_neighbors(self):\n        \"\"\"BGP neighbor information.\n\n        Currently no VRF support. Supports both IPv4 and IPv6.\n        \"\"\"\n        supported_afi = [\"ipv4\", \"ipv6\"]\n\n        bgp_neighbor_data = dict()\n        bgp_neighbor_data[\"global\"] = {}\n\n        # get summary output from device\n        cmd_bgp_all_sum = \"show bgp all summary\"\n        summary_output = self._send_command(cmd_bgp_all_sum).strip()\n\n        if \"Invalid input detected\" in summary_output:\n            raise CommandErrorException(\"BGP is not running on this device\")\n\n        # get neighbor output from device\n        neighbor_output = \"\"\n        for afi in supported_afi:\n            cmd_bgp_neighbor = \"show bgp %s unicast neighbors\" % afi\n            neighbor_output += self._send_command(cmd_bgp_neighbor).strip()\n            # trailing newline required for parsing\n            neighbor_output += \"\\n\"\n\n        # Regular expressions used for parsing BGP summary\n        parse_summary = {\n            \"patterns\": [\n                # For address family: IPv4 Unicast\n                {\n                    \"regexp\": re.compile(r\"^For address family: (?P<afi>\\S+) \"),\n                    \"record\": False,\n                },\n                # Capture router_id and local_as values, e.g.:\n                # BGP router identifier 10.0.1.1, local AS number 65000\n                {\n                    \"regexp\": re.compile(\n                        r\"^.* router identifier (?P<router_id>{}), \"\n                        r\"local AS number (?P<local_as>{})\".format(\n                            IPV4_ADDR_REGEX, ASN_REGEX\n                        )\n                    ),\n                    \"record\": False,\n                },\n                # Match neighbor summary row, capturing useful details and\n                # discarding the 5 columns that we don't care about, e.g.:\n                # Neighbor   V          AS MsgRcvd MsgSent   TblVer  InQ OutQ Up/Down  State/PfxRcd\n                # 10.0.0.2   4       65000 1336020 64337701 1011343614    0    0 8w0d         3143\n                {\n                    \"regexp\": re.compile(\n                        r\"^\\*?(?P<remote_addr>({})|({}))\"\n                        r\"\\s+\\d+\\s+(?P<remote_as>{})(\\s+\\S+){{5}}\\s+\"\n                        r\"(?P<uptime>(never)|\\d+\\S+)\"\n                        r\"\\s+(?P<accepted_prefixes>\\d+)\".format(\n                            IPV4_ADDR_REGEX, IPV6_ADDR_REGEX, ASN_REGEX\n                        )\n                    ),\n                    \"record\": True,\n                },\n                # Same as above, but for peer that are not Established, e.g.:\n                # Neighbor      V       AS MsgRcvd MsgSent   TblVer  InQ OutQ Up/Down  State/PfxRcd\n                # 192.168.0.2   4    65002       0       0        1    0    0 never    Active\n                {\n                    \"regexp\": re.compile(\n                        r\"^\\*?(?P<remote_addr>({})|({}))\"\n                        r\"\\s+\\d+\\s+(?P<remote_as>{})(\\s+\\S+){{5}}\\s+\"\n                        r\"(?P<uptime>(never)|\\d+\\S+)\\s+(?P<state>\\D.*)\".format(\n                            IPV4_ADDR_REGEX, IPV6_ADDR_REGEX, ASN_REGEX\n                        )\n                    ),\n                    \"record\": True,\n                },\n                # ipv6 peers often break accross rows because of the longer peer address,\n                # match as above, but in separate expressions, e.g.:\n                # Neighbor      V       AS MsgRcvd MsgSent   TblVer  InQ OutQ Up/Down  State/PfxRcd\n                # 2001:DB8::4\n                #               4    65004 9900690  612449 155362939    0    0 26w6d       36391\n                {\n                    \"regexp\": re.compile(\n                        r\"^\\*?(?P<remote_addr>({})|({}))\".format(\n                            IPV4_ADDR_REGEX, IPV6_ADDR_REGEX\n                        )\n                    ),\n                    \"record\": False,\n                },\n                {\n                    \"regexp\": re.compile(\n                        r\"^\\s+\\d+\\s+(?P<remote_as>{})(\\s+\\S+){{5}}\\s+\"\n                        r\"(?P<uptime>(never)|\\d+\\S+)\"\n                        r\"\\s+(?P<accepted_prefixes>\\d+)\".format(ASN_REGEX)\n                    ),\n                    \"record\": True,\n                },\n                # Same as above, but for peers that are not Established, e.g.:\n                # Neighbor      V       AS MsgRcvd MsgSent   TblVer  InQ OutQ Up/Down  State/PfxRcd\n                # 2001:DB8::3\n                #               4    65003       0       0        1    0    0 never    Idle (Admin)\n                {\n                    \"regexp\": re.compile(\n                        r\"^\\s+\\d+\\s+(?P<remote_as>{})(\\s+\\S+){{5}}\\s+\"\n                        r\"(?P<uptime>(never)|\\d+\\S+)\\s+(?P<state>\\D.*)\".format(\n                            ASN_REGEX\n                        )\n                    ),\n                    \"record\": True,\n                },\n            ],\n            \"no_fill_fields\": [\n                \"accepted_prefixes\",\n                \"state\",\n                \"uptime\",\n                \"remote_as\",\n                \"remote_addr\",\n            ],\n        }\n\n        parse_neighbors = {\n            \"patterns\": [\n                # Capture BGP neighbor is 10.0.0.2,  remote AS 65000, internal link\n                {\n                    \"regexp\": re.compile(\n                        r\"^BGP neighbor is (?P<remote_addr>({})|({})),\"\n                        r\"\\s+remote AS (?P<remote_as>{}).*\".format(\n                            IPV4_ADDR_REGEX, IPV6_ADDR_REGEX, ASN_REGEX\n                        )\n                    ),\n                    \"record\": False,\n                },\n                # Capture description\n                {\n                    \"regexp\": re.compile(r\"^\\s+Description: (?P<description>.+)\"),\n                    \"record\": False,\n                },\n                # Capture remote_id, e.g.:\n                # BGP version 4, remote router ID 10.0.1.2\n                {\n                    \"regexp\": re.compile(\n                        r\"^\\s+BGP version \\d+, remote router ID \"\n                        r\"(?P<remote_id>{})\".format(IPV4_ADDR_REGEX)\n                    ),\n                    \"record\": False,\n                },\n                # Capture state\n                {\n                    \"regexp\": re.compile(r\"^\\s+BGP state = (?P<state>\\w+)\"),\n                    \"record\": False,\n                },\n                # Capture AFI and SAFI names, e.g.:\n                # For address family: IPv4 Unicast\n                {\n                    \"regexp\": re.compile(r\"^\\s+For address family: (?P<afi>\\S+) \"),\n                    \"record\": False,\n                },\n                # Capture current sent and accepted prefixes, e.g.:\n                #     Prefixes Current:          637213       3142 (Consumes 377040 bytes)\n                {\n                    \"regexp\": re.compile(\n                        r\"^\\s+Prefixes Current:\\s+(?P<sent_prefixes>\\d+)\\s+\"\n                        r\"(?P<accepted_prefixes>\\d+).*\"\n                    ),\n                    \"record\": False,\n                },\n                # Capture received_prefixes if soft-reconfig is enabled for the peer\n                {\n                    \"regexp\": re.compile(\n                        r\"^\\s+Saved (soft-reconfig):.+(?P<received_prefixes>\\d+).*\"\n                    ),\n                    \"record\": True,\n                },\n                # Otherwise, use the following as an end of row marker\n                {\n                    \"regexp\": re.compile(r\"^\\s+Local Policy Denied Prefixes:.+\"),\n                    \"record\": True,\n                },\n            ],\n            # fields that should not be \"filled down\" across table rows\n            \"no_fill_fields\": [\n                \"received_prefixes\",\n                \"accepted_prefixes\",\n                \"sent_prefixes\",\n            ],\n        }\n\n        # Parse outputs into a list of dicts\n        summary_data = []\n        summary_data_entry = {}\n\n        for line in summary_output.splitlines():\n            # check for matches against each pattern\n            for item in parse_summary[\"patterns\"]:\n                match = item[\"regexp\"].match(line)\n                if match:\n                    # a match was found, so update the temp entry with the match's groupdict\n                    summary_data_entry.update(match.groupdict())\n                    if item[\"record\"]:\n                        # Record indicates the last piece of data has been obtained; move\n                        # on to next entry\n                        summary_data.append(copy.deepcopy(summary_data_entry))\n                        # remove keys that are listed in no_fill_fields before the next pass\n                        for field in parse_summary[\"no_fill_fields\"]:\n                            try:\n                                del summary_data_entry[field]\n                            except KeyError:\n                                pass\n                    break\n\n        neighbor_data = []\n        neighbor_data_entry = {}\n        for line in neighbor_output.splitlines():\n            # check for matches against each pattern\n            for item in parse_neighbors[\"patterns\"]:\n                match = item[\"regexp\"].match(line)\n                if match:\n                    # a match was found, so update the temp entry with the match's groupdict\n                    neighbor_data_entry.update(match.groupdict())\n                    if item[\"record\"]:\n                        # Record indicates the last piece of data has been obtained; move\n                        # on to next entry\n                        neighbor_data.append(copy.deepcopy(neighbor_data_entry))\n                        # remove keys that are listed in no_fill_fields before the next pass\n                        for field in parse_neighbors[\"no_fill_fields\"]:\n                            try:\n                                del neighbor_data_entry[field]\n                            except KeyError:\n                                pass\n                    break\n\n        router_id = None\n\n        for entry in summary_data:\n            if not router_id:\n                router_id = entry[\"router_id\"]\n            elif entry[\"router_id\"] != router_id:\n                raise ValueError\n\n        # check the router_id looks like an ipv4 address\n        router_id = napalm.base.helpers.ip(router_id, version=4)\n\n        # add parsed data to output dict\n        bgp_neighbor_data[\"global\"][\"router_id\"] = router_id\n        bgp_neighbor_data[\"global\"][\"peers\"] = {}\n        for entry in summary_data:\n            remote_addr = napalm.base.helpers.ip(entry[\"remote_addr\"])\n            afi = entry[\"afi\"].lower()\n            # check that we're looking at a supported afi\n            if afi not in supported_afi:\n                continue\n            # get neighbor_entry out of neighbor data\n            neighbor_entry = None\n            for neighbor in neighbor_data:\n                if (\n                    neighbor[\"afi\"].lower() == afi\n                    and napalm.base.helpers.ip(neighbor[\"remote_addr\"]) == remote_addr\n                ):\n                    neighbor_entry = neighbor\n                    break\n            # check for proper session data for the afi\n            if neighbor_entry is None:\n                continue\n            elif not isinstance(neighbor_entry, dict):\n                raise ValueError(\n                    msg=\"Couldn't find neighbor data for %s in afi %s\"\n                    % (remote_addr, afi)\n                )\n\n            # check for admin down state\n            try:\n                if \"(Admin)\" in entry[\"state\"]:\n                    is_enabled = False\n                else:\n                    is_enabled = True\n            except KeyError:\n                is_enabled = True\n\n            # parse uptime value\n            uptime = self.bgp_time_conversion(entry[\"uptime\"])\n\n            # BGP is up if state is Established\n            is_up = \"Established\" in neighbor_entry[\"state\"]\n\n            # check whether session is up for address family and get prefix count\n            try:\n                accepted_prefixes = int(entry[\"accepted_prefixes\"])\n            except (ValueError, KeyError):\n                accepted_prefixes = -1\n\n            # Only parse neighbor detailed data if BGP session is-up\n            if is_up:\n                try:\n                    # overide accepted_prefixes with neighbor data if possible (since that's newer)\n                    accepted_prefixes = int(neighbor_entry[\"accepted_prefixes\"])\n                except (ValueError, KeyError):\n                    pass\n\n                # try to get received prefix count, otherwise set to accepted_prefixes\n                received_prefixes = neighbor_entry.get(\n                    \"received_prefixes\", accepted_prefixes\n                )\n\n                # try to get sent prefix count and convert to int, otherwise set to -1\n                sent_prefixes = int(neighbor_entry.get(\"sent_prefixes\", -1))\n            else:\n                received_prefixes = -1\n                sent_prefixes = -1\n                uptime = -1\n\n            # get description\n            try:\n                description = py23_compat.text_type(neighbor_entry[\"description\"])\n            except KeyError:\n                description = \"\"\n\n            # check the remote router_id looks like an ipv4 address\n            remote_id = napalm.base.helpers.ip(neighbor_entry[\"remote_id\"], version=4)\n\n            if remote_addr not in bgp_neighbor_data[\"global\"][\"peers\"]:\n                bgp_neighbor_data[\"global\"][\"peers\"][remote_addr] = {\n                    \"local_as\": napalm.base.helpers.as_number(entry[\"local_as\"]),\n                    \"remote_as\": napalm.base.helpers.as_number(entry[\"remote_as\"]),\n                    \"remote_id\": remote_id,\n                    \"is_up\": is_up,\n                    \"is_enabled\": is_enabled,\n                    \"description\": description,\n                    \"uptime\": uptime,\n                    \"address_family\": {\n                        afi: {\n                            \"received_prefixes\": received_prefixes,\n                            \"accepted_prefixes\": accepted_prefixes,\n                            \"sent_prefixes\": sent_prefixes,\n                        }\n                    },\n                }\n            else:\n                # found previous data for matching remote_addr, but for different afi\n                existing = bgp_neighbor_data[\"global\"][\"peers\"][remote_addr]\n                assert afi not in existing[\"address_family\"]\n                # compare with existing values and croak if they don't match\n                assert existing[\"local_as\"] == napalm.base.helpers.as_number(\n                    entry[\"local_as\"]\n                )\n                assert existing[\"remote_as\"] == napalm.base.helpers.as_number(\n                    entry[\"remote_as\"]\n                )\n                assert existing[\"remote_id\"] == remote_id\n                assert existing[\"is_enabled\"] == is_enabled\n                assert existing[\"description\"] == description\n                # merge other values in a sane manner\n                existing[\"is_up\"] = existing[\"is_up\"] or is_up\n                existing[\"uptime\"] = max(existing[\"uptime\"], uptime)\n                existing[\"address_family\"][afi] = {\n                    \"received_prefixes\": received_prefixes,\n                    \"accepted_prefixes\": accepted_prefixes,\n                    \"sent_prefixes\": sent_prefixes,\n                }\n        return bgp_neighbor_data"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ngets the counters for all interfaces.", "response": "def get_interfaces_counters(self):\n        \"\"\"\n        Return interface counters and errors.\n\n        'tx_errors': int,\n        'rx_errors': int,\n        'tx_discards': int,\n        'rx_discards': int,\n        'tx_octets': int,\n        'rx_octets': int,\n        'tx_unicast_packets': int,\n        'rx_unicast_packets': int,\n        'tx_multicast_packets': int,\n        'rx_multicast_packets': int,\n        'tx_broadcast_packets': int,\n        'rx_broadcast_packets': int,\n\n        Currently doesn't determine output broadcasts, multicasts\n        \"\"\"\n        counters = {}\n        command = \"show interfaces\"\n        output = self._send_command(command)\n        sh_int_sum_cmd = \"show interface summary\"\n        sh_int_sum_cmd_out = self._send_command(sh_int_sum_cmd)\n\n        # Break output into per-interface sections\n        interface_strings = re.split(r\".* line protocol is .*\", output, flags=re.M)\n        header_strings = re.findall(r\".* line protocol is .*\", output, flags=re.M)\n\n        empty = interface_strings.pop(0).strip()\n        if empty:\n            raise ValueError(\"Unexpected output from: {}\".format(command))\n\n        # Parse out the interface names\n        intf = []\n        for intf_line in header_strings:\n            interface, _ = re.split(r\" is .* line protocol is \", intf_line)\n            intf.append(interface.strip())\n\n        if len(intf) != len(interface_strings):\n            raise ValueError(\"Unexpected output from: {}\".format(command))\n\n        # Re-join interface names with interface strings\n        for interface, interface_str in zip(intf, interface_strings):\n            counters.setdefault(interface, {})\n            for line in interface_str.splitlines():\n                if \"packets input\" in line:\n                    # '0 packets input, 0 bytes, 0 no buffer'\n                    match = re.search(r\"(\\d+) packets input.* (\\d+) bytes\", line)\n                    counters[interface][\"rx_unicast_packets\"] = int(match.group(1))\n                    counters[interface][\"rx_octets\"] = int(match.group(2))\n                elif \"broadcast\" in line:\n                    # 'Received 0 broadcasts (0 multicasts)'\n                    # 'Received 264071 broadcasts (39327 IP multicasts)'\n                    # 'Received 338 broadcasts, 0 runts, 0 giants, 0 throttles'\n                    match = re.search(\n                        r\"Received (\\d+) broadcasts.*(\\d+).*multicasts\", line\n                    )\n                    alt_match = re.search(r\"Received (\\d+) broadcasts.*\", line)\n                    if match:\n                        counters[interface][\"rx_broadcast_packets\"] = int(\n                            match.group(1)\n                        )\n                        counters[interface][\"rx_multicast_packets\"] = int(\n                            match.group(2)\n                        )\n                    elif alt_match:\n                        counters[interface][\"rx_broadcast_packets\"] = int(\n                            alt_match.group(1)\n                        )\n                        counters[interface][\"rx_multicast_packets\"] = -1\n                    else:\n                        counters[interface][\"rx_broadcast_packets\"] = -1\n                        counters[interface][\"rx_multicast_packets\"] = -1\n                elif \"packets output\" in line:\n                    # '0 packets output, 0 bytes, 0 underruns'\n                    match = re.search(r\"(\\d+) packets output.* (\\d+) bytes\", line)\n                    counters[interface][\"tx_unicast_packets\"] = int(match.group(1))\n                    counters[interface][\"tx_octets\"] = int(match.group(2))\n                    counters[interface][\"tx_broadcast_packets\"] = -1\n                    counters[interface][\"tx_multicast_packets\"] = -1\n                elif \"input errors\" in line:\n                    # '0 input errors, 0 CRC, 0 frame, 0 overrun, 0 ignored'\n                    match = re.search(r\"(\\d+) input errors\", line)\n                    counters[interface][\"rx_errors\"] = int(match.group(1))\n                    counters[interface][\"rx_discards\"] = -1\n                elif \"output errors\" in line:\n                    # '0 output errors, 0 collisions, 1 interface resets'\n                    match = re.search(r\"(\\d+) output errors\", line)\n                    counters[interface][\"tx_errors\"] = int(match.group(1))\n                    counters[interface][\"tx_discards\"] = -1\n            for line in sh_int_sum_cmd_out.splitlines():\n                if interface in line:\n                    # Line is tabular output with columns\n                    # Interface  IHQ  IQD  OHQ  OQD  RXBS  RXPS  TXBS  TXPS  TRTL\n                    # where columns (excluding interface) are integers\n                    regex = (\n                        r\"\\b\"\n                        + interface\n                        + r\"\\b\\s+(\\d+)\\s+(?P<IQD>\\d+)\\s+(\\d+)\"\n                        + r\"\\s+(?P<OQD>\\d+)\\s+(\\d+)\\s+(\\d+)\"\n                        + r\"\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\"\n                    )\n                    match = re.search(regex, line)\n                    if match:\n                        counters[interface][\"rx_discards\"] = int(match.group(\"IQD\"))\n                        counters[interface][\"tx_discards\"] = int(match.group(\"OQD\"))\n\n        return counters"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nget the environment of the current system.", "response": "def get_environment(self):\n        \"\"\"\n        Get environment facts.\n\n        power and fan are currently not implemented\n        cpu is using 1-minute average\n        cpu hard-coded to cpu0 (i.e. only a single CPU)\n        \"\"\"\n        environment = {}\n        cpu_cmd = \"show proc cpu\"\n        mem_cmd = \"show memory statistics\"\n        temp_cmd = \"show env temperature status\"\n\n        output = self._send_command(cpu_cmd)\n        environment.setdefault(\"cpu\", {})\n        environment[\"cpu\"][0] = {}\n        environment[\"cpu\"][0][\"%usage\"] = 0.0\n        for line in output.splitlines():\n            if \"CPU utilization\" in line:\n                # CPU utilization for five seconds: 2%/0%; one minute: 2%; five minutes: 1%\n                cpu_regex = r\"^.*one minute: (\\d+)%; five.*$\"\n                match = re.search(cpu_regex, line)\n                environment[\"cpu\"][0][\"%usage\"] = float(match.group(1))\n                break\n\n        output = self._send_command(mem_cmd)\n        for line in output.splitlines():\n            if \"Processor\" in line:\n                _, _, proc_total_mem, proc_used_mem, _ = line.split()[:5]\n            elif \"I/O\" in line or \"io\" in line:\n                _, _, io_total_mem, io_used_mem, _ = line.split()[:5]\n        total_mem = int(proc_total_mem) + int(io_total_mem)\n        used_mem = int(proc_used_mem) + int(io_used_mem)\n        environment.setdefault(\"memory\", {})\n        environment[\"memory\"][\"used_ram\"] = used_mem\n        environment[\"memory\"][\"available_ram\"] = total_mem\n\n        environment.setdefault(\"temperature\", {})\n        re_temp_value = re.compile(\"(.*) Temperature Value\")\n        # The 'show env temperature status' is not ubiquitous in Cisco IOS\n        output = self._send_command(temp_cmd)\n        if \"% Invalid\" not in output:\n            for line in output.splitlines():\n                m = re_temp_value.match(line)\n                if m is not None:\n                    temp_name = m.group(1).lower()\n                    temp_value = float(line.split(\":\")[1].split()[0])\n                    env_value = {\n                        \"is_alert\": False,\n                        \"is_critical\": False,\n                        \"temperature\": temp_value,\n                    }\n                    environment[\"temperature\"][temp_name] = env_value\n                elif \"Yellow Threshold\" in line:\n                    system_temp_alert = float(line.split(\":\")[1].split()[0])\n                    if temp_value > system_temp_alert:\n                        env_value[\"is_alert\"] = True\n                elif \"Red Threshold\" in line:\n                    system_temp_crit = float(line.split(\":\")[1].split()[0])\n                    if temp_value > system_temp_crit:\n                        env_value[\"is_critical\"] = True\n        else:\n            env_value = {\"is_alert\": False, \"is_critical\": False, \"temperature\": -1.0}\n            environment[\"temperature\"][\"invalid\"] = env_value\n\n        # Initialize 'power' and 'fan' to default values (not implemented)\n        environment.setdefault(\"power\", {})\n        environment[\"power\"][\"invalid\"] = {\n            \"status\": True,\n            \"output\": -1.0,\n            \"capacity\": -1.0,\n        }\n        environment.setdefault(\"fans\", {})\n        environment[\"fans\"][\"invalid\"] = {\"status\": True}\n\n        return environment"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_ntp_peers(self):\n        ntp_stats = self.get_ntp_stats()\n\n        return {\n            ntp_peer.get(\"remote\"): {}\n            for ntp_peer in ntp_stats\n            if ntp_peer.get(\"remote\")\n        }", "response": "Implementation of get_ntp_peers for IOS."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns a list of dictionaries representing the MAC Address Table and the MAC Address Types of the Mac Address.", "response": "def get_mac_address_table(self):\n        \"\"\"\n        Returns a lists of dictionaries. Each dictionary represents an entry in the MAC Address\n        Table, having the following keys\n            * mac (string)\n            * interface (string)\n            * vlan (int)\n            * active (boolean)\n            * static (boolean)\n            * moves (int)\n            * last_move (float)\n\n        Format1:\n        Destination Address  Address Type  VLAN  Destination Port\n        -------------------  ------------  ----  --------------------\n        6400.f1cf.2cc6          Dynamic       1     Wlan-GigabitEthernet0\n\n        Cat 6500:\n        Legend: * - primary entry\n                age - seconds since last seen\n                n/a - not available\n\n          vlan   mac address     type    learn     age              ports\n        ------+----------------+--------+-----+----------+--------------------------\n        *  999  1111.2222.3333   dynamic  Yes          0   Port-channel1\n           999  1111.2222.3333   dynamic  Yes          0   Port-channel1\n\n        Cat 4948\n        Unicast Entries\n         vlan   mac address     type        protocols               port\n        -------+---------------+--------+---------------------+--------------------\n         999    1111.2222.3333   dynamic ip                    Port-channel1\n\n        Cat 2960\n        Mac Address Table\n        -------------------------------------------\n\n        Vlan    Mac Address       Type        Ports\n        ----    -----------       --------    -----\n        All    1111.2222.3333    STATIC      CPU\n        \"\"\"\n\n        RE_MACTABLE_DEFAULT = r\"^\" + MAC_REGEX\n        RE_MACTABLE_6500_1 = r\"^\\*\\s+{}\\s+{}\\s+\".format(\n            VLAN_REGEX, MAC_REGEX\n        )  # 7 fields\n        RE_MACTABLE_6500_2 = r\"^{}\\s+{}\\s+\".format(VLAN_REGEX, MAC_REGEX)  # 6 fields\n        RE_MACTABLE_6500_3 = r\"^\\s{51}\\S+\"  # Fill down prior\n        RE_MACTABLE_6500_4 = r\"^R\\s+{}\\s+.*Router\".format(\n            VLAN_REGEX, MAC_REGEX\n        )  # Router field\n        RE_MACTABLE_6500_5 = r\"^R\\s+N/A\\s+{}.*Router\".format(\n            MAC_REGEX\n        )  # Router skipped\n        RE_MACTABLE_4500_1 = r\"^{}\\s+{}\\s+\".format(VLAN_REGEX, MAC_REGEX)  # 5 fields\n        RE_MACTABLE_4500_2 = r\"^\\s{32,34}\\S+\"  # Fill down prior\n        RE_MACTABLE_4500_3 = r\"^{}\\s+{}\\s+\".format(\n            INT_REGEX, MAC_REGEX\n        )  # Matches PHY int\n        RE_MACTABLE_2960_1 = r\"^All\\s+{}\".format(MAC_REGEX)\n        RE_MACTABLE_GEN_1 = r\"^{}\\s+{}\\s+\".format(\n            VLAN_REGEX, MAC_REGEX\n        )  # 4 fields-2960/4500\n\n        def process_mac_fields(vlan, mac, mac_type, interface):\n            \"\"\"Return proper data for mac address fields.\"\"\"\n            if mac_type.lower() in [\"self\", \"static\", \"system\"]:\n                static = True\n                if vlan.lower() == \"all\":\n                    vlan = 0\n                if (\n                    interface.lower() == \"cpu\"\n                    or re.search(r\"router\", interface.lower())\n                    or re.search(r\"switch\", interface.lower())\n                ):\n                    interface = \"\"\n            else:\n                static = False\n\n            return {\n                \"mac\": napalm.base.helpers.mac(mac),\n                \"interface\": self._canonical_int(interface),\n                \"vlan\": int(vlan),\n                \"static\": static,\n                \"active\": True,\n                \"moves\": -1,\n                \"last_move\": -1.0,\n            }\n\n        mac_address_table = []\n        command = IOS_COMMANDS[\"show_mac_address\"]\n        output = self._send_command(command)\n\n        # Skip the header lines\n        output = re.split(r\"^----.*\", output, flags=re.M)[1:]\n        output = \"\\n\".join(output).strip()\n        # Strip any leading asterisks\n        output = re.sub(r\"^\\*\", \"\", output, flags=re.M)\n        fill_down_vlan = fill_down_mac = fill_down_mac_type = \"\"\n        for line in output.splitlines():\n            # Cat6500 one off and 4500 multicast format\n            if re.search(RE_MACTABLE_6500_3, line) or re.search(\n                RE_MACTABLE_4500_2, line\n            ):\n                interface = line.strip()\n                if \",\" in interface:\n                    interfaces = interface.split(\",\")\n                else:\n                    interfaces = [interface]\n                for single_interface in interfaces:\n                    mac_address_table.append(\n                        process_mac_fields(\n                            fill_down_vlan,\n                            fill_down_mac,\n                            fill_down_mac_type,\n                            single_interface,\n                        )\n                    )\n                continue\n            line = line.strip()\n            if line == \"\":\n                continue\n            if re.search(r\"^---\", line):\n                # Convert any '---' to VLAN 0\n                line = re.sub(r\"^---\", \"0\", line, flags=re.M)\n\n            # Format1\n            if re.search(RE_MACTABLE_DEFAULT, line):\n                if len(line.split()) == 4:\n                    mac, mac_type, vlan, interface = line.split()\n                    mac_address_table.append(\n                        process_mac_fields(vlan, mac, mac_type, interface)\n                    )\n                else:\n                    raise ValueError(\"Unexpected output from: {}\".format(line.split()))\n            # Cat6500 format\n            elif (\n                re.search(RE_MACTABLE_6500_1, line)\n                or re.search(RE_MACTABLE_6500_2, line)\n            ) and len(line.split()) >= 6:\n                if len(line.split()) == 7:\n                    _, vlan, mac, mac_type, _, _, interface = line.split()\n                elif len(line.split()) == 6:\n                    vlan, mac, mac_type, _, _, interface = line.split()\n                if \",\" in interface:\n                    interfaces = interface.split(\",\")\n                    fill_down_vlan = vlan\n                    fill_down_mac = mac\n                    fill_down_mac_type = mac_type\n                    for single_interface in interfaces:\n                        mac_address_table.append(\n                            process_mac_fields(vlan, mac, mac_type, single_interface)\n                        )\n                else:\n                    mac_address_table.append(\n                        process_mac_fields(vlan, mac, mac_type, interface)\n                    )\n            # Cat4500 format\n            elif re.search(RE_MACTABLE_4500_1, line) and len(line.split()) == 5:\n                vlan, mac, mac_type, _, interface = line.split()\n                mac_address_table.append(\n                    process_mac_fields(vlan, mac, mac_type, interface)\n                )\n            # Cat4500 w/PHY interface in Mac Table. Vlan will be -1.\n            elif re.search(RE_MACTABLE_4500_3, line) and len(line.split()) == 5:\n                interface, mac, mac_type, _, _ = line.split()\n                interface = canonical_interface_name(interface)\n                vlan = \"-1\"\n                mac_address_table.append(\n                    process_mac_fields(vlan, mac, mac_type, interface)\n                )\n            # Cat2960 format - ignore extra header line\n            elif re.search(r\"^Vlan\\s+Mac Address\\s+\", line):\n                continue\n            # Cat2960 format (Cat4500 format multicast entries)\n            elif (\n                re.search(RE_MACTABLE_2960_1, line)\n                or re.search(RE_MACTABLE_GEN_1, line)\n            ) and len(line.split()) == 4:\n                vlan, mac, mac_type, interface = line.split()\n                if \",\" in interface:\n                    interfaces = interface.split(\",\")\n                    fill_down_vlan = vlan\n                    fill_down_mac = mac\n                    fill_down_mac_type = mac_type\n                    for single_interface in interfaces:\n                        mac_address_table.append(\n                            process_mac_fields(vlan, mac, mac_type, single_interface)\n                        )\n                else:\n                    mac_address_table.append(\n                        process_mac_fields(vlan, mac, mac_type, interface)\n                    )\n            # 4500 in case of unused Vlan 1.\n            elif re.search(RE_MACTABLE_4500_1, line) and len(line.split()) == 3:\n                vlan, mac, mac_type = line.split()\n                mac_address_table.append(\n                    process_mac_fields(vlan, mac, mac_type, interface=\"\")\n                )\n            # 4500 w/PHY interface in Multicast table. Vlan will be -1.\n            elif re.search(RE_MACTABLE_4500_3, line) and len(line.split()) == 4:\n                vlan, mac, mac_type, interface = line.split()\n                vlan = \"-1\"\n                mac_address_table.append(\n                    process_mac_fields(vlan, mac, mac_type, interface)\n                )\n            elif re.search(RE_MACTABLE_6500_4, line) and len(line.split()) == 7:\n                line = re.sub(r\"^R\\s+\", \"\", line)\n                vlan, mac, mac_type, _, _, interface = line.split()\n                mac_address_table.append(\n                    process_mac_fields(vlan, mac, mac_type, interface)\n                )\n                continue\n            elif re.search(RE_MACTABLE_6500_5, line):\n                line = re.sub(r\"^R\\s+\", \"\", line)\n                vlan, mac, mac_type, _, _, interface = line.split()\n                # Convert 'N/A' VLAN to to 0\n                vlan = re.sub(r\"N/A\", \"0\", vlan)\n                mac_address_table.append(\n                    process_mac_fields(vlan, mac, mac_type, interface)\n                )\n                continue\n            elif re.search(r\"Total Mac Addresses\", line):\n                continue\n            elif re.search(r\"Multicast Entries\", line):\n                continue\n            elif re.search(r\"vlan.*mac.*address.*type.*\", line):\n                continue\n            elif re.search(\n                r\"Displaying entries from active supervisor:\\s+\\w+\\s+\\[\\d\\]:\", line\n            ):\n                continue\n            else:\n                raise ValueError(\"Unexpected output from: {}\".format(repr(line)))\n\n        return mac_address_table"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef get_snmp_information(self):\n        # default values\n        snmp_dict = {\n            \"chassis_id\": \"unknown\",\n            \"community\": {},\n            \"contact\": \"unknown\",\n            \"location\": \"unknown\",\n        }\n        command = \"show run | include snmp-server\"\n        output = self._send_command(command)\n        for line in output.splitlines():\n            fields = line.split()\n            if \"snmp-server community\" in line:\n                name = fields[2]\n                if \"community\" not in snmp_dict.keys():\n                    snmp_dict.update({\"community\": {}})\n                snmp_dict[\"community\"].update({name: {}})\n                try:\n                    snmp_dict[\"community\"][name].update({\"mode\": fields[3].lower()})\n                except IndexError:\n                    snmp_dict[\"community\"][name].update({\"mode\": \"N/A\"})\n                try:\n                    snmp_dict[\"community\"][name].update({\"acl\": fields[4]})\n                except IndexError:\n                    snmp_dict[\"community\"][name].update({\"acl\": \"N/A\"})\n            elif \"snmp-server location\" in line:\n                snmp_dict[\"location\"] = \" \".join(fields[2:])\n            elif \"snmp-server contact\" in line:\n                snmp_dict[\"contact\"] = \" \".join(fields[2:])\n            elif \"snmp-server chassis-id\" in line:\n                snmp_dict[\"chassis_id\"] = \" \".join(fields[2:])\n        # If SNMP Chassis wasn't found; obtain using direct command\n        if snmp_dict[\"chassis_id\"] == \"unknown\":\n            command = \"show snmp chassis\"\n            snmp_chassis = self._send_command(command)\n            snmp_dict[\"chassis_id\"] = snmp_chassis\n        return snmp_dict", "response": "Returns a dict of dicts with the SNMP information for the current locale."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning a dictionary with the configured users.", "response": "def get_users(self):\n        \"\"\"\n        Returns a dictionary with the configured users.\n        The keys of the main dictionary represents the username.\n        The values represent the details of the user,\n        represented by the following keys:\n\n            * level (int)\n            * password (str)\n            * sshkeys (list)\n\n        *Note: sshkeys on ios is the ssh key fingerprint\n\n        The level is an integer between 0 and 15, where 0 is the\n        lowest access and 15 represents full access to the device.\n        \"\"\"\n        username_regex = (\n            r\"^username\\s+(?P<username>\\S+)\\s+(?:privilege\\s+(?P<priv_level>\\S+)\"\n            r\"\\s+)?(?:secret \\d+\\s+(?P<pwd_hash>\\S+))?$\"\n        )\n        pub_keychain_regex = (\n            r\"^\\s+username\\s+(?P<username>\\S+)(?P<keys>(?:\\n\\s+key-hash\\s+\"\n            r\"(?P<hash_type>\\S+)\\s+(?P<hash>\\S+)(?:\\s+\\S+)?)+)$\"\n        )\n        users = {}\n        command = \"show run | section username\"\n        output = self._send_command(command)\n        for match in re.finditer(username_regex, output, re.M):\n            users[match.groupdict()[\"username\"]] = {\n                \"level\": int(match.groupdict()[\"priv_level\"])\n                if match.groupdict()[\"priv_level\"]\n                else 1,\n                \"password\": match.groupdict()[\"pwd_hash\"]\n                if match.groupdict()[\"pwd_hash\"]\n                else \"\",\n                \"sshkeys\": [],\n            }\n        for match in re.finditer(pub_keychain_regex, output, re.M):\n            if match.groupdict()[\"username\"] not in users:\n                continue\n            users[match.groupdict()[\"username\"]][\"sshkeys\"] = list(\n                map(\n                    lambda s: s.strip()[9:],\n                    filter(None, match.groupdict()[\"keys\"].splitlines()),\n                )\n            )\n        return users"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nexecuting ping on the device and returns the result.", "response": "def ping(\n        self,\n        destination,\n        source=C.PING_SOURCE,\n        ttl=C.PING_TTL,\n        timeout=C.PING_TIMEOUT,\n        size=C.PING_SIZE,\n        count=C.PING_COUNT,\n        vrf=C.PING_VRF,\n    ):\n        \"\"\"\n        Execute ping on the device and returns a dictionary with the result.\n\n        Output dictionary has one of following keys:\n            * success\n            * error\n        In case of success, inner dictionary will have the followin keys:\n            * probes_sent (int)\n            * packet_loss (int)\n            * rtt_min (float)\n            * rtt_max (float)\n            * rtt_avg (float)\n            * rtt_stddev (float)\n            * results (list)\n        'results' is a list of dictionaries with the following keys:\n            * ip_address (str)\n            * rtt (float)\n        \"\"\"\n        ping_dict = {}\n        # vrf needs to be right after the ping command\n        if vrf:\n            command = \"ping vrf {} {}\".format(vrf, destination)\n        else:\n            command = \"ping {}\".format(destination)\n        command += \" timeout {}\".format(timeout)\n        command += \" size {}\".format(size)\n        command += \" repeat {}\".format(count)\n        if source != \"\":\n            command += \" source {}\".format(source)\n\n        output = self._send_command(command)\n        if \"%\" in output:\n            ping_dict[\"error\"] = output\n        elif \"Sending\" in output:\n            ping_dict[\"success\"] = {\n                \"probes_sent\": 0,\n                \"packet_loss\": 0,\n                \"rtt_min\": 0.0,\n                \"rtt_max\": 0.0,\n                \"rtt_avg\": 0.0,\n                \"rtt_stddev\": 0.0,\n                \"results\": [],\n            }\n\n            for line in output.splitlines():\n                if \"Success rate is\" in line:\n                    sent_and_received = re.search(r\"\\((\\d*)/(\\d*)\\)\", line)\n                    probes_sent = int(sent_and_received.group(2))\n                    probes_received = int(sent_and_received.group(1))\n                    ping_dict[\"success\"][\"probes_sent\"] = probes_sent\n                    ping_dict[\"success\"][\"packet_loss\"] = probes_sent - probes_received\n                    # If there were zero valid response packets, we are done\n                    if \"Success rate is 0 \" in line:\n                        break\n\n                    min_avg_max = re.search(r\"(\\d*)/(\\d*)/(\\d*)\", line)\n                    ping_dict[\"success\"].update(\n                        {\n                            \"rtt_min\": float(min_avg_max.group(1)),\n                            \"rtt_avg\": float(min_avg_max.group(2)),\n                            \"rtt_max\": float(min_avg_max.group(3)),\n                        }\n                    )\n                    results_array = []\n                    for i in range(probes_received):\n                        results_array.append(\n                            {\n                                \"ip_address\": py23_compat.text_type(destination),\n                                \"rtt\": 0.0,\n                            }\n                        )\n                    ping_dict[\"success\"].update({\"results\": results_array})\n\n        return ping_dict"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_ipv6_neighbors_table(self):\n\n        ipv6_neighbors_table = []\n        command = \"show ipv6 neighbors\"\n        output = self._send_command(command)\n\n        ipv6_neighbors = \"\"\n        fields = re.split(r\"^IPv6\\s+Address.*Interface$\", output, flags=(re.M | re.I))\n        if len(fields) == 2:\n            ipv6_neighbors = fields[1].strip()\n        for entry in ipv6_neighbors.splitlines():\n            # typical format of an entry in the IOS IPv6 neighbors table:\n            # 2002:FFFF:233::1 0 2894.0fed.be30  REACH Fa3/1/2.233\n            ip, age, mac, state, interface = entry.split()\n            mac = \"\" if mac == \"-\" else napalm.base.helpers.mac(mac)\n            ip = napalm.base.helpers.ip(ip)\n            ipv6_neighbors_table.append(\n                {\n                    \"interface\": interface,\n                    \"mac\": mac,\n                    \"ip\": ip,\n                    \"age\": float(age),\n                    \"state\": state,\n                }\n            )\n        return ipv6_neighbors_table", "response": "Get the IPv6 neighbors table information."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef open(self):\n        try:\n            connection = self.transport_class(\n                host=self.hostname,\n                username=self.username,\n                password=self.password,\n                timeout=self.timeout,\n                **self.eapi_kwargs\n            )\n\n            if self.device is None:\n                self.device = pyeapi.client.Node(connection, enablepwd=self.enablepwd)\n            # does not raise an Exception if unusable\n\n            # let's try to run a very simple command\n            self.device.run_commands([\"show clock\"], encoding=\"text\")\n        except ConnectionError as ce:\n            # and this is raised either if device not avaiable\n            # either if HTTP(S) agent is not enabled\n            # show management api http-commands\n            raise ConnectionException(py23_compat.text_type(ce))", "response": "Implementation of NAPALM method open."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nconvert running - config HEREDOC into EAPI JSON dict", "response": "def _multiline_convert(config, start=\"banner login\", end=\"EOF\", depth=1):\n        \"\"\"Converts running-config HEREDOC into EAPI JSON dict\"\"\"\n        ret = list(config)  # Don't modify list in-place\n        try:\n            s = ret.index(start)\n            e = s\n            while depth:\n                e = ret.index(end, e + 1)\n                depth = depth - 1\n        except ValueError:  # Couldn't find end, abort\n            return ret\n        ret[s] = {\"cmd\": ret[s], \"input\": \"\\n\".join(ret[s + 1 : e])}\n        del ret[s + 1 : e + 1]\n\n        return ret"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _mode_comment_convert(commands):\n\n        ret = []\n        comment_count = 0\n        for idx, element in enumerate(commands):\n            # Check first for stringiness, as we may have dicts in the command list already\n            if isinstance(element, py23_compat.string_types) and element.startswith(\n                \"!!\"\n            ):\n                comment_count += 1\n                continue\n            else:\n                if comment_count > 0:\n                    # append the previous comment\n                    ret.append(\n                        {\n                            \"cmd\": \"comment\",\n                            \"input\": \"\\n\".join(\n                                map(\n                                    lambda s: s.lstrip(\"! \"),\n                                    commands[idx - comment_count : idx],\n                                )\n                            ),\n                        }\n                    )\n                    comment_count = 0\n                ret.append(element)\n\n        return ret", "response": "Convert multi - line mode comments into a list of commands."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get_network_instances(self, name=\"\"):\n\n        output = self._show_vrf()\n        vrfs = {}\n        all_vrf_interfaces = {}\n        for vrf in output:\n            if (\n                vrf.get(\"route_distinguisher\", \"\") == \"<not set>\"\n                or vrf.get(\"route_distinguisher\", \"\") == \"None\"\n            ):\n                vrf[\"route_distinguisher\"] = \"\"\n            else:\n                vrf[\"route_distinguisher\"] = py23_compat.text_type(\n                    vrf[\"route_distinguisher\"]\n                )\n            interfaces = {}\n            for interface_raw in vrf.get(\"interfaces\", []):\n                interface = interface_raw.split(\",\")\n                for line in interface:\n                    if line.strip() != \"\":\n                        interfaces[py23_compat.text_type(line.strip())] = {}\n                        all_vrf_interfaces[py23_compat.text_type(line.strip())] = {}\n\n            vrfs[py23_compat.text_type(vrf[\"name\"])] = {\n                \"name\": py23_compat.text_type(vrf[\"name\"]),\n                \"type\": \"L3VRF\",\n                \"state\": {\"route_distinguisher\": vrf[\"route_distinguisher\"]},\n                \"interfaces\": {\"interface\": interfaces},\n            }\n        all_interfaces = self.get_interfaces_ip().keys()\n        vrfs[\"default\"] = {\n            \"name\": \"default\",\n            \"type\": \"DEFAULT_INSTANCE\",\n            \"state\": {\"route_distinguisher\": \"\"},\n            \"interfaces\": {\n                \"interface\": {\n                    k: {} for k in all_interfaces if k not in all_vrf_interfaces.keys()\n                }\n            },\n        }\n\n        if name:\n            if name in vrfs:\n                return {py23_compat.text_type(name): vrfs[name]}\n            return {}\n        else:\n            return vrfs", "response": "get_network_instances implementation for EOS."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nexecute ping on the device and return a dictionary with the result.", "response": "def ping(\n        self,\n        destination,\n        source=c.PING_SOURCE,\n        ttl=c.PING_TTL,\n        timeout=c.PING_TIMEOUT,\n        size=c.PING_SIZE,\n        count=c.PING_COUNT,\n        vrf=c.PING_VRF,\n    ):\n        \"\"\"\n        Execute ping on the device and returns a dictionary with the result.\n        Output dictionary has one of following keys:\n            * success\n            * error\n        In case of success, inner dictionary will have the followin keys:\n            * probes_sent (int)\n            * packet_loss (int)\n            * rtt_min (float)\n            * rtt_max (float)\n            * rtt_avg (float)\n            * rtt_stddev (float)\n            * results (list)\n        'results' is a list of dictionaries with the following keys:\n            * ip_address (str)\n            * rtt (float)\n        \"\"\"\n        ping_dict = {}\n        commands = []\n\n        if vrf:\n            commands.append(\"routing-context vrf {vrf}\".format(vrf=vrf))\n\n        command = \"ping {}\".format(destination)\n        command += \" timeout {}\".format(timeout)\n        command += \" size {}\".format(size)\n        command += \" repeat {}\".format(count)\n        if source != \"\":\n            command += \" source {}\".format(source)\n\n        commands.append(command)\n        output = self.device.run_commands(commands, encoding=\"text\")[-1][\"output\"]\n\n        if \"connect:\" in output:\n            ping_dict[\"error\"] = output\n        elif \"PING\" in output:\n            ping_dict[\"success\"] = {\n                \"probes_sent\": 0,\n                \"packet_loss\": 0,\n                \"rtt_min\": 0.0,\n                \"rtt_max\": 0.0,\n                \"rtt_avg\": 0.0,\n                \"rtt_stddev\": 0.0,\n                \"results\": [],\n            }\n            results_array = []\n            for line in output.splitlines():\n                fields = line.split()\n                if \"icmp\" in line:\n                    if \"Unreachable\" in line:\n                        if \"(\" in fields[2]:\n                            results_array.append(\n                                {\n                                    \"ip_address\": py23_compat.text_type(\n                                        fields[2][1:-1]\n                                    ),\n                                    \"rtt\": 0.0,\n                                }\n                            )\n                        else:\n                            results_array.append(\n                                {\n                                    \"ip_address\": py23_compat.text_type(fields[1]),\n                                    \"rtt\": 0.0,\n                                }\n                            )\n                    elif \"truncated\" in line:\n                        if \"(\" in fields[4]:\n                            results_array.append(\n                                {\n                                    \"ip_address\": py23_compat.text_type(\n                                        fields[4][1:-2]\n                                    ),\n                                    \"rtt\": 0.0,\n                                }\n                            )\n                        else:\n                            results_array.append(\n                                {\n                                    \"ip_address\": py23_compat.text_type(fields[3][:-1]),\n                                    \"rtt\": 0.0,\n                                }\n                            )\n                    elif fields[1] == \"bytes\":\n                        m = fields[6][5:]\n                        results_array.append(\n                            {\n                                \"ip_address\": py23_compat.text_type(fields[3][:-1]),\n                                \"rtt\": float(m),\n                            }\n                        )\n                elif \"packets transmitted\" in line:\n                    ping_dict[\"success\"][\"probes_sent\"] = int(fields[0])\n                    ping_dict[\"success\"][\"packet_loss\"] = int(fields[0]) - int(\n                        fields[3]\n                    )\n                elif \"min/avg/max\" in line:\n                    m = fields[3].split(\"/\")\n                    ping_dict[\"success\"].update(\n                        {\n                            \"rtt_min\": float(m[0]),\n                            \"rtt_avg\": float(m[1]),\n                            \"rtt_max\": float(m[2]),\n                            \"rtt_stddev\": float(m[3]),\n                        }\n                    )\n            ping_dict[\"success\"].update({\"results\": results_array})\n        return ping_dict"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef build_napalm_ansible_module_docs(app):\n\n    # Add script to clone napalm-ansible repo\n    status = subprocess.call(\n        \"./build-ansible-module-docs.sh\", stdout=sys.stdout, stderr=sys.stderr\n    )\n\n    if status != 0:\n        print(\"Something bad happened when processing the Ansible modules.\")\n        sys.exit(-1)\n\n    env = Environment(loader=FileSystemLoader(\".\"))\n\n    modules_dir = \"./integrations/ansible/modules/source\"\n    module_files = glob(\"{0}/*.json\".format(modules_dir))\n    for module_file in module_files:\n        with open(module_file, \"r\") as f:\n            module = module_file.split(\"/\")[-1].split(\".\")[0]\n            data = json.loads(f.read())\n            data[\"name\"] = module\n\n        module_dir = \"./integrations/ansible/modules/{0}\".format(module)\n\n        try:\n            os.stat(module_dir)\n        except Exception:\n            os.mkdir(module_dir)\n\n        template_file = env.get_template(\"ansible-module.j2\")\n        rendered_template = template_file.render(**data)\n\n        with open(\"{0}/index.rst\".format(module_dir), \"w\") as f:\n            f.write(rendered_template)", "response": "Build the Ansible modules documentation."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nbuilding the getters support matrix.", "response": "def build_getters_support_matrix(app):\n    \"\"\"Build the getters support matrix.\"\"\"\n    status = subprocess.call(\"./test.sh\", stdout=sys.stdout, stderr=sys.stderr)\n\n    if status != 0:\n        print(\"Something bad happened when processing the test reports.\")\n        sys.exit(-1)\n\n    drivers = set()\n    matrix = {\n        m: defaultdict(dict)\n        for m in dir(NetworkDriver)\n        if not (m.startswith(\"_\") or m in EXCLUDE_METHODS)\n    }\n\n    regex_name = re.compile(r\"(?P<driver>\\w+)\\/.*::test_(?P<getter>\\w+)\")\n\n    filename = \"./support/tests/report.json\"\n    with open(filename, \"r\") as f:\n        data = json.loads(f.read())\n        for test in data[\"report\"][\"tests\"]:\n            match = regex_name.search(test[\"name\"])\n            if match:\n                driver = match.group(\"driver\")\n                drivers.add(driver)\n                method = match.group(\"getter\")\n            else:\n                continue\n            if method in EXCLUDE_IN_REPORT:\n                continue\n            result = test[\"outcome\"]\n\n            if method in METHOD_ALIASES.keys():\n                method = METHOD_ALIASES[method]\n\n            intermediate_result = matrix[method].get(driver, None)\n            matrix[method][driver] = _merge_results(result, intermediate_result)\n\n    sorted_methods = sorted(matrix.keys())\n    drivers = sorted(drivers)\n    env = Environment(loader=FileSystemLoader(\".\"))\n    template_file = env.get_template(\"matrix.j2\")\n    rendered_template = template_file.render(\n        matrix=matrix, drivers=drivers, sorted_methods=sorted_methods\n    )\n\n    with open(\"support/matrix.rst\", \"w\") as f:\n        f.write(rendered_template)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get_bgp_neighbors(self):\n        def generate_vrf_query(vrf_name):\n            \"\"\"\n            Helper to provide XML-query for the VRF-type we're interested in.\n            \"\"\"\n            if vrf_name == \"global\":\n                rpc_command = \"<Get><Operational><BGP><InstanceTable><Instance><Naming>\\\n                <InstanceName>default</InstanceName></Naming><InstanceActive><DefaultVRF>\\\n                <GlobalProcessInfo></GlobalProcessInfo><NeighborTable></NeighborTable></DefaultVRF>\\\n                </InstanceActive></Instance></InstanceTable></BGP></Operational></Get>\"\n\n            else:\n                rpc_command = \"<Get><Operational><BGP><InstanceTable><Instance><Naming>\\\n                <InstanceName>default</InstanceName></Naming><InstanceActive><VRFTable><VRF>\\\n                <Naming>{vrf_name}</Naming><GlobalProcessInfo></GlobalProcessInfo><NeighborTable>\\\n                </NeighborTable></VRF></VRFTable></InstanceActive></Instance></InstanceTable>\\\n                </BGP></Operational></Get>\".format(\n                    vrf_name=vrf_name\n                )\n            return rpc_command\n\n        \"\"\"\n        Initial run to figure out what VRF's are available\n        Decided to get this one from Configured-section\n        because bulk-getting all instance-data to do the same could get ridiculously heavy\n        Assuming we're always interested in the DefaultVRF\n        \"\"\"\n\n        active_vrfs = [\"global\"]\n\n        rpc_command = \"<Get><Operational><BGP><ConfigInstanceTable><ConfigInstance><Naming>\\\n        <InstanceName>default</InstanceName></Naming><ConfigInstanceVRFTable>\\\n        </ConfigInstanceVRFTable></ConfigInstance></ConfigInstanceTable></BGP></Operational></Get>\"\n\n        result_tree = ETREE.fromstring(self.device.make_rpc_call(rpc_command))\n\n        for node in result_tree.xpath(\".//ConfigVRF\"):\n            active_vrfs.append(napalm.base.helpers.find_txt(node, \"Naming/VRFName\"))\n\n        result = {}\n\n        for vrf in active_vrfs:\n            rpc_command = generate_vrf_query(vrf)\n            result_tree = ETREE.fromstring(self.device.make_rpc_call(rpc_command))\n\n            this_vrf = {}\n            this_vrf[\"peers\"] = {}\n\n            if vrf == \"global\":\n                this_vrf[\"router_id\"] = napalm.base.helpers.convert(\n                    text_type,\n                    napalm.base.helpers.find_txt(\n                        result_tree,\n                        \"Get/Operational/BGP/InstanceTable/Instance/InstanceActive/DefaultVRF\"\n                        \"/GlobalProcessInfo/VRF/RouterID\",\n                    ),\n                )\n            else:\n                this_vrf[\"router_id\"] = napalm.base.helpers.convert(\n                    text_type,\n                    napalm.base.helpers.find_txt(\n                        result_tree,\n                        \"Get/Operational/BGP/InstanceTable/Instance/InstanceActive/VRFTable/VRF\"\n                        \"/GlobalProcessInfo/VRF/RouterID\",\n                    ),\n                )\n\n            neighbors = {}\n\n            for neighbor in result_tree.xpath(\".//Neighbor\"):\n                this_neighbor = {}\n                this_neighbor[\"local_as\"] = napalm.base.helpers.convert(\n                    int, napalm.base.helpers.find_txt(neighbor, \"LocalAS\")\n                )\n                this_neighbor[\"remote_as\"] = napalm.base.helpers.convert(\n                    int, napalm.base.helpers.find_txt(neighbor, \"RemoteAS\")\n                )\n                this_neighbor[\"remote_id\"] = napalm.base.helpers.convert(\n                    text_type, napalm.base.helpers.find_txt(neighbor, \"RouterID\")\n                )\n\n                if (\n                    napalm.base.helpers.find_txt(neighbor, \"ConnectionAdminStatus\")\n                    == \"1\"\n                ):\n                    this_neighbor[\"is_enabled\"] = True\n\n                try:\n                    this_neighbor[\"description\"] = napalm.base.helpers.convert(\n                        text_type, napalm.base.helpers.find_txt(neighbor, \"Description\")\n                    )\n                except AttributeError:\n                    this_neighbor[\"description\"] = \"\"\n\n                this_neighbor[\"is_enabled\"] = (\n                    napalm.base.helpers.find_txt(neighbor, \"ConnectionAdminStatus\")\n                    == \"1\"\n                )\n\n                if (\n                    text_type(\n                        napalm.base.helpers.find_txt(neighbor, \"ConnectionAdminStatus\")\n                    )\n                    == \"1\"\n                ):\n                    this_neighbor[\"is_enabled\"] = True\n                else:\n                    this_neighbor[\"is_enabled\"] = False\n\n                if (\n                    text_type(napalm.base.helpers.find_txt(neighbor, \"ConnectionState\"))\n                    == \"BGP_ST_ESTAB\"\n                ):\n                    this_neighbor[\"is_up\"] = True\n                    this_neighbor[\"uptime\"] = napalm.base.helpers.convert(\n                        int,\n                        napalm.base.helpers.find_txt(\n                            neighbor, \"ConnectionEstablishedTime\"\n                        ),\n                    )\n                else:\n                    this_neighbor[\"is_up\"] = False\n                    this_neighbor[\"uptime\"] = -1\n\n                this_neighbor[\"address_family\"] = {}\n\n                if (\n                    napalm.base.helpers.find_txt(\n                        neighbor, \"ConnectionRemoteAddress/AFI\"\n                    )\n                    == \"IPv4\"\n                ):\n                    this_afi = \"ipv4\"\n                elif (\n                    napalm.base.helpers.find_txt(\n                        neighbor, \"ConnectionRemoteAddress/AFI\"\n                    )\n                    == \"IPv6\"\n                ):\n                    this_afi = \"ipv6\"\n                else:\n                    this_afi = napalm.base.helpers.find_txt(\n                        neighbor, \"ConnectionRemoteAddress/AFI\"\n                    )\n\n                this_neighbor[\"address_family\"][this_afi] = {}\n\n                try:\n                    this_neighbor[\"address_family\"][this_afi][\n                        \"received_prefixes\"\n                    ] = napalm.base.helpers.convert(\n                        int,\n                        napalm.base.helpers.find_txt(\n                            neighbor, \"AFData/Entry/PrefixesAccepted\"\n                        ),\n                        0,\n                    ) + napalm.base.helpers.convert(\n                        int,\n                        napalm.base.helpers.find_txt(\n                            neighbor, \"AFData/Entry/PrefixesDenied\"\n                        ),\n                        0,\n                    )\n                    this_neighbor[\"address_family\"][this_afi][\n                        \"accepted_prefixes\"\n                    ] = napalm.base.helpers.convert(\n                        int,\n                        napalm.base.helpers.find_txt(\n                            neighbor, \"AFData/Entry/PrefixesAccepted\"\n                        ),\n                        0,\n                    )\n                    this_neighbor[\"address_family\"][this_afi][\n                        \"sent_prefixes\"\n                    ] = napalm.base.helpers.convert(\n                        int,\n                        napalm.base.helpers.find_txt(\n                            neighbor, \"AFData/Entry/PrefixesAdvertised\"\n                        ),\n                        0,\n                    )\n                except AttributeError:\n                    this_neighbor[\"address_family\"][this_afi][\"received_prefixes\"] = -1\n                    this_neighbor[\"address_family\"][this_afi][\"accepted_prefixes\"] = -1\n                    this_neighbor[\"address_family\"][this_afi][\"sent_prefixes\"] = -1\n\n                neighbor_ip = napalm.base.helpers.ip(\n                    napalm.base.helpers.find_txt(\n                        neighbor, \"Naming/NeighborAddress/IPV4Address\"\n                    )\n                    or napalm.base.helpers.find_txt(\n                        neighbor, \"Naming/NeighborAddress/IPV6Address\"\n                    )\n                )\n\n                neighbors[neighbor_ip] = this_neighbor\n\n            this_vrf[\"peers\"] = neighbors\n            result[vrf] = this_vrf\n\n        return result", "response": "This method returns a list of BGP neighbors for the current system."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef netmiko_args(optional_args):\n    fields = py23_compat.argspec(BaseConnection.__init__)\n    args = fields[0]\n    defaults = fields[3]\n\n    check_self = args.pop(0)\n    if check_self != \"self\":\n        raise ValueError(\"Error processing Netmiko arguments\")\n\n    netmiko_argument_map = dict(zip(args, defaults))\n\n    # Netmiko arguments that are integrated into NAPALM already\n    netmiko_filter = [\"ip\", \"host\", \"username\", \"password\", \"device_type\", \"timeout\"]\n\n    # Filter out all of the arguments that are integrated into NAPALM\n    for k in netmiko_filter:\n        netmiko_argument_map.pop(k)\n\n    # Check if any of these arguments were passed in as NAPALM optional_args\n    netmiko_optional_args = {}\n    for k, v in netmiko_argument_map.items():\n        try:\n            netmiko_optional_args[k] = optional_args[k]\n        except KeyError:\n            pass\n\n    # Return these arguments for use with establishing Netmiko SSH connection\n    return netmiko_optional_args", "response": "Check for Netmiko arguments that were passed in as NAPALM optional arguments."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef ensure_netmiko_conn(func):\n\n    def wrap_function(self, filename=None, config=None):\n        try:\n            netmiko_object = self._netmiko_device\n            if netmiko_object is None:\n                raise AttributeError()\n        except AttributeError:\n            device_type = c.NETMIKO_MAP[self.platform]\n            netmiko_optional_args = self.netmiko_optional_args\n            if \"port\" in netmiko_optional_args:\n                netmiko_optional_args[\"port\"] = 22\n            self._netmiko_open(\n                device_type=device_type, netmiko_optional_args=netmiko_optional_args\n            )\n        func(self, filename=filename, config=config)\n\n    return wrap_function", "response": "Decorator that ensures Netmiko connection exists."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn the merge diff of the current configuration and the merge candidate.", "response": "def _get_merge_diff(self):\n        \"\"\"\n        The merge diff is not necessarily what needs to be loaded\n        for example under NTP, even though the 'ntp commit' command might be\n        alread configured, it is mandatory to be sent\n        otherwise it won't take the new configuration - see:\n        https://github.com/napalm-automation/napalm-nxos/issues/59\n        therefore this method will return the real diff (but not necessarily what is\n        being sent by the merge_load_config()\n        \"\"\"\n        diff = []\n        running_config = self.get_config(retrieve=\"running\")[\"running\"]\n        running_lines = running_config.splitlines()\n        for line in self.merge_candidate.splitlines():\n            if line not in running_lines and line:\n                if line[0].strip() != \"!\":\n                    diff.append(line)\n        return \"\\n\".join(diff)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _get_diff(self):\n        diff = []\n        self._create_sot_file()\n        diff_out = self._send_command(\n            \"show diff rollback-patch file {} file {}\".format(\n                \"sot_file\", self.candidate_cfg\n            ),\n            raw_text=True,\n        )\n        try:\n            diff_out = (\n                diff_out.split(\"Generating Rollback Patch\")[1]\n                .replace(\"Rollback Patch is Empty\", \"\")\n                .strip()\n            )\n            for line in diff_out.splitlines():\n                if line:\n                    if line[0].strip() != \"!\" and line[0].strip() != \".\":\n                        diff.append(line.rstrip(\" \"))\n        except (AttributeError, KeyError):\n            raise ReplaceConfigException(\n                \"Could not calculate diff. It's possible the given file doesn't exist.\"\n            )\n        return \"\\n\".join(diff)", "response": "Get a diff between running config and a proposed file."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _create_sot_file(self):\n\n        # Bug on on NX-OS 6.2.16 where overwriting sot_file would take exceptionally long time\n        # (over 12 minutes); so just delete the sot_file\n        try:\n            self._delete_file(filename=\"sot_file\")\n        except Exception:\n            pass\n        commands = [\n            \"terminal dont-ask\",\n            \"checkpoint file sot_file\",\n            \"no terminal dont-ask\",\n        ]\n        self._send_command_list(commands)", "response": "Create Source of Truth file to compare."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef ping(\n        self,\n        destination,\n        source=c.PING_SOURCE,\n        ttl=c.PING_TTL,\n        timeout=c.PING_TIMEOUT,\n        size=c.PING_SIZE,\n        count=c.PING_COUNT,\n        vrf=c.PING_VRF,\n    ):\n        \"\"\"\n        Execute ping on the device and returns a dictionary with the result.\n        Output dictionary has one of following keys:\n            * success\n            * error\n        In case of success, inner dictionary will have the followin keys:\n            * probes_sent (int)\n            * packet_loss (int)\n            * rtt_min (float)\n            * rtt_max (float)\n            * rtt_avg (float)\n            * rtt_stddev (float)\n            * results (list)\n        'results' is a list of dictionaries with the following keys:\n            * ip_address (str)\n            * rtt (float)\n        \"\"\"\n        ping_dict = {}\n\n        version = \"\"\n        try:\n            version = \"6\" if IPAddress(destination).version == 6 else \"\"\n        except AddrFormatError:\n            # Allow use of DNS names\n            pass\n\n        command = \"ping{version} {destination}\".format(\n            version=version, destination=destination\n        )\n        command += \" timeout {}\".format(timeout)\n        command += \" packet-size {}\".format(size)\n        command += \" count {}\".format(count)\n        if source != \"\":\n            command += \" source {}\".format(source)\n\n        if vrf != \"\":\n            command += \" vrf {}\".format(vrf)\n        output = self._send_command(command, raw_text=True)\n\n        if \"connect:\" in output:\n            ping_dict[\"error\"] = output\n        elif \"PING\" in output:\n            ping_dict[\"success\"] = {\n                \"probes_sent\": 0,\n                \"packet_loss\": 0,\n                \"rtt_min\": 0.0,\n                \"rtt_max\": 0.0,\n                \"rtt_avg\": 0.0,\n                \"rtt_stddev\": 0.0,\n                \"results\": [],\n            }\n            results_array = []\n            for line in output.splitlines():\n                fields = line.split()\n                if \"icmp\" in line:\n                    if \"Unreachable\" in line:\n                        if \"(\" in fields[2]:\n                            results_array.append(\n                                {\n                                    \"ip_address\": py23_compat.text_type(\n                                        fields[2][1:-1]\n                                    ),\n                                    \"rtt\": 0.0,\n                                }\n                            )\n                        else:\n                            results_array.append(\n                                {\n                                    \"ip_address\": py23_compat.text_type(fields[1]),\n                                    \"rtt\": 0.0,\n                                }\n                            )\n                    elif \"truncated\" in line:\n                        if \"(\" in fields[4]:\n                            results_array.append(\n                                {\n                                    \"ip_address\": py23_compat.text_type(\n                                        fields[4][1:-2]\n                                    ),\n                                    \"rtt\": 0.0,\n                                }\n                            )\n                        else:\n                            results_array.append(\n                                {\n                                    \"ip_address\": py23_compat.text_type(fields[3][:-1]),\n                                    \"rtt\": 0.0,\n                                }\n                            )\n                    elif fields[1] == \"bytes\":\n                        if version == \"6\":\n                            m = fields[5][5:]\n                        else:\n                            m = fields[6][5:]\n                        results_array.append(\n                            {\n                                \"ip_address\": py23_compat.text_type(fields[3][:-1]),\n                                \"rtt\": float(m),\n                            }\n                        )\n                elif \"packets transmitted\" in line:\n                    ping_dict[\"success\"][\"probes_sent\"] = int(fields[0])\n                    ping_dict[\"success\"][\"packet_loss\"] = int(fields[0]) - int(\n                        fields[3]\n                    )\n                elif \"min/avg/max\" in line:\n                    m = fields[3].split(\"/\")\n                    ping_dict[\"success\"].update(\n                        {\n                            \"rtt_min\": float(m[0]),\n                            \"rtt_avg\": float(m[1]),\n                            \"rtt_max\": float(m[2]),\n                        }\n                    )\n            ping_dict[\"success\"].update({\"results\": results_array})\n        return ping_dict", "response": "Execute ping on the device and return the result."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_lldp_neighbors(self):\n        lldp = {}\n        neighbors_detail = self.get_lldp_neighbors_detail()\n        for intf_name, entries in neighbors_detail.items():\n            lldp[intf_name] = []\n            for lldp_entry in entries:\n                hostname = lldp_entry[\"remote_system_name\"]\n                # Match IOS behaviour of taking remote chassis ID\n                # When lacking a system name (in show lldp neighbors)\n                if hostname == \"N/A\":\n                    hostname = lldp_entry[\"remote_chassis_id\"]\n                lldp_dict = {\"port\": lldp_entry[\"remote_port\"], \"hostname\": hostname}\n                lldp[intf_name].append(lldp_dict)\n\n        return lldp", "response": "IOS implementation of get_lldp_neighbors."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _send_command(self, command, raw_text=False):\n        return self.device.show(command, raw_text=raw_text)", "response": "Wrapper for NX - API show method."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncomputes the timestamp from the output of a cisco event.", "response": "def _compute_timestamp(stupid_cisco_output):\n        \"\"\"\n        Some fields such `uptime` are returned as: 23week(s) 3day(s)\n        This method will determine the epoch of the event.\n        e.g.: 23week(s) 3day(s) -> 1462248287\n        \"\"\"\n        if not stupid_cisco_output or stupid_cisco_output == \"never\":\n            return -1.0\n\n        if \"(s)\" in stupid_cisco_output:\n            pass\n        elif \":\" in stupid_cisco_output:\n            stupid_cisco_output = stupid_cisco_output.replace(\":\", \"hour(s) \", 1)\n            stupid_cisco_output = stupid_cisco_output.replace(\":\", \"minute(s) \", 1)\n            stupid_cisco_output += \"second(s)\"\n        else:\n            stupid_cisco_output = stupid_cisco_output.replace(\"d\", \"day(s) \")\n            stupid_cisco_output = stupid_cisco_output.replace(\"h\", \"hour(s)\")\n\n        things = {\n            \"second(s)\": {\"weight\": 1},\n            \"minute(s)\": {\"weight\": 60},\n            \"hour(s)\": {\"weight\": 3600},\n            \"day(s)\": {\"weight\": 24 * 3600},\n            \"week(s)\": {\"weight\": 7 * 24 * 3600},\n            \"year(s)\": {\"weight\": 365.25 * 24 * 3600},\n        }\n\n        things_keys = things.keys()\n        for part in stupid_cisco_output.split():\n            for key in things_keys:\n                if key in part:\n                    things[key][\"count\"] = napalm.base.helpers.convert(\n                        int, part.replace(key, \"\"), 0\n                    )\n\n        delta = sum(\n            [det.get(\"count\", 0) * det.get(\"weight\") for det in things.values()]\n        )\n        return time.time() - delta"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ngets the rows of a table.", "response": "def _get_table_rows(parent_table, table_name, row_name):\n        \"\"\"\n        Inconsistent behavior:\n        {'TABLE_intf': [{'ROW_intf': {\n        vs\n        {'TABLE_mac_address': {'ROW_mac_address': [{\n        vs\n        {'TABLE_vrf': {'ROW_vrf': {'TABLE_adj': {'ROW_adj': {\n        \"\"\"\n        if parent_table is None:\n            return []\n        _table = parent_table.get(table_name)\n        _table_rows = []\n        if isinstance(_table, list):\n            _table_rows = [_table_row.get(row_name) for _table_row in _table]\n        elif isinstance(_table, dict):\n            _table_rows = _table.get(row_name)\n        if not isinstance(_table_rows, list):\n            _table_rows = [_table_rows]\n        return _table_rows"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nget BGP neighbors from NX - API.", "response": "def get_bgp_neighbors(self):\n        results = {}\n        bgp_state_dict = {\n            \"Idle\": {\"is_up\": False, \"is_enabled\": True},\n            \"Active\": {\"is_up\": False, \"is_enabled\": True},\n            \"Open\": {\"is_up\": False, \"is_enabled\": True},\n            \"Established\": {\"is_up\": True, \"is_enabled\": True},\n            \"Closing\": {\"is_up\": True, \"is_enabled\": True},\n            \"Shutdown\": {\"is_up\": False, \"is_enabled\": False},\n        }\n        \"\"\"\n        af_name_dict = {\n            'af-id': {'safi': \"af-name\"},\n            'af-id': {'safi': \"af-name\"},\n            'af-id': {'safi': \"af-name\"}\n        }\n        \"\"\"\n        af_name_dict = {\n            1: {1: \"ipv4\", 128: \"vpnv4\"},\n            2: {1: \"ipv6\", 128: \"vpnv6\"},\n            25: {70: \"l2vpn\"},\n        }\n\n        try:\n            cmd = \"show bgp all summary vrf all\"\n            vrf_list = self._get_command_table(cmd, \"TABLE_vrf\", \"ROW_vrf\")\n        except NXAPICommandError:\n            vrf_list = []\n\n        for vrf_dict in vrf_list:\n            result_vrf_dict = {\n                \"router_id\": py23_compat.text_type(vrf_dict[\"vrf-router-id\"]),\n                \"peers\": {},\n            }\n\n            af_list = vrf_dict.get(\"TABLE_af\", {}).get(\"ROW_af\", [])\n            if isinstance(af_list, dict):\n                af_list = [af_list]\n\n            for af_dict in af_list:\n                saf_dict = af_dict.get(\"TABLE_saf\", {}).get(\"ROW_saf\", {})\n                neighbors_list = saf_dict.get(\"TABLE_neighbor\", {}).get(\n                    \"ROW_neighbor\", []\n                )\n\n                if isinstance(neighbors_list, dict):\n                    neighbors_list = [neighbors_list]\n\n                for neighbor_dict in neighbors_list:\n                    neighborid = napalm.base.helpers.ip(neighbor_dict[\"neighborid\"])\n                    remoteas = napalm.base.helpers.as_number(\n                        neighbor_dict[\"neighboras\"]\n                    )\n                    state = py23_compat.text_type(neighbor_dict[\"state\"])\n\n                    bgp_state = bgp_state_dict[state]\n                    afid_dict = af_name_dict[int(af_dict[\"af-id\"])]\n                    safi_name = afid_dict[int(saf_dict[\"safi\"])]\n\n                    result_peer_dict = {\n                        \"local_as\": int(vrf_dict[\"vrf-local-as\"]),\n                        \"remote_as\": remoteas,\n                        \"remote_id\": neighborid,\n                        \"is_enabled\": bgp_state[\"is_enabled\"],\n                        \"uptime\": -1,\n                        \"description\": \"\",\n                        \"is_up\": bgp_state[\"is_up\"],\n                        \"address_family\": {\n                            safi_name: {\n                                \"sent_prefixes\": -1,\n                                \"accepted_prefixes\": -1,\n                                \"received_prefixes\": int(\n                                    neighbor_dict[\"prefixreceived\"]\n                                ),\n                            }\n                        },\n                    }\n                    result_vrf_dict[\"peers\"][neighborid] = result_peer_dict\n\n            vrf_name = vrf_dict[\"vrf-name-out\"]\n            if vrf_name == \"default\":\n                vrf_name = \"global\"\n            results[vrf_name] = result_vrf_dict\n        return results"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_network_instances(self, name=\"\"):\n\n        # command 'show vrf detail' returns all VRFs with detailed information\n        # format: list of dictionaries with keys such as 'vrf_name' and 'rd'\n        command = \"show vrf detail\"\n        vrf_table_raw = self._get_command_table(command, \"TABLE_vrf\", \"ROW_vrf\")\n\n        # command 'show vrf interface' returns all interfaces including their assigned VRF\n        # format: list of dictionaries with keys 'if_name', 'vrf_name', 'vrf_id' and 'soo'\n        command = \"show vrf interface\"\n        intf_table_raw = self._get_command_table(command, \"TABLE_if\", \"ROW_if\")\n\n        # create a dictionary with key = 'vrf_name' and value = list of interfaces\n        vrf_intfs = defaultdict(list)\n        for intf in intf_table_raw:\n            vrf_intfs[intf[\"vrf_name\"]].append(py23_compat.text_type(intf[\"if_name\"]))\n\n        vrfs = {}\n        for vrf in vrf_table_raw:\n            vrf_name = py23_compat.text_type(vrf.get(\"vrf_name\"))\n            vrfs[vrf_name] = {}\n            vrfs[vrf_name][\"name\"] = vrf_name\n\n            # differentiate between VRF type 'DEFAULT_INSTANCE' and 'L3VRF'\n            if vrf_name == \"default\":\n                vrfs[vrf_name][\"type\"] = \"DEFAULT_INSTANCE\"\n            else:\n                vrfs[vrf_name][\"type\"] = \"L3VRF\"\n\n            vrfs[vrf_name][\"state\"] = {\n                \"route_distinguisher\": py23_compat.text_type(vrf.get(\"rd\"))\n            }\n\n            # convert list of interfaces (vrf_intfs[vrf_name]) to expected format\n            # format = dict with key = interface name and empty values\n            vrfs[vrf_name][\"interfaces\"] = {}\n            vrfs[vrf_name][\"interfaces\"][\"interface\"] = dict.fromkeys(\n                vrf_intfs[vrf_name], {}\n            )\n\n        # if name of a specific VRF was passed as an argument\n        # only return results for this particular VRF\n        if name:\n            if name in vrfs.keys():\n                return {py23_compat.text_type(name): vrfs[name]}\n            else:\n                return {}\n        # else return results for all VRFs\n        else:\n            return vrfs", "response": "get_network_instances implementation for NX - OS"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns the first class derived from NetworkDriver in a specific library.", "response": "def get_network_driver(name, prepend=True):\n    \"\"\"\n    Searches for a class derived form the base NAPALM class NetworkDriver in a specific library.\n    The library name must repect the following pattern: napalm_[DEVICE_OS].\n    NAPALM community supports a list of devices and provides the corresponding libraries; for\n    full reference please refer to the `Supported Network Operation Systems`_ paragraph on\n    `Read the Docs`_.\n\n    .. _`Supported Network Operation Systems`: \\\n    http://napalm.readthedocs.io/en/latest/#supported-network-operating-systems\n    .. _`Read the Docs`: \\\n    http://napalm.readthedocs.io/\n\n    :param name:         the name of the device operating system or the name of the library.\n    :return:                    the first class derived from NetworkDriver, found in the library.\n    :raise ModuleImportError:   when the library is not installed or a derived class from \\\n    NetworkDriver was not found.\n\n    Example::\n\n    .. code-block:: python\n\n        >>> get_network_driver('junos')\n        <class 'napalm.junos.junos.JunOSDriver'>\n        >>> get_network_driver('IOS-XR')\n        <class 'napalm.iosxr.iosxr.IOSXRDriver'>\n        >>> get_network_driver('napalm.eos')\n        <class 'napalm.eos.eos.EOSDriver'>\n        >>> get_network_driver('wrong')\n        napalm.base.exceptions.ModuleImportError: Cannot import \"napalm_wrong\". Is the library \\\n        installed?\n    \"\"\"\n    if name == \"mock\":\n        return MockDriver\n\n    if not (isinstance(name, py23_compat.string_types) and len(name) > 0):\n        raise ModuleImportError(\"Please provide a valid driver name.\")\n\n    # Only lowercase allowed\n    name = name.lower()\n    # Try to not raise error when users requests IOS-XR for e.g.\n    module_install_name = name.replace(\"-\", \"\")\n    community_install_name = \"napalm_{name}\".format(name=module_install_name)\n    custom_install_name = \"custom_napalm.{name}\".format(name=module_install_name)\n    # Can also request using napalm_[SOMETHING]\n    if \"napalm\" not in module_install_name and prepend is True:\n        module_install_name = \"napalm.{name}\".format(name=module_install_name)\n    # Order is custom_napalm_os (local only) ->  napalm.os (core) ->  napalm_os (community)\n    for module_name in [\n        custom_install_name,\n        module_install_name,\n        community_install_name,\n    ]:\n        try:\n            module = importlib.import_module(module_name)\n            break\n        except ImportError as e:\n            message = py23_compat.text_type(e)\n            if \"No module named\" in message:\n                # py2 doesn't have ModuleNotFoundError exception\n                failed_module = message.split()[-1]\n                if failed_module.replace(\"'\", \"\") in module_name:\n                    continue\n            raise e\n    else:\n        raise ModuleImportError(\n            'Cannot import \"{install_name}\". Is the library installed?'.format(\n                install_name=name\n            )\n        )\n\n    for name, obj in inspect.getmembers(module):\n        if inspect.isclass(obj) and issubclass(obj, NetworkDriver):\n            return obj\n\n    # looks like you don't have any Driver class in your module...\n    raise ModuleImportError(\n        'No class inheriting \"napalm.base.base.NetworkDriver\" found in \"{install_name}\".'.format(\n            install_name=module_install_name\n        )\n    )"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef main(config_file):\n\n    if not (os.path.exists(config_file) and os.path.isfile(config_file)):\n        msg = \"Missing or invalid config file {0}\".format(config_file)\n        raise ValueError(msg)\n\n    print(\"Loading config file {0}.\".format(config_file))\n\n    # Use the appropriate network driver to connect to the device:\n    driver = napalm.get_network_driver(\"eos\")\n\n    # Connect:\n    device = driver(\n        hostname=\"127.0.0.1\",\n        username=\"vagrant\",\n        password=\"vagrant\",\n        optional_args={\"port\": 12443},\n    )\n\n    print(\"Opening ...\")\n    device.open()\n\n    print(\"Loading replacement candidate ...\")\n    device.load_replace_candidate(filename=config_file)\n\n    # Note that the changes have not been applied yet. Before applying\n    # the configuration you can check the changes:\n    print(\"\\nDiff:\")\n    print(device.compare_config())\n\n    # You can commit or discard the candidate changes.\n    try:\n        choice = raw_input(\"\\nWould you like to commit these changes? [yN]: \")\n    except NameError:\n        choice = input(\"\\nWould you like to commit these changes? [yN]: \")\n    if choice == \"y\":\n        print(\"Committing ...\")\n        device.commit_config()\n    else:\n        print(\"Discarding ...\")\n        device.discard_config()\n\n    # close the session with the device.\n    device.close()\n    print(\"Done.\")", "response": "Load a config for the device."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns whether an OC attribute has been defined or not.", "response": "def oc_attr_isdefault(o):\n    \"\"\"Return wether an OC attribute has been defined or not.\"\"\"\n    if not o._changed() and not o.default():\n        return True\n    if o == o.default():\n        return True\n    return False"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef Column(self, column_name):\n    column_idx = None\n    for idx, column in enumerate(self.header.columns):\n      if column.name == column_name:\n        column_idx = idx\n        break\n\n    if column_idx is None:\n      raise KeyError(\"Column '{}' not found\".format(column_name))\n\n    for row in self.rows:\n      yield row.values[column_idx]", "response": "Iterates over values of a given column."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncreate n clients to run in a pool.", "response": "def CreateClientPool(n):\n  \"\"\"Create n clients to run in a pool.\"\"\"\n  clients = []\n\n  # Load previously stored clients.\n  try:\n    certificates = []\n    with open(flags.FLAGS.cert_file, \"rb\") as fd:\n      # Certificates are base64-encoded, so that we can use new-lines as\n      # separators.\n      for l in fd:\n        cert = rdf_crypto.RSAPrivateKey(initializer=base64.b64decode(l))\n        certificates.append(cert)\n\n    for certificate in certificates[:n]:\n      clients.append(\n          PoolGRRClient(\n              private_key=certificate,\n              ca_cert=config.CONFIG[\"CA.certificate\"],\n              fast_poll=flags.FLAGS.fast_poll,\n              send_foreman_request=flags.FLAGS.send_foreman_request,\n          ))\n\n    clients_loaded = True\n  except (IOError, EOFError):\n    clients_loaded = False\n\n  if clients_loaded and len(clients) < n:\n    raise RuntimeError(\n        \"Loaded %d clients, but expected %d.\" % (len(clients), n))\n\n  while len(clients) < n:\n    # Generate a new RSA key pair for each client.\n    bits = config.CONFIG[\"Client.rsa_key_length\"]\n    key = rdf_crypto.RSAPrivateKey.GenerateKey(bits=bits)\n    clients.append(\n        PoolGRRClient(private_key=key, ca_cert=config.CONFIG[\"CA.certificate\"]))\n\n  # Start all the clients now.\n  for c in clients:\n    c.start()\n\n  start_time = time.time()\n  try:\n    if flags.FLAGS.enroll_only:\n      while True:\n        time.sleep(1)\n        enrolled = len([x for x in clients if x.enrolled])\n\n        if enrolled == n:\n          logging.info(\"All clients enrolled, exiting.\")\n          break\n\n        else:\n          logging.info(\"%s: Enrolled %d/%d clients.\", int(time.time()),\n                       enrolled, n)\n    else:\n      try:\n        while True:\n          time.sleep(100)\n      except KeyboardInterrupt:\n        pass\n\n  finally:\n    # Stop all pool clients.\n    for cl in clients:\n      cl.Stop()\n\n  # Note: code below is going to be executed after SIGTERM is sent to this\n  # process.\n  logging.info(\"Pool done in %s seconds.\", time.time() - start_time)\n\n  # The way benchmarking is supposed to work is that we execute poolclient with\n  # --enroll_only flag, it dumps the certificates to the flags.FLAGS.cert_file.\n  # Then, all further poolclient invocations just read private keys back\n  # from that file. Therefore if private keys were loaded from\n  # flags.FLAGS.cert_file, then there's no need to rewrite it again with the\n  # same data.\n  if not clients_loaded:\n    logging.info(\"Saving certificates.\")\n    with open(flags.FLAGS.cert_file, \"wb\") as fd:\n      # We're base64-encoding ceritificates so that we can use new-lines\n      # as separators.\n      b64_certs = [\n          base64.b64encode(x.private_key.SerializeToString()) for x in clients\n      ]\n      fd.write(\"\\n\".join(b64_certs))"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ncheck that the poolclient is not accidentally ran against production.", "response": "def CheckLocation():\n  \"\"\"Checks that the poolclient is not accidentally ran against production.\"\"\"\n  for url in (config.CONFIG[\"Client.server_urls\"] +\n              config.CONFIG[\"Client.control_urls\"]):\n    if \"staging\" in url or \"localhost\" in url:\n      # This is ok.\n      return\n  logging.error(\"Poolclient should only be run against test or staging.\")\n  exit()"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _ProcessMessageHandlerRequests(self, requests):\n    logging.debug(\"Leased message handler request ids: %s\",\n                  \",\".join(str(r.request_id) for r in requests))\n    grouped_requests = collection.Group(requests, lambda r: r.handler_name)\n    for handler_name, requests_for_handler in iteritems(grouped_requests):\n      handler_cls = handler_registry.handler_name_map.get(handler_name)\n      if not handler_cls:\n        logging.error(\"Unknown message handler: %s\", handler_name)\n        continue\n\n      stats_collector_instance.Get().IncrementCounter(\n          \"well_known_flow_requests\", fields=[handler_name])\n\n      try:\n        logging.debug(\"Running %d messages for handler %s\",\n                      len(requests_for_handler), handler_name)\n        handler_cls(token=self.token).ProcessMessages(requests_for_handler)\n      except Exception as e:  # pylint: disable=broad-except\n        logging.exception(\"Exception while processing message handler %s: %s\",\n                          handler_name, e)\n\n    logging.debug(\"Deleting message handler request ids: %s\",\n                  \",\".join(str(r.request_id) for r in requests))\n    data_store.REL_DB.DeleteMessageHandlerRequests(requests)", "response": "Processes the given message handler requests."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nprocesses one set of messages from Task Scheduler. The worker processes new jobs from the task master. For each job we retrieve the session from the Task Scheduler. Returns: Total number of messages processed by this call.", "response": "def RunOnce(self):\n    \"\"\"Processes one set of messages from Task Scheduler.\n\n    The worker processes new jobs from the task master. For each job\n    we retrieve the session from the Task Scheduler.\n\n    Returns:\n        Total number of messages processed by this call.\n    \"\"\"\n    start_time = time.time()\n    processed = 0\n\n    queue_manager = queue_manager_lib.QueueManager(token=self.token)\n    for queue in self.queues:\n      # Freezeing the timestamp used by queue manager to query/delete\n      # notifications to avoid possible race conditions.\n      queue_manager.FreezeTimestamp()\n\n      fetch_messages_start = time.time()\n      notifications = queue_manager.GetNotifications(queue)\n      stats_collector_instance.Get().RecordEvent(\n          \"worker_time_to_retrieve_notifications\",\n          time.time() - fetch_messages_start)\n\n      stuck_flows = []\n      for n in notifications:\n        if n.in_progress:\n          stuck_flows.append(n)\n\n      # Process stuck flows first\n      if stuck_flows:\n        self.ProcessStuckFlows(stuck_flows, queue_manager)\n\n      notifications_available = []\n      for notification in notifications:\n        # Filter out session ids we already tried to lock but failed.\n        if notification.session_id not in self.queued_flows:\n          notifications_available.append(notification)\n\n      try:\n        # If we spent too much time processing what we have so far, the\n        # active_sessions list might not be current. We therefore break here\n        # so we can re-fetch a more up to date version of the list, and try\n        # again later. The risk with running with an old active_sessions list\n        # is that another worker could have already processed this message,\n        # and when we try to process it, there is nothing to do - costing us a\n        # lot of processing time. This is a tradeoff between checking the data\n        # store for current information and processing out of date\n        # information.\n        processed += self.ProcessMessages(\n            notifications_available, queue_manager,\n            self.RUN_ONCE_MAX_SECONDS - (time.time() - start_time))\n\n      # We need to keep going no matter what.\n      except Exception as e:  # pylint: disable=broad-except\n        logging.error(\"Error processing message %s. %s.\", e,\n                      traceback.format_exc())\n        stats_collector_instance.Get().IncrementCounter(\"grr_worker_exceptions\")\n        if flags.FLAGS.pdb_post_mortem:\n          pdb.post_mortem()\n\n      queue_manager.UnfreezeTimestamp()\n      # If we have spent too much time, stop.\n      if (time.time() - start_time) > self.RUN_ONCE_MAX_SECONDS:\n        return processed\n    return processed"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nprocesses all the messages in the active_notifications queue_manager.", "response": "def ProcessMessages(self, active_notifications, queue_manager, time_limit=0):\n    \"\"\"Processes all the flows in the messages.\n\n    Precondition: All tasks come from the same queue.\n\n    Note that the server actually completes the requests in the\n    flow when receiving the messages from the client. We do not really\n    look at the messages here at all any more - we just work from the\n    completed messages in the flow RDFValue.\n\n    Args:\n        active_notifications: The list of notifications.\n        queue_manager: QueueManager object used to manage notifications,\n          requests and responses.\n        time_limit: If set return as soon as possible after this many seconds.\n\n    Returns:\n        The number of processed flows.\n    \"\"\"\n    now = time.time()\n    processed = 0\n    for notification in active_notifications:\n      if notification.session_id not in self.queued_flows:\n        if time_limit and time.time() - now > time_limit:\n          break\n\n        processed += 1\n        self.queued_flows.Put(notification.session_id, 1)\n        self.thread_pool.AddTask(\n            target=self._ProcessMessages,\n            args=(notification, queue_manager.Copy()),\n            name=self.__class__.__name__)\n\n    return processed"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _ProcessRegularFlowMessages(self, flow_obj, notification):\n    session_id = notification.session_id\n    if not isinstance(flow_obj, flow.FlowBase):\n      logging.warning(\"%s is not a proper flow object (got %s)\", session_id,\n                      type(flow_obj))\n\n      stats_collector_instance.Get().IncrementCounter(\n          \"worker_bad_flow_objects\", fields=[str(type(flow_obj))])\n      raise FlowProcessingError(\"Not a GRRFlow.\")\n\n    runner = flow_obj.GetRunner()\n    try:\n      runner.ProcessCompletedRequests(notification, self.thread_pool)\n    except Exception as e:  # pylint: disable=broad-except\n      # Something went wrong - log it in the flow.\n      runner.context.state = rdf_flow_runner.FlowContext.State.ERROR\n      runner.context.backtrace = traceback.format_exc()\n      logging.error(\"Flow %s: %s\", flow_obj, e)\n      raise FlowProcessingError(e)", "response": "Processes messages for a given flow."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _ProcessMessages(self, notification, queue_manager):\n    flow_obj = None\n    session_id = notification.session_id\n\n    try:\n      # Take a lease on the flow:\n      flow_name = session_id.FlowName()\n      if flow_name in self.well_known_flows:\n        # Well known flows are not necessarily present in the data store so\n        # we need to create them instead of opening.\n        expected_flow = self.well_known_flows[flow_name].__class__\n        flow_obj = aff4.FACTORY.CreateWithLock(\n            session_id,\n            expected_flow,\n            lease_time=self.well_known_flow_lease_time,\n            blocking=False,\n            token=self.token)\n      else:\n        flow_obj = aff4.FACTORY.OpenWithLock(\n            session_id,\n            lease_time=self.flow_lease_time,\n            blocking=False,\n            token=self.token)\n\n      now = time.time()\n      logging.debug(\"Got lock on %s\", session_id)\n\n      # If we get here, we now own the flow. We can delete the notifications\n      # we just retrieved but we need to make sure we don't delete any that\n      # came in later.\n      queue_manager.DeleteNotification(session_id, end=notification.timestamp)\n\n      if flow_name in self.well_known_flows:\n        stats_collector_instance.Get().IncrementCounter(\n            \"well_known_flow_requests\", fields=[str(session_id)])\n\n        # We remove requests first and then process them in the thread pool.\n        # On one hand this approach increases the risk of losing requests in\n        # case the worker process dies. On the other hand, it doesn't hold\n        # the lock while requests are processed, so other workers can\n        # process well known flows requests as well.\n        with flow_obj:\n          responses = flow_obj.FetchAndRemoveRequestsAndResponses(session_id)\n\n        flow_obj.ProcessResponses(responses, self.thread_pool)\n\n      else:\n        with flow_obj:\n          self._ProcessRegularFlowMessages(flow_obj, notification)\n\n      elapsed = time.time() - now\n      logging.debug(\"Done processing %s: %s sec\", session_id, elapsed)\n      stats_collector_instance.Get().RecordEvent(\n          \"worker_flow_processing_time\", elapsed, fields=[flow_obj.Name()])\n\n      # Everything went well -> session can be run again.\n      self.queued_flows.ExpireObject(session_id)\n\n    except aff4.LockError:\n      # Another worker is dealing with this flow right now, we just skip it.\n      # We expect lots of these when there are few messages (the system isn't\n      # highly loaded) but it is interesting when the system is under load to\n      # know if we are pulling the optimal number of messages off the queue.\n      # A high number of lock fails when there is plenty of work to do would\n      # indicate we are wasting time trying to process work that has already\n      # been completed by other workers.\n      stats_collector_instance.Get().IncrementCounter(\"worker_flow_lock_error\")\n\n    except FlowProcessingError:\n      # Do nothing as we expect the error to be correctly logged and accounted\n      # already.\n      pass\n\n    except Exception as e:  # pylint: disable=broad-except\n      # Something went wrong when processing this session. In order not to spin\n      # here, we just remove the notification.\n      logging.exception(\"Error processing session %s: %s\", session_id, e)\n      stats_collector_instance.Get().IncrementCounter(\n          \"worker_session_errors\", fields=[str(type(e))])\n      queue_manager.DeleteNotification(session_id)", "response": "Does the real work with a single flow."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef WaitUntilDone(self, timeout=None):\n\n    utils.Poll(\n        generator=self.GetState,\n        condition=lambda s: s != self.__class__.STATE_RUNNING,\n        timeout=timeout)\n    self.target_file = self.target_file.Get()\n    return self", "response": "Wait until the operation is done."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef Get(self):\n\n    args = vfs_pb2.ApiGetFileDetailsArgs(\n        client_id=self.client_id, file_path=self.path)\n    data = self._context.SendRequest(\"GetFileDetails\", args).file\n    return File(client_id=self.client_id, data=data, context=self._context)", "response": "Fetch file s data and return proper File object."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ngetting an output plugin index for a given plugin id.", "response": "def GetOutputPluginIndex(\n    plugin_descriptors,\n    plugin_id):\n  \"\"\"Gets an output plugin index for a plugin with a given id.\n\n  Historically output plugins descriptors were stored in dicts-like\n  structures with unique identifiers as keys. In REL_DB-based implementation,\n  however, both plugin descriptors and their states are stored in flat\n  lists (see Flow definition in flows.proto).\n\n  The ids were formed as \"<plugin name>_<plugin index>\" where plugin index\n  was incremented for every plugin with a same name. For example, if we had\n  EmailOutputPlugin and 2 BigQueryOutputPlugins, their ids would be:\n  EmailOutputPlugin_0, BigQueryOutputPlugin_0, BigQueryOutputPlugin_1.\n\n  To preserve backwards API compatibility, we emulate the old behavior by\n  identifying plugins with same plugin ids as before..\n\n  Args:\n    plugin_descriptors: An iterable of OutputPluginDescriptor objects.\n    plugin_id: Plugin id to search for.\n\n  Returns:\n    An index of a plugin in plugin_descriptors iterable corresponding to a\n    given plugin_id.\n\n  Raises:\n    OutputPluginNotFoundError: if no plugin corresponding to a given plugin_id\n    was found.\n  \"\"\"\n\n  used_names = collections.Counter()\n  for (index, desc) in enumerate(plugin_descriptors):\n    cur_plugin_id = \"%s_%d\" % (desc.plugin_name, used_names[desc.plugin_name])\n    used_names[desc.plugin_name] += 1\n\n    if cur_plugin_id == plugin_id:\n      return index\n\n  raise OutputPluginNotFoundError(\"Can't find output plugin %s\" % plugin_id)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef ResolveCronJobFlowURN(self, cron_job_id):\n    if not self._value:\n      raise ValueError(\"Can't call ResolveCronJobFlowURN on an empty \"\n                       \"client id.\")\n\n    return cron_job_id.ToURN().Add(self._value)", "response": "Resolves a URN of a flow with this id belonging to a given cron job."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef ResolveClientFlowURN(self, client_id, token=None):\n    if not self._value:\n      raise ValueError(\"Can't call ResolveClientFlowURN on an empty client id.\")\n\n    components = self.Split()\n    if len(components) == 1:\n      return self._FlowIdToUrn(self._value, client_id)\n    else:\n      root_urn = self._FlowIdToUrn(components[0], client_id)\n      try:\n        flow_symlink = aff4.FACTORY.Open(\n            root_urn,\n            aff4_type=aff4.AFF4Symlink,\n            follow_symlinks=False,\n            token=token)\n\n        return flow_symlink.Get(flow_symlink.Schema.SYMLINK_TARGET).Add(\n            \"/\".join(components[1:]))\n      except aff4.InstantiationError:\n        return self._FlowIdToUrn(self._value, client_id)", "response": "Resolves a URN of a flow with this id belonging to a given client."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nget a simplified description of the args_type for a flow.", "response": "def _GetArgsDescription(self, args_type):\n    \"\"\"Get a simplified description of the args_type for a flow.\"\"\"\n    args = {}\n    if args_type:\n      for type_descriptor in args_type.type_infos:\n        if not type_descriptor.hidden:\n          args[type_descriptor.name] = {\n              \"description\": type_descriptor.description,\n              \"default\": type_descriptor.default,\n              \"type\": \"\",\n          }\n          if type_descriptor.type:\n            args[type_descriptor.name][\"type\"] = type_descriptor.type.__name__\n    return args"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nget a description of the calling prototype for this flow class.", "response": "def _GetCallingPrototypeAsString(self, flow_cls):\n    \"\"\"Get a description of the calling prototype for this flow class.\"\"\"\n    output = []\n    output.append(\"flow.StartAFF4Flow(client_id=client_id, \")\n    output.append(\"flow_name=\\\"%s\\\", \" % flow_cls.__name__)\n    prototypes = []\n    if flow_cls.args_type:\n      for type_descriptor in flow_cls.args_type.type_infos:\n        if not type_descriptor.hidden:\n          prototypes.append(\"%s=%s\" %\n                            (type_descriptor.name, type_descriptor.name))\n    output.append(\", \".join(prototypes))\n    output.append(\")\")\n    return \"\".join(output)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nget a string description of the calling prototype for this flow.", "response": "def _GetFlowArgsHelpAsString(self, flow_cls):\n    \"\"\"Get a string description of the calling prototype for this flow.\"\"\"\n    output = [\n        \"  Call Spec:\",\n        \"    %s\" % self._GetCallingPrototypeAsString(flow_cls), \"\"\n    ]\n    arg_list = sorted(\n        iteritems(self._GetArgsDescription(flow_cls.args_type)),\n        key=lambda x: x[0])\n    if not arg_list:\n      output.append(\"  Args: None\")\n    else:\n      output.append(\"  Args:\")\n      for arg, val in arg_list:\n        output.append(\"    %s\" % arg)\n        output.append(\"      description: %s\" % val[\"description\"])\n        output.append(\"      type: %s\" % val[\"type\"])\n        output.append(\"      default: %s\" % val[\"default\"])\n        output.append(\"\")\n    return \"\\n\".join(output)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef Handle(self, args, token=None):\n\n    if data_store.RelationalDBEnabled():\n      flow_iterator = iteritems(registry.FlowRegistry.FLOW_REGISTRY)\n    else:\n      flow_iterator = iteritems(registry.AFF4FlowRegistry.FLOW_REGISTRY)\n\n    result = []\n    for name, cls in sorted(flow_iterator):\n\n      # Flows without a category do not show up in the GUI.\n      if not getattr(cls, \"category\", None):\n        continue\n\n      # Only show flows that the user is allowed to start.\n      try:\n        if self.access_check_fn:\n          self.access_check_fn(token.username, name)\n      except access_control.UnauthorizedAccess:\n        continue\n\n      result.append(ApiFlowDescriptor().InitFromFlowClass(cls, token=token))\n\n    return ApiListFlowDescriptorsResult(items=result)", "response": "Renders list of descriptors for all the flows."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nallows given user access to a given subject.", "response": "def AuthorizeUser(self, user, subject):\n    \"\"\"Allow given user access to a given subject.\"\"\"\n\n    user_set = self.authorized_users.setdefault(subject, set())\n    user_set.add(user)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nallow given group access to a subject.", "response": "def AuthorizeGroup(self, group, subject):\n    \"\"\"Allow given group access to a given subject.\"\"\"\n\n    # Add the subject to the dict if is isn't present, so it will get checked in\n    # CheckPermissions\n    self.authorized_users.setdefault(subject, set())\n    self.group_access_manager.AuthorizeGroup(group, subject)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncheck if a given user has access to a given subject.", "response": "def CheckPermissions(self, username, subject):\n    \"\"\"Checks if a given user has access to a given subject.\"\"\"\n\n    if subject in self.authorized_users:\n      return ((username in self.authorized_users[subject]) or\n              self.group_access_manager.MemberOfAuthorizedGroup(\n                  username, subject))\n\n    # In case the subject is not found, the safest thing to do is to raise.\n    # It's up to the users of this class to handle this exception and\n    # grant/not grant permissions to the user in question.\n    raise InvalidSubject(\"Subject %s was not found.\" % subject)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _GetFilePaths(self, path, pathtype, kb):\n\n    environ_vars = artifact_utils.GetWindowsEnvironmentVariablesMap(kb)\n    path_guesses = path_detection_windows.DetectExecutablePaths([path],\n                                                                environ_vars)\n\n    if not path_guesses:\n      # TODO(user): yield a ParserAnomaly object\n      return []\n\n    return [\n        rdf_paths.PathSpec(path=path, pathtype=pathtype)\n        for path in path_guesses\n    ]", "response": "Guess windows filenames from a commandline string."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef Parse(self, persistence, knowledge_base, download_pathtype):\n    pathspecs = []\n\n    if isinstance(persistence, rdf_client.WindowsServiceInformation):\n      if persistence.HasField(\"binary\"):\n        pathspecs.append(persistence.binary.pathspec)\n      elif persistence.HasField(\"image_path\"):\n        pathspecs = self._GetFilePaths(persistence.image_path,\n                                       download_pathtype, knowledge_base)\n\n    if isinstance(\n        persistence,\n        rdf_client_fs.StatEntry) and persistence.HasField(\"registry_type\"):\n      pathspecs = self._GetFilePaths(persistence.registry_data.string,\n                                     download_pathtype, knowledge_base)\n\n    for pathspec in pathspecs:\n      yield rdf_standard.PersistenceFile(pathspec=pathspec)", "response": "Parse the persistence collector output."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nhack to make the filesystem look normal.", "response": "def FileHacks(self):\n    \"\"\"Hacks to make the filesystem look normal.\"\"\"\n    if sys.platform == \"win32\":\n      import win32api  # pylint: disable=g-import-not-at-top\n\n      # Make the filesystem look like the topmost level are the drive letters.\n      if self.path == \"/\":\n        self.files = win32api.GetLogicalDriveStrings().split(\"\\x00\")\n        # Remove empty strings and strip trailing backslashes.\n        self.files = [drive.rstrip(\"\\\\\") for drive in self.files if drive]\n\n      # This regex will match the various windows devices. Raw hard disk devices\n      # must be considered files, however in windows, if we try to list them as\n      # directories this also works. Since the code above distinguished between\n      # files and directories using the file listing property, we must force\n      # treating raw devices as files.\n      elif re.match(r\"/*\\\\\\\\.\\\\[^\\\\]+\\\\?$\", self.path) is not None:\n        # Special case windows devices cant seek to the end so just lie about\n        # the size\n        self.size = 0x7fffffffffffffff\n\n        # Windows raw devices can be opened in two incompatible modes. With a\n        # trailing \\ they look like a directory, but without they are the raw\n        # device. In GRR we only support opening devices in raw mode so ensure\n        # that we never append a \\ to raw device name.\n        self.path = self.path.rstrip(\"\\\\\")\n\n        # In windows raw devices must be accessed using sector alignment.\n        self.alignment = 512\n    elif sys.platform == \"darwin\":\n      # On Mac, raw disk devices are also not seekable to the end and have no\n      # size so we use the same approach as on Windows.\n      if re.match(\"/dev/r?disk.*\", self.path):\n        self.size = 0x7fffffffffffffff\n        self.alignment = 512"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nread from the file.", "response": "def Read(self, length=None):\n    \"\"\"Read from the file.\"\"\"\n\n    if self.progress_callback:\n      self.progress_callback()\n\n    available_to_read = max(0, (self.size or 0) - self.offset)\n    if length is None:\n      to_read = available_to_read\n    else:\n      to_read = min(length, available_to_read)\n\n    with FileHandleManager(self.filename) as fd:\n      offset = self.file_offset + self.offset\n      pre_padding = offset % self.alignment\n\n      # Due to alignment we read some more data than we need to.\n      aligned_offset = offset - pre_padding\n\n      fd.Seek(aligned_offset)\n\n      data = fd.Read(to_read + pre_padding)\n      self.offset += len(data) - pre_padding\n\n      return data[pre_padding:]"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn a StatResponse object for a specific path.", "response": "def _Stat(self, path, ext_attrs=False):\n    \"\"\"Returns stat information of a specific path.\n\n    Args:\n      path: A unicode string containing the path.\n      ext_attrs: Whether the call should also collect extended attributes.\n\n    Returns:\n      a StatResponse proto\n\n    Raises:\n      IOError when call to os.stat() fails\n    \"\"\"\n    # Note that the encoding of local path is system specific\n    local_path = client_utils.CanonicalPathToLocalPath(path)\n    result = client_utils.StatEntryFromPath(\n        local_path, self.pathspec, ext_attrs=ext_attrs)\n\n    # Is this a symlink? If so we need to note the real location of the file.\n    try:\n      result.symlink = utils.SmartUnicode(os.readlink(local_path))\n    except (OSError, AttributeError):\n      pass\n\n    return result"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef ListFiles(self, ext_attrs=False):\n    if not self.IsDirectory():\n      raise IOError(\"%s is not a directory.\" % self.path)\n\n    for path in self.files:\n      try:\n        filepath = utils.JoinPath(self.path, path)\n        response = self._Stat(filepath, ext_attrs=ext_attrs)\n        pathspec = self.pathspec.Copy()\n        pathspec.last.path = utils.JoinPath(pathspec.last.path, path)\n        response.pathspec = pathspec\n\n        yield response\n      except OSError:\n        pass", "response": "List all files in the dir."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef StatFS(self, path=None):\n    if platform.system() == \"Windows\":\n      raise RuntimeError(\"os.statvfs not available on Windows\")\n\n    local_path = client_utils.CanonicalPathToLocalPath(path or self.path)\n\n    return os.statvfs(local_path)", "response": "Calls os. statvfs on a given path."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nwalking back from the path until the mount point is found.", "response": "def GetMountPoint(self, path=None):\n    \"\"\"Walk back from the path to find the mount point.\n\n    Args:\n      path: a Unicode string containing the path or None. If path is None the\n        value in self.path is used.\n\n    Returns:\n      path string of the mount point\n    \"\"\"\n    path = os.path.abspath(\n        client_utils.CanonicalPathToLocalPath(path or self.path))\n\n    while not os.path.ismount(path):\n      path = os.path.dirname(path)\n\n    return path"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nwrites a hunt object to the database.", "response": "def WriteHuntObject(self, hunt_obj, cursor=None):\n    \"\"\"Writes a hunt object to the database.\"\"\"\n    query = \"\"\"\n    INSERT INTO hunts (hunt_id, creator, description, duration_micros,\n                       hunt_state,\n                       client_rate, client_limit,\n                       hunt)\n    VALUES (%(hunt_id)s, %(creator)s, %(description)s, %(duration_micros)s,\n            %(hunt_state)s,\n            %(client_rate)s, %(client_limit)s,\n            %(hunt)s)\n    \"\"\"\n\n    args = {\n        \"hunt_id\": db_utils.HuntIDToInt(hunt_obj.hunt_id),\n        \"creator\": hunt_obj.creator,\n        \"description\": hunt_obj.description,\n        \"duration_micros\": hunt_obj.duration.microseconds,\n        \"hunt_state\": int(rdf_hunt_objects.Hunt.HuntState.PAUSED),\n        \"client_rate\": hunt_obj.client_rate,\n        \"client_limit\": hunt_obj.client_limit,\n        \"hunt\": hunt_obj.SerializeToString(),\n    }\n\n    try:\n      cursor.execute(query, args)\n    except MySQLdb.IntegrityError as error:\n      raise db.DuplicatedHuntError(hunt_id=hunt_obj.hunt_id, cause=error)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef UpdateHuntObject(self,\n                       hunt_id,\n                       duration=None,\n                       client_rate=None,\n                       client_limit=None,\n                       hunt_state=None,\n                       hunt_state_comment=None,\n                       start_time=None,\n                       num_clients_at_start_time=None,\n                       cursor=None):\n    \"\"\"Updates the hunt object by applying the update function.\"\"\"\n    vals = []\n    args = {}\n\n    if duration is not None:\n      vals.append(\"duration_micros = %(duration_micros)s\")\n      args[\"duration_micros\"] = duration.microseconds\n\n    if client_rate is not None:\n      vals.append(\"client_rate = %(client_rate)s\")\n      args[\"client_rate\"] = client_rate\n\n    if client_limit is not None:\n      vals.append(\"client_limit = %(client_limit)s\")\n      args[\"client_limit\"] = client_limit\n\n    if hunt_state is not None:\n      vals.append(\"hunt_state = %(hunt_state)s\")\n      args[\"hunt_state\"] = int(hunt_state)\n\n    if hunt_state_comment is not None:\n      vals.append(\"hunt_state_comment = %(hunt_state_comment)s\")\n      args[\"hunt_state_comment\"] = hunt_state_comment\n\n    if start_time is not None:\n      vals.append(\"\"\"\n      init_start_time = IFNULL(init_start_time, FROM_UNIXTIME(%(start_time)s))\n      \"\"\")\n      vals.append(\"\"\"\n      last_start_time = FROM_UNIXTIME(%(start_time)s)\n      \"\"\")\n      args[\"start_time\"] = mysql_utils.RDFDatetimeToTimestamp(start_time)\n\n    if num_clients_at_start_time is not None:\n      vals.append(\"num_clients_at_start_time = %(num_clients_at_start_time)s\")\n      args[\"num_clients_at_start_time\"] = num_clients_at_start_time\n\n    vals.append(\"last_update_timestamp = NOW(6)\")\n\n    query = \"\"\"\n    UPDATE hunts\n       SET {updates}\n     WHERE hunt_id = %(hunt_id)s\n    \"\"\".format(updates=\", \".join(vals))\n    args[\"hunt_id\"] = db_utils.HuntIDToInt(hunt_id)\n\n    rows_modified = cursor.execute(query, args)\n    if rows_modified == 0:\n      raise db.UnknownHuntError(hunt_id)", "response": "Updates the hunt object by applying the update function."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ndelete a given hunt object.", "response": "def DeleteHuntObject(self, hunt_id, cursor=None):\n    \"\"\"Deletes a given hunt object.\"\"\"\n    query = \"DELETE FROM hunts WHERE hunt_id = %s\"\n    hunt_id_int = db_utils.HuntIDToInt(hunt_id)\n\n    rows_deleted = cursor.execute(query, [hunt_id_int])\n    if rows_deleted == 0:\n      raise db.UnknownHuntError(hunt_id)\n\n    query = \"DELETE FROM hunt_output_plugins_states WHERE hunt_id = %s\"\n    cursor.execute(query, [hunt_id_int])"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ngenerating a hunt object from a database row.", "response": "def _HuntObjectFromRow(self, row):\n    \"\"\"Generates a flow object from a database row.\"\"\"\n    (\n        create_time,\n        last_update_time,\n        creator,\n        duration_micros,\n        client_rate,\n        client_limit,\n        hunt_state,\n        hunt_state_comment,\n        init_start_time,\n        last_start_time,\n        num_clients_at_start_time,\n        description,\n        body,\n    ) = row\n    hunt_obj = rdf_hunt_objects.Hunt.FromSerializedString(body)\n    hunt_obj.duration = rdfvalue.Duration.FromMicroseconds(duration_micros)\n    hunt_obj.create_time = mysql_utils.TimestampToRDFDatetime(create_time)\n    hunt_obj.last_update_time = mysql_utils.TimestampToRDFDatetime(\n        last_update_time)\n\n    # Checks below are needed for hunts that were written to the database before\n    # respective fields became part of F1 schema.\n    if creator is not None:\n      hunt_obj.creator = creator\n\n    if client_rate is not None:\n      hunt_obj.client_rate = client_rate\n\n    if client_limit is not None:\n      hunt_obj.client_limit = client_limit\n\n    if hunt_state is not None:\n      hunt_obj.hunt_state = hunt_state\n\n    if hunt_state_comment is not None:\n      hunt_obj.hunt_state_comment = hunt_state_comment\n\n    if init_start_time is not None:\n      hunt_obj.init_start_time = mysql_utils.TimestampToRDFDatetime(\n          init_start_time)\n\n    if last_start_time is not None:\n      hunt_obj.last_start_time = mysql_utils.TimestampToRDFDatetime(\n          last_start_time)\n\n    if num_clients_at_start_time is not None:\n      hunt_obj.num_clients_at_start_time = num_clients_at_start_time\n\n    if description is not None:\n      hunt_obj.description = description\n\n    return hunt_obj"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef ReadHuntObject(self, hunt_id, cursor=None):\n    query = (\"SELECT {columns} \"\n             \"FROM hunts WHERE hunt_id = %s\".format(\n                 columns=_HUNT_COLUMNS_SELECT))\n\n    nr_results = cursor.execute(query, [db_utils.HuntIDToInt(hunt_id)])\n    if nr_results == 0:\n      raise db.UnknownHuntError(hunt_id)\n\n    return self._HuntObjectFromRow(cursor.fetchone())", "response": "Reads a hunt object from the database."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef ReadHuntObjects(self,\n                      offset,\n                      count,\n                      with_creator=None,\n                      created_after=None,\n                      with_description_match=None,\n                      cursor=None):\n    \"\"\"Reads multiple hunt objects from the database.\"\"\"\n    query = \"SELECT {columns} FROM hunts \".format(columns=_HUNT_COLUMNS_SELECT)\n    args = []\n\n    components = []\n    if with_creator is not None:\n      components.append(\"creator = %s \")\n      args.append(with_creator)\n\n    if created_after is not None:\n      components.append(\"create_timestamp > FROM_UNIXTIME(%s) \")\n      args.append(mysql_utils.RDFDatetimeToTimestamp(created_after))\n\n    if with_description_match is not None:\n      components.append(\"description LIKE %s\")\n      args.append(\"%\" + with_description_match + \"%\")\n\n    if components:\n      query += \"WHERE \" + \" AND \".join(components)\n\n    query += \" ORDER BY create_timestamp DESC LIMIT %s OFFSET %s\"\n    args.append(count)\n    args.append(offset)\n\n    cursor.execute(query, args)\n    return [self._HuntObjectFromRow(row) for row in cursor.fetchall()]", "response": "Reads multiple hunt objects from the database."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nbuilding an OutputPluginState object from a DB row.", "response": "def _HuntOutputPluginStateFromRow(self, row):\n    \"\"\"Builds OutputPluginState object from a DB row.\"\"\"\n    plugin_name, plugin_args_bytes, plugin_state_bytes = row\n\n    plugin_descriptor = rdf_output_plugin.OutputPluginDescriptor(\n        plugin_name=plugin_name)\n    if plugin_args_bytes is not None:\n      plugin_args_cls = plugin_descriptor.GetPluginArgsClass()\n      # If plugin_args_cls is None, we have no clue what class plugin args\n      # should be and therefore no way to deserialize it. This can happen if\n      # a plugin got renamed or removed, for example. In this case we\n      # still want to get plugin's definition and state back and not fail hard,\n      # so that all other plugins can be read.\n      if plugin_args_cls is not None:\n        plugin_descriptor.plugin_args = plugin_args_cls.FromSerializedString(\n            plugin_args_bytes)\n\n    plugin_state = rdf_protodict.AttributedDict.FromSerializedString(\n        plugin_state_bytes)\n    return rdf_flow_runner.OutputPluginState(\n        plugin_descriptor=plugin_descriptor, plugin_state=plugin_state)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nread all hunt output plugins states of a given hunt.", "response": "def ReadHuntOutputPluginsStates(self, hunt_id, cursor=None):\n    \"\"\"Reads all hunt output plugins states of a given hunt.\"\"\"\n\n    columns = \", \".join(_HUNT_OUTPUT_PLUGINS_STATES_COLUMNS)\n\n    query = (\"SELECT {columns} FROM hunt_output_plugins_states \"\n             \"WHERE hunt_id = %s\".format(columns=columns))\n    rows_returned = cursor.execute(query, [db_utils.HuntIDToInt(hunt_id)])\n    if rows_returned > 0:\n      states = []\n      for row in cursor.fetchall():\n        states.append(self._HuntOutputPluginStateFromRow(row))\n      return states\n\n    query = \"SELECT hunt_id FROM hunts WHERE hunt_id = %s\"\n    rows_returned = cursor.execute(query, [db_utils.HuntIDToInt(hunt_id)])\n    if rows_returned == 0:\n      raise db.UnknownHuntError(hunt_id)\n\n    return []"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef WriteHuntOutputPluginsStates(self, hunt_id, states, cursor=None):\n\n    columns = \", \".join(_HUNT_OUTPUT_PLUGINS_STATES_COLUMNS)\n    placeholders = mysql_utils.Placeholders(\n        2 + len(_HUNT_OUTPUT_PLUGINS_STATES_COLUMNS))\n    hunt_id_int = db_utils.HuntIDToInt(hunt_id)\n\n    for index, state in enumerate(states):\n      query = (\"INSERT INTO hunt_output_plugins_states \"\n               \"(hunt_id, plugin_id, {columns}) \"\n               \"VALUES {placeholders}\".format(\n                   columns=columns, placeholders=placeholders))\n      args = [hunt_id_int, index, state.plugin_descriptor.plugin_name]\n\n      if state.plugin_descriptor.plugin_args is None:\n        args.append(None)\n      else:\n        args.append(state.plugin_descriptor.plugin_args.SerializeToString())\n\n      args.append(state.plugin_state.SerializeToString())\n\n      try:\n        cursor.execute(query, args)\n      except MySQLdb.IntegrityError as e:\n        raise db.UnknownHuntError(hunt_id=hunt_id, cause=e)", "response": "Writes a list of hunt output plugin states for a given hunt."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef UpdateHuntOutputPluginState(self,\n                                  hunt_id,\n                                  state_index,\n                                  update_fn,\n                                  cursor=None):\n    \"\"\"Updates hunt output plugin state for a given output plugin.\"\"\"\n\n    hunt_id_int = db_utils.HuntIDToInt(hunt_id)\n\n    query = \"SELECT hunt_id FROM hunts WHERE hunt_id = %s\"\n    rows_returned = cursor.execute(query, [hunt_id_int])\n    if rows_returned == 0:\n      raise db.UnknownHuntError(hunt_id)\n\n    columns = \", \".join(_HUNT_OUTPUT_PLUGINS_STATES_COLUMNS)\n    query = (\"SELECT {columns} FROM hunt_output_plugins_states \"\n             \"WHERE hunt_id = %s AND plugin_id = %s\".format(columns=columns))\n    rows_returned = cursor.execute(query, [hunt_id_int, state_index])\n    if rows_returned == 0:\n      raise db.UnknownHuntOutputPluginStateError(hunt_id, state_index)\n\n    state = self._HuntOutputPluginStateFromRow(cursor.fetchone())\n    modified_plugin_state = update_fn(state.plugin_state)\n\n    query = (\"UPDATE hunt_output_plugins_states \"\n             \"SET plugin_state = %s \"\n             \"WHERE hunt_id = %s AND plugin_id = %s\")\n    args = [modified_plugin_state.SerializeToString(), hunt_id_int, state_index]\n    cursor.execute(query, args)\n    return state", "response": "Updates hunt output plugin state for a given hunt."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef ReadHuntLogEntries(self,\n                         hunt_id,\n                         offset,\n                         count,\n                         with_substring=None,\n                         cursor=None):\n    \"\"\"Reads hunt log entries of a given hunt using given query options.\"\"\"\n    hunt_id_int = db_utils.HuntIDToInt(hunt_id)\n\n    query = (\"SELECT client_id, flow_id, message, UNIX_TIMESTAMP(timestamp) \"\n             \"FROM flow_log_entries \"\n             \"FORCE INDEX(flow_log_entries_by_hunt) \"\n             \"WHERE hunt_id = %s AND flow_id = hunt_id \")\n\n    args = [hunt_id_int]\n\n    if with_substring is not None:\n      query += \"AND message LIKE %s \"\n      args.append(\"%\" + db_utils.EscapeWildcards(with_substring) + \"%\")\n\n    query += \"ORDER BY timestamp ASC LIMIT %s OFFSET %s\"\n\n    args.append(count)\n    args.append(offset)\n\n    cursor.execute(query, args)\n\n    flow_log_entries = []\n    for client_id_int, flow_id_int, message, timestamp in cursor.fetchall():\n      flow_log_entries.append(\n          rdf_flow_objects.FlowLogEntry(\n              client_id=db_utils.IntToClientID(client_id_int),\n              flow_id=db_utils.IntToFlowID(flow_id_int),\n              hunt_id=hunt_id,\n              message=message,\n              timestamp=mysql_utils.TimestampToRDFDatetime(timestamp)))\n\n    return flow_log_entries", "response": "Reads hunt log entries of a given hunt."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef CountHuntLogEntries(self, hunt_id, cursor=None):\n    hunt_id_int = db_utils.HuntIDToInt(hunt_id)\n\n    query = (\"SELECT COUNT(*) FROM flow_log_entries \"\n             \"FORCE INDEX(flow_log_entries_by_hunt) \"\n             \"WHERE hunt_id = %s AND flow_id = hunt_id\")\n    cursor.execute(query, [hunt_id_int])\n    return cursor.fetchone()[0]", "response": "Returns the number of hunt log entries of a given hunt."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreading hunt results of a given hunt.", "response": "def ReadHuntResults(self,\n                      hunt_id,\n                      offset,\n                      count,\n                      with_tag=None,\n                      with_type=None,\n                      with_substring=None,\n                      with_timestamp=None,\n                      cursor=None):\n    \"\"\"Reads hunt results of a given hunt using given query options.\"\"\"\n    hunt_id_int = db_utils.HuntIDToInt(hunt_id)\n\n    query = (\"SELECT client_id, flow_id, hunt_id, payload, type, \"\n             \"UNIX_TIMESTAMP(timestamp), tag \"\n             \"FROM flow_results \"\n             \"FORCE INDEX(flow_results_hunt_id_flow_id_timestamp) \"\n             \"WHERE hunt_id = %s \")\n\n    args = [hunt_id_int]\n\n    if with_tag:\n      query += \"AND tag = %s \"\n      args.append(with_tag)\n\n    if with_type:\n      query += \"AND type = %s \"\n      args.append(with_type)\n\n    if with_substring:\n      query += \"AND payload LIKE %s \"\n      args.append(\"%\" + db_utils.EscapeWildcards(with_substring) + \"%\")\n\n    if with_timestamp:\n      query += \"AND timestamp = FROM_UNIXTIME(%s) \"\n      args.append(mysql_utils.RDFDatetimeToTimestamp(with_timestamp))\n\n    query += \"ORDER BY timestamp ASC LIMIT %s OFFSET %s\"\n    args.append(count)\n    args.append(offset)\n\n    cursor.execute(query, args)\n\n    ret = []\n    for (\n        client_id_int,\n        flow_id_int,\n        hunt_id_int,\n        serialized_payload,\n        payload_type,\n        timestamp,\n        tag,\n    ) in cursor.fetchall():\n      if payload_type in rdfvalue.RDFValue.classes:\n        payload = rdfvalue.RDFValue.classes[payload_type]()\n        payload.ParseFromString(serialized_payload)\n      else:\n        payload = rdf_objects.SerializedValueOfUnrecognizedType(\n            type_name=payload_type, value=serialized_payload)\n\n      result = rdf_flow_objects.FlowResult(\n          client_id=db_utils.IntToClientID(client_id_int),\n          flow_id=db_utils.IntToFlowID(flow_id_int),\n          hunt_id=hunt_id,\n          payload=payload,\n          timestamp=mysql_utils.TimestampToRDFDatetime(timestamp))\n      if tag is not None:\n        result.tag = tag\n\n      ret.append(result)\n\n    return ret"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef CountHuntResults(self,\n                       hunt_id,\n                       with_tag=None,\n                       with_type=None,\n                       cursor=None):\n    \"\"\"Counts hunt results of a given hunt using given query options.\"\"\"\n    hunt_id_int = db_utils.HuntIDToInt(hunt_id)\n\n    query = \"SELECT COUNT(*) FROM flow_results WHERE hunt_id = %s \"\n\n    args = [hunt_id_int]\n\n    if with_tag is not None:\n      query += \"AND tag = %s \"\n      args.append(with_tag)\n\n    if with_type is not None:\n      query += \"AND type = %s \"\n      args.append(with_type)\n\n    cursor.execute(query, args)\n    return cursor.fetchone()[0]", "response": "Counts hunt results of a given hunt."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef CountHuntResultsByType(self, hunt_id, cursor=None):\n    hunt_id_int = db_utils.HuntIDToInt(hunt_id)\n\n    query = (\"SELECT type, COUNT(*) FROM flow_results \"\n             \"WHERE hunt_id = %s GROUP BY type\")\n\n    cursor.execute(query, [hunt_id_int])\n    return dict(cursor.fetchall())", "response": "Counts number of hunts results per type."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nbuild an SQL condition matching the given hunt flow condition.", "response": "def _HuntFlowCondition(self, condition):\n    \"\"\"Builds an SQL condition matching db.HuntFlowsCondition.\"\"\"\n    if condition == db.HuntFlowsCondition.UNSET:\n      return \"\", []\n    elif condition == db.HuntFlowsCondition.FAILED_FLOWS_ONLY:\n      return (\"AND flow_state = %s \",\n              [int(rdf_flow_objects.Flow.FlowState.ERROR)])\n    elif condition == db.HuntFlowsCondition.SUCCEEDED_FLOWS_ONLY:\n      return (\"AND flow_state = %s \",\n              [int(rdf_flow_objects.Flow.FlowState.FINISHED)])\n    elif condition == db.HuntFlowsCondition.COMPLETED_FLOWS_ONLY:\n      return (\"AND (flow_state = %s OR flow_state = %s) \", [\n          int(rdf_flow_objects.Flow.FlowState.FINISHED),\n          int(rdf_flow_objects.Flow.FlowState.ERROR)\n      ])\n    elif condition == db.HuntFlowsCondition.FLOWS_IN_PROGRESS_ONLY:\n      return (\"AND flow_state = %s \",\n              [int(rdf_flow_objects.Flow.FlowState.RUNNING)])\n    elif condition == db.HuntFlowsCondition.CRASHED_FLOWS_ONLY:\n      return (\"AND flow_state = %s \",\n              [int(rdf_flow_objects.Flow.FlowState.CRASHED)])\n    else:\n      raise ValueError(\"Invalid condition value: %r\" % condition)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef ReadHuntFlows(self,\n                    hunt_id,\n                    offset,\n                    count,\n                    filter_condition=db.HuntFlowsCondition.UNSET,\n                    cursor=None):\n    \"\"\"Reads hunt flows matching given conditins.\"\"\"\n    hunt_id_int = db_utils.HuntIDToInt(hunt_id)\n\n    query = (\"SELECT {columns} FROM flows \"\n             \"FORCE INDEX(flows_by_hunt) \"\n             \"WHERE parent_hunt_id = %s AND parent_flow_id IS NULL \"\n             \"{filter_condition} \"\n             \"ORDER BY last_update ASC \"\n             \"LIMIT %s OFFSET %s\")\n\n    filter_query, extra_args = self._HuntFlowCondition(filter_condition)\n    query = query.format(\n        columns=self.FLOW_DB_FIELDS, filter_condition=filter_query)\n    args = [hunt_id_int] + extra_args + [count, offset]\n\n    cursor.execute(query, args)\n    return [self._FlowObjectFromRow(row) for row in cursor.fetchall()]", "response": "Reads hunt flows matching given conditins."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncounts hunt flows matching given conditions.", "response": "def CountHuntFlows(self,\n                     hunt_id,\n                     filter_condition=db.HuntFlowsCondition.UNSET,\n                     cursor=None):\n    \"\"\"Counts hunt flows matching given conditions.\"\"\"\n    hunt_id_int = db_utils.HuntIDToInt(hunt_id)\n\n    query = (\"SELECT COUNT(*) FROM flows \"\n             \"FORCE INDEX(flows_by_hunt) \"\n             \"WHERE parent_hunt_id = %s AND parent_flow_id IS NULL \"\n             \"{filter_condition}\")\n\n    filter_query, extra_args = self._HuntFlowCondition(filter_condition)\n    args = [hunt_id_int] + extra_args\n    query = query.format(filter_condition=filter_query)\n    cursor.execute(query, args)\n    return cursor.fetchone()[0]"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nbuild an SQL query part to fetch counts corresponding to given bins.", "response": "def _BinsToQuery(self, bins, column_name):\n    \"\"\"Builds an SQL query part to fetch counts corresponding to given bins.\"\"\"\n    result = []\n    # With the current StatsHistogram implementation the last bin simply\n    # takes all the values that are greater than range_max_value of\n    # the one-before-the-last bin. range_max_value of the last bin\n    # is thus effectively ignored.\n    for prev_b, next_b in zip([0] + bins[:-1], bins[:-1] + [None]):\n      query = \"COUNT(CASE WHEN %s >= %f\" % (column_name, prev_b)\n      if next_b is not None:\n        query += \" AND %s < %f\" % (column_name, next_b)\n\n      query += \" THEN 1 END)\"\n\n      result.append(query)\n\n    return \", \".join(result)"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreads the hunt client resources stats for a given hunt.", "response": "def ReadHuntClientResourcesStats(self, hunt_id, cursor=None):\n    \"\"\"Read/calculate hunt client resources stats.\"\"\"\n    hunt_id_int = db_utils.HuntIDToInt(hunt_id)\n\n    query = \"\"\"\n      SELECT\n        COUNT(*),\n        SUM(user_cpu_time_used_micros),\n        SUM((user_cpu_time_used_micros) * (user_cpu_time_used_micros)),\n        SUM(system_cpu_time_used_micros),\n        SUM((system_cpu_time_used_micros) * (system_cpu_time_used_micros)),\n        SUM(network_bytes_sent),\n        SUM(network_bytes_sent * network_bytes_sent),\n    \"\"\"\n\n    scaled_bins = [\n        int(1000000 * b) for b in rdf_stats.ClientResourcesStats.CPU_STATS_BINS\n    ]\n\n    query += self._BinsToQuery(scaled_bins, \"(user_cpu_time_used_micros)\")\n    query += \",\"\n    query += self._BinsToQuery(scaled_bins, \"(system_cpu_time_used_micros)\")\n    query += \",\"\n    query += self._BinsToQuery(\n        rdf_stats.ClientResourcesStats.NETWORK_STATS_BINS, \"network_bytes_sent\")\n\n    query += \" FROM flows \"\n    query += \"FORCE INDEX(flows_by_hunt) \"\n    query += \"WHERE parent_hunt_id = %s AND parent_flow_id IS NULL\"\n\n    cursor.execute(query, [hunt_id_int])\n\n    response = cursor.fetchone()\n    (count, user_sum, user_sq_sum, system_sum, system_sq_sum, network_sum,\n     network_sq_sum) = response[:7]\n\n    stats = rdf_stats.ClientResourcesStats(\n        user_cpu_stats=rdf_stats.RunningStats(\n            num=count,\n            sum=db_utils.MicrosToSeconds(int(user_sum or 0)),\n            sum_sq=int(user_sq_sum or 0) / 1e12,\n        ),\n        system_cpu_stats=rdf_stats.RunningStats(\n            num=count,\n            sum=db_utils.MicrosToSeconds(int(system_sum or 0)),\n            sum_sq=int(system_sq_sum or 0) / 1e12,\n        ),\n        network_bytes_sent_stats=rdf_stats.RunningStats(\n            num=count,\n            sum=float(network_sum or 0),\n            sum_sq=float(network_sq_sum or 0),\n        ),\n    )\n\n    offset = 7\n    stats.user_cpu_stats.histogram = rdf_stats.StatsHistogram()\n    for b_num, b_max_value in zip(\n        response[offset:], rdf_stats.ClientResourcesStats.CPU_STATS_BINS):\n      stats.user_cpu_stats.histogram.bins.append(\n          rdf_stats.StatsHistogramBin(range_max_value=b_max_value, num=b_num))\n\n    offset += len(rdf_stats.ClientResourcesStats.CPU_STATS_BINS)\n    stats.system_cpu_stats.histogram = rdf_stats.StatsHistogram()\n    for b_num, b_max_value in zip(\n        response[offset:], rdf_stats.ClientResourcesStats.CPU_STATS_BINS):\n      stats.system_cpu_stats.histogram.bins.append(\n          rdf_stats.StatsHistogramBin(range_max_value=b_max_value, num=b_num))\n\n    offset += len(rdf_stats.ClientResourcesStats.CPU_STATS_BINS)\n    stats.network_bytes_sent_stats.histogram = rdf_stats.StatsHistogram()\n    for b_num, b_max_value in zip(\n        response[offset:], rdf_stats.ClientResourcesStats.NETWORK_STATS_BINS):\n      stats.network_bytes_sent_stats.histogram.bins.append(\n          rdf_stats.StatsHistogramBin(range_max_value=b_max_value, num=b_num))\n\n    query = \"\"\"\n      SELECT\n        client_id, flow_id, user_cpu_time_used_micros,\n        system_cpu_time_used_micros, network_bytes_sent\n      FROM flows\n      FORCE INDEX(flows_by_hunt)\n      WHERE parent_hunt_id = %s AND parent_flow_id IS NULL AND\n            (user_cpu_time_used_micros > 0 OR\n             system_cpu_time_used_micros > 0 OR\n             network_bytes_sent > 0)\n      ORDER BY (user_cpu_time_used_micros + system_cpu_time_used_micros) DESC\n      LIMIT 10\n    \"\"\"\n\n    cursor.execute(query, [hunt_id_int])\n\n    for cid, fid, ucpu, scpu, nbs in cursor.fetchall():\n      client_id = db_utils.IntToClientID(cid)\n      flow_id = db_utils.IntToFlowID(fid)\n      stats.worst_performers.append(\n          rdf_client_stats.ClientResources(\n              client_id=client_id,\n              session_id=rdfvalue.RDFURN(client_id).Add(flow_id),\n              cpu_usage=rdf_client_stats.CpuSeconds(\n                  user_cpu_time=db_utils.MicrosToSeconds(ucpu),\n                  system_cpu_time=db_utils.MicrosToSeconds(scpu),\n              ),\n              network_bytes_sent=nbs))\n\n    return stats"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nread hunt flows states and timestamps.", "response": "def ReadHuntFlowsStatesAndTimestamps(self, hunt_id, cursor=None):\n    \"\"\"Reads hunt flows states and timestamps.\"\"\"\n\n    query = \"\"\"\n      SELECT\n        flow_state, UNIX_TIMESTAMP(timestamp), UNIX_TIMESTAMP(last_update)\n      FROM flows\n      FORCE INDEX(flows_by_hunt)\n      WHERE parent_hunt_id = %s AND parent_flow_id IS NULL\n    \"\"\"\n\n    cursor.execute(query, [db_utils.HuntIDToInt(hunt_id)])\n\n    result = []\n    for fs, ct, lup in cursor.fetchall():\n      result.append(\n          db.FlowStateAndTimestamps(\n              flow_state=rdf_flow_objects.Flow.FlowState.FromInt(fs),\n              create_time=mysql_utils.TimestampToRDFDatetime(ct),\n              last_update_time=mysql_utils.TimestampToRDFDatetime(lup)))\n\n    return result"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef ReadHuntOutputPluginLogEntries(self,\n                                     hunt_id,\n                                     output_plugin_id,\n                                     offset,\n                                     count,\n                                     with_type=None,\n                                     cursor=None):\n    \"\"\"Reads hunt output plugin log entries.\"\"\"\n    query = (\"SELECT client_id, flow_id, log_entry_type, message, \"\n             \"UNIX_TIMESTAMP(timestamp) \"\n             \"FROM flow_output_plugin_log_entries \"\n             \"FORCE INDEX (flow_output_plugin_log_entries_by_hunt) \"\n             \"WHERE hunt_id = %s AND output_plugin_id = %s \")\n    args = [\n        db_utils.HuntIDToInt(hunt_id),\n        db_utils.OutputPluginIDToInt(output_plugin_id)\n    ]\n\n    if with_type is not None:\n      query += \"AND log_entry_type = %s \"\n      args.append(int(with_type))\n\n    query += \"ORDER BY log_id ASC LIMIT %s OFFSET %s\"\n    args.append(count)\n    args.append(offset)\n\n    cursor.execute(query, args)\n\n    ret = []\n    for (client_id_int, flow_id_int, log_entry_type, message,\n         timestamp) in cursor.fetchall():\n      ret.append(\n          rdf_flow_objects.FlowOutputPluginLogEntry(\n              hunt_id=hunt_id,\n              client_id=db_utils.IntToClientID(client_id_int),\n              flow_id=db_utils.IntToFlowID(flow_id_int),\n              output_plugin_id=output_plugin_id,\n              log_entry_type=log_entry_type,\n              message=message,\n              timestamp=mysql_utils.TimestampToRDFDatetime(timestamp)))\n\n    return ret", "response": "Reads hunt output plugin log entries."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef CountHuntOutputPluginLogEntries(self,\n                                      hunt_id,\n                                      output_plugin_id,\n                                      with_type=None,\n                                      cursor=None):\n    \"\"\"Counts hunt output plugin log entries.\"\"\"\n    query = (\"SELECT COUNT(*) \"\n             \"FROM flow_output_plugin_log_entries \"\n             \"FORCE INDEX (flow_output_plugin_log_entries_by_hunt) \"\n             \"WHERE hunt_id = %s AND output_plugin_id = %s \")\n    args = [\n        db_utils.HuntIDToInt(hunt_id),\n        db_utils.OutputPluginIDToInt(output_plugin_id)\n    ]\n\n    if with_type is not None:\n      query += \"AND log_entry_type = %s\"\n      args.append(int(with_type))\n\n    cursor.execute(query, args)\n    return cursor.fetchone()[0]", "response": "Counts hunt output plugin log entries."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nruns all the actions specified in the rule.", "response": "def _RunAction(self, rule, client_id):\n    \"\"\"Run all the actions specified in the rule.\n\n    Args:\n      rule: Rule which actions are to be executed.\n      client_id: Id of a client where rule's actions are to be executed.\n\n    Returns:\n      Number of actions started.\n    \"\"\"\n    actions_count = 0\n\n    try:\n      if self._CheckIfHuntTaskWasAssigned(client_id, rule.hunt_id):\n        logging.info(\n            \"Foreman: ignoring hunt %s on client %s: was started \"\n            \"here before\", client_id, rule.hunt_id)\n      else:\n        logging.info(\"Foreman: Starting hunt %s on client %s.\", rule.hunt_id,\n                     client_id)\n\n        # hunt_name is only used for legacy hunts.\n        if rule.hunt_name:\n          flow_cls = registry.AFF4FlowRegistry.FlowClassByName(rule.hunt_name)\n          hunt_urn = rdfvalue.RDFURN(\"aff4:/hunts/%s\" % rule.hunt_id)\n          flow_cls.StartClients(hunt_urn, [client_id])\n        else:\n          hunt.StartHuntFlowOnClient(client_id, rule.hunt_id)\n\n        actions_count += 1\n\n    # There could be all kinds of errors we don't know about when starting the\n    # hunt so we catch everything here.\n    except Exception as e:  # pylint: disable=broad-except\n      logging.exception(\"Failure running foreman action on client %s: %s\",\n                        rule.hunt_id, e)\n\n    return actions_count"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nassigning tasks to a client.", "response": "def AssignTasksToClient(self, client_id):\n    \"\"\"Examines our rules and starts up flows based on the client.\n\n    Args:\n      client_id: Client id of the client for tasks to be assigned.\n\n    Returns:\n      Number of assigned tasks.\n    \"\"\"\n    rules = data_store.REL_DB.ReadAllForemanRules()\n    if not rules:\n      return 0\n\n    last_foreman_run = self._GetLastForemanRunTime(client_id)\n\n    latest_rule_creation_time = max(rule.creation_time for rule in rules)\n\n    if latest_rule_creation_time <= last_foreman_run:\n      return 0\n\n    # Update the latest checked rule on the client.\n    self._SetLastForemanRunTime(client_id, latest_rule_creation_time)\n\n    relevant_rules = []\n    expired_rules = False\n\n    now = rdfvalue.RDFDatetime.Now()\n\n    for rule in rules:\n      if rule.expiration_time < now:\n        expired_rules = True\n        continue\n      if rule.creation_time <= last_foreman_run:\n        continue\n\n      relevant_rules.append(rule)\n\n    actions_count = 0\n    if relevant_rules:\n      client_data = data_store.REL_DB.ReadClientFullInfo(client_id)\n      if client_data is None:\n        return\n\n      for rule in relevant_rules:\n        if rule.Evaluate(client_data):\n          actions_count += self._RunAction(rule, client_id)\n\n    if expired_rules:\n      data_store.REL_DB.RemoveExpiredForemanRules()\n\n    return actions_count"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nfetches metadata for the given binary from the datastore.", "response": "def _GetSignedBinaryMetadata(binary_type, relative_path):\n  \"\"\"Fetches metadata for the given binary from the datastore.\n\n  Args:\n    binary_type: ApiGrrBinary.Type of the binary.\n    relative_path: Relative path of the binary, relative to the canonical URN\n      roots for signed binaries (see _GetSignedBlobsRoots()).\n\n  Returns:\n    An ApiGrrBinary RDFProtoStruct containing metadata for the binary.\n  \"\"\"\n  root_urn = _GetSignedBlobsRoots()[binary_type]\n  binary_urn = root_urn.Add(relative_path)\n  blob_iterator, timestamp = signed_binary_utils.FetchBlobsForSignedBinary(\n      binary_urn)\n  binary_size = 0\n  has_valid_signature = True\n  for blob in blob_iterator:\n    binary_size += len(blob.data)\n    if not has_valid_signature:\n      # No need to check the signature if a previous blob had an invalid\n      # signature.\n      continue\n    try:\n      blob.Verify(config.CONFIG[\"Client.executable_signing_public_key\"])\n    except rdf_crypto.Error:\n      has_valid_signature = False\n\n  return ApiGrrBinary(\n      path=relative_path,\n      type=binary_type,\n      size=binary_size,\n      timestamp=timestamp,\n      has_valid_signature=has_valid_signature)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef Handle(self, unused_args, token=None):\n\n    sections = {}\n    for descriptor in config.CONFIG.type_infos:\n      if descriptor.section in sections:\n        continue\n\n      section_data = {}\n      for parameter in self._ListParametersInSection(descriptor.section):\n        section_data[parameter] = ApiConfigOption().InitFromConfigOption(\n            parameter)\n\n      sections[descriptor.section] = section_data\n\n    result = ApiGetConfigResult()\n    for section_name in sorted(sections):\n      section = sections[section_name]\n\n      api_section = ApiConfigSection(name=section_name)\n      api_section.options = []\n      for param_name in sorted(section):\n        api_section.options.append(section[param_name])\n      result.sections.append(api_section)\n\n    return result", "response": "Builds the data structure representing the config."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef Handle(self, args, token=None):\n\n    if not args.name:\n      raise ValueError(\"Name not specified.\")\n\n    return ApiConfigOption().InitFromConfigOption(args.name)", "response": "Renders specified config option."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef Proxy(f):\n\n  def Wrapped(self, *args):\n    return getattr(self, f)(*args)\n\n  return Wrapped", "response": "A helper to create a proxy method in a class."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning a unicode object.", "response": "def SmartUnicode(string):\n  \"\"\"Returns a unicode object.\n\n  This function will always return a unicode object. It should be used to\n  guarantee that something is always a unicode object.\n\n  Args:\n    string: The string to convert.\n\n  Returns:\n    a unicode object.\n  \"\"\"\n  if isinstance(string, Text):\n    return string\n\n  if isinstance(string, bytes):\n    return string.decode(\"utf-8\", \"ignore\")\n\n  # TODO: We need to call `__native__` because otherwise one of the\n  # Selenium tests becomes flaky. This should be investigated.\n  if compatibility.PY2:\n    return str(string).__native__()\n  else:\n    return str(string)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a bytes object where each byte has been xored with key.", "response": "def Xor(bytestr, key):\n  \"\"\"Returns a `bytes` object where each byte has been xored with key.\"\"\"\n  # TODO(hanuszczak): Remove this import when string migration is done.\n  # pytype: disable=import-error\n  from builtins import bytes  # pylint: disable=redefined-builtin, g-import-not-at-top\n  # pytype: enable=import-error\n  precondition.AssertType(bytestr, bytes)\n\n  # TODO: This seemingly no-op operation actually changes things.\n  # In Python 2 this function receives a `str` object which has different\n  # iterator semantics. So we use a `bytes` wrapper from the `future` package to\n  # get the Python 3 behaviour. In Python 3 this should be indeed a no-op. Once\n  # the migration is completed and support for Python 2 is dropped, this line\n  # can be removed.\n  bytestr = bytes(bytestr)\n\n  return bytes([byte ^ key for byte in bytestr])"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef FormatAsHexString(num, width=None, prefix=\"0x\"):\n  # Strip \"0x\".\n  hex_str = hex(num)[2:]\n  # Strip \"L\" for long values.\n  hex_str = hex_str.replace(\"L\", \"\")\n  if width:\n    hex_str = hex_str.rjust(width, \"0\")\n  return \"%s%s\" % (prefix, hex_str)", "response": "Takes an int and returns the number formatted as a hex string."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef NormalizePath(path, sep=\"/\"):\n  if not path:\n    return sep\n  path = SmartUnicode(path)\n\n  path_list = path.split(sep)\n\n  # This is a relative path and the first element is . or ..\n  if path_list[0] in [\".\", \"..\", \"\"]:\n    path_list.pop(0)\n\n  # Deliberately begin at index 1 to preserve a single leading /\n  i = 0\n\n  while True:\n    list_len = len(path_list)\n\n    # We begin at the last known good position so we never iterate over path\n    # elements which are already examined\n    for i in range(i, len(path_list)):\n      # Remove /./ form\n      if path_list[i] == \".\" or not path_list[i]:\n        path_list.pop(i)\n        break\n\n      # Remove /../ form\n      elif path_list[i] == \"..\":\n        path_list.pop(i)\n        # Anchor at the top level\n        if (i == 1 and path_list[0]) or i > 1:\n          i -= 1\n          path_list.pop(i)\n        break\n\n    # If we didnt alter the path so far we can quit\n    if len(path_list) == list_len:\n      return sep + sep.join(path_list)", "response": "This function normalizes a path."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef JoinPath(stem=\"\", *parts):\n  # Ensure all path components are unicode\n  parts = [SmartUnicode(path) for path in parts]\n\n  result = (stem + NormalizePath(u\"/\".join(parts))).replace(\"//\", \"/\")\n  result = result.rstrip(\"/\")\n\n  return result or \"/\"", "response": "A sane version of os. path. join."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef GeneratePassphrase(length=20):\n  valid_chars = \"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ\"\n  valid_chars += \"0123456789 ,-_&$#\"\n  return \"\".join(random.choice(valid_chars) for i in range(length))", "response": "Generate a 20 char passphrase with easily typeable chars."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef ResolveHostnameToIP(host, port):\n  ip_addrs = socket.getaddrinfo(host, port, socket.AF_UNSPEC, 0,\n                                socket.IPPROTO_TCP)\n  # getaddrinfo returns tuples (family, socktype, proto, canonname, sockaddr).\n  # We are interested in sockaddr which is in turn a tuple\n  # (address, port) for IPv4 or (address, port, flow info, scope id)\n  # for IPv6. In both cases, we want the first element, the address.\n  result = ip_addrs[0][4][0]\n  # TODO: In Python 2, this value is a byte string instead of UTF-8\n  # string. To ensure type correctness until support for Python 2 is dropped,\n  # we always decode this value.\n  if compatibility.PY2:\n    result = result.decode(\"ascii\")\n  return result", "response": "Resolves a hostname to an IP address."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef Expire(self):\n    while len(self._age) > self._limit:\n      node = self._age.PopLeft()\n      self._hash.pop(node.key, None)\n      self.KillObject(node.data)", "response": "Expires old cache entries."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef Put(self, key, obj):\n    # Remove the old entry if it is there.\n    node = self._hash.pop(key, None)\n    if node:\n      self._age.Unlink(node)\n\n    # Make a new node and insert it.\n    node = Node(key=key, data=obj)\n    self._hash[key] = node\n    self._age.AppendNode(node)\n\n    self.Expire()\n\n    return key", "response": "Adds the object to the cache."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nexpire a specific object from cache.", "response": "def ExpireObject(self, key):\n    \"\"\"Expire a specific object from cache.\"\"\"\n    node = self._hash.pop(key, None)\n    if node:\n      self._age.Unlink(node)\n      self.KillObject(node.data)\n\n      return node.data"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nexpiring all the objects with the key matching the regex.", "response": "def ExpireRegEx(self, regex):\n    \"\"\"Expire all the objects with the key matching the regex.\"\"\"\n    reg = re.compile(regex)\n    for key in list(self._hash):\n      if reg.match(key):\n        self.ExpireObject(key)"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nexpires all the objects with the given prefix.", "response": "def ExpirePrefix(self, prefix):\n    \"\"\"Expire all the objects with the key having a given prefix.\"\"\"\n    for key in list(self._hash):\n      if key.startswith(prefix):\n        self.ExpireObject(key)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef Pop(self, key):\n    node = self._hash.get(key)\n\n    if node:\n      del self._hash[key]\n      self._age.Unlink(node)\n\n      return node.data", "response": "Removes the object from the cache completely."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nretrieve the object from the cache.", "response": "def Get(self, key):\n    \"\"\"Fetch the object from cache.\n\n    Objects may be flushed from cache at any time. Callers must always\n    handle the possibility of KeyError raised here.\n\n    Args:\n      key: The key used to access the object.\n\n    Returns:\n      Cached object.\n\n    Raises:\n      KeyError: If the object is not present in the cache.\n    \"\"\"\n    if key not in self._hash:\n      raise KeyError(key)\n\n    node = self._hash[key]\n\n    self._age.Unlink(node)\n    self._age.AppendNode(node)\n\n    return node.data"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef Flush(self):\n    while self._age:\n      node = self._age.PopLeft()\n      self.KillObject(node.data)\n\n    self._hash = dict()", "response": "Flush all items from cache."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncalculate the size of the struct.", "response": "def GetSize(cls):\n    \"\"\"Calculate the size of the struct.\"\"\"\n    format_str = \"\".join([x[0] for x in cls._fields])\n    return struct.calcsize(format_str)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ngetting the value from the stream since the last call.", "response": "def GetValueAndReset(self):\n    \"\"\"Gets stream buffer since the last GetValueAndReset() call.\"\"\"\n    if not self._stream:\n      raise ArchiveAlreadyClosedError(\n          \"Attempting to get a value from a closed stream.\")\n\n    value = self._stream.getvalue()\n    self._stream.seek(0)\n    self._stream.truncate()\n\n    return value"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ngenerating a new Info - ZIP file for the given name compression and stat.", "response": "def _GenerateZipInfo(self, arcname=None, compress_type=None, st=None):\n    \"\"\"Generate ZipInfo instance for the given name, compression and stat.\n\n    Args:\n      arcname: The name in the archive this should take.\n      compress_type: Compression type (zipfile.ZIP_DEFLATED, or ZIP_STORED)\n      st: An optional stat object to be used for setting headers.\n\n    Returns:\n      ZipInfo instance.\n\n    Raises:\n      ValueError: If arcname is not provided.\n    \"\"\"\n    # Fake stat response.\n    if st is None:\n      # TODO(user):pytype: stat_result typing is not correct.\n      # pytype: disable=wrong-arg-count\n      st = os.stat_result((0o100644, 0, 0, 0, 0, 0, 0, 0, 0, 0))\n      # pytype: enable=wrong-arg-count\n\n    mtime = time.localtime(st.st_mtime or time.time())\n    date_time = mtime[0:6]\n    # Create ZipInfo instance to store file information\n    if arcname is None:\n      raise ValueError(\"An arcname must be provided.\")\n\n    zinfo = zipfile.ZipInfo(arcname, date_time)\n    zinfo.external_attr = (st[0] & 0xFFFF) << 16  # Unix attributes\n\n    if compress_type is None:\n      zinfo.compress_type = self._compression\n    else:\n      zinfo.compress_type = compress_type\n\n    zinfo.file_size = 0\n    zinfo.compress_size = 0\n    zinfo.flag_bits = 0x08  # Setting data descriptor flag.\n    zinfo.CRC = 0x08074b50  # Predefined CRC for archives using data\n    # descriptors.\n    # This fills an empty Info-ZIP Unix extra field.\n    zinfo.extra = struct.pack(\n        \"<HHIIHH\",\n        0x5855,\n        12,\n        0,  # time of last access (UTC/GMT)\n        0,  # time of last modification (UTC/GMT)\n        0,  # user ID\n        0)  # group ID\n    return zinfo"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef WriteSymlink(self, src_arcname, dst_arcname):\n    # Inspired by:\n    # http://www.mail-archive.com/python-list@python.org/msg34223.html\n\n    if not self._stream:\n      raise ArchiveAlreadyClosedError(\n          \"Attempting to write to a ZIP archive that was already closed.\")\n\n    src_arcname = SmartStr(src_arcname)\n    dst_arcname = SmartStr(dst_arcname)\n\n    zinfo = zipfile.ZipInfo(dst_arcname)\n    # This marks a symlink.\n    zinfo.external_attr = (0o644 | 0o120000) << 16\n    # This marks create_system as UNIX.\n    zinfo.create_system = 3\n\n    # This fills the ASi UNIX extra field, see:\n    # http://www.opensource.apple.com/source/zip/zip-6/unzip/unzip/proginfo/extra.fld\n    zinfo.extra = struct.pack(\n        \"<HHIHIHHs\",\n        0x756e,\n        len(src_arcname) + 14,\n        0,  # CRC-32 of the remaining data\n        0o120000,  # file permissions\n        0,  # target file size\n        0,  # user ID\n        0,  # group ID\n        src_arcname)\n\n    self._zip_fd.writestr(zinfo, src_arcname)\n\n    return self._stream.GetValueAndReset()", "response": "Writes a symlink into the archive."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef WriteFileHeader(self, arcname=None, compress_type=None, st=None):\n\n    if not self._stream:\n      raise ArchiveAlreadyClosedError(\n          \"Attempting to write to a ZIP archive that was already closed.\")\n\n    self.cur_zinfo = self._GenerateZipInfo(\n        arcname=arcname, compress_type=compress_type, st=st)\n    self.cur_file_size = 0\n    self.cur_compress_size = 0\n\n    if self.cur_zinfo.compress_type == zipfile.ZIP_DEFLATED:\n      self.cur_cmpr = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION,\n                                       zlib.DEFLATED, -15)\n    else:\n      self.cur_cmpr = None\n\n    self.cur_crc = 0\n\n    if not self._stream:\n      raise ArchiveAlreadyClosedError(\n          \"Attempting to write to a ZIP archive that was already closed.\")\n\n    self.cur_zinfo.header_offset = self._stream.tell()\n    # Call _writeCheck(self.cur_zinfo) to do sanity checking on zinfo structure\n    # that we've constructed.\n    self._zip_fd._writecheck(self.cur_zinfo)  # pylint: disable=protected-access\n    # Mark ZipFile as dirty. We have to keep self._zip_fd's internal state\n    # coherent so that it behaves correctly when close() is called.\n    self._zip_fd._didModify = True  # pylint: disable=protected-access\n\n    # Write FileHeader now. It's incomplete, but CRC and uncompressed/compressed\n    # sized will be written later in data descriptor.\n    self._stream.write(self.cur_zinfo.FileHeader())\n\n    return self._stream.GetValueAndReset()", "response": "Writes a file header."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nwriting a chunk to the archive.", "response": "def WriteFileChunk(self, chunk):\n    \"\"\"Writes file chunk.\"\"\"\n    precondition.AssertType(chunk, bytes)\n\n    if not self._stream:\n      raise ArchiveAlreadyClosedError(\n          \"Attempting to write to a ZIP archive that was already closed.\")\n\n    self.cur_file_size += len(chunk)\n    # TODO(user):pytype: crc32 is not visible outside of zipfile.\n    # pytype: disable=module-attr\n    self.cur_crc = zipfile.crc32(chunk, self.cur_crc) & 0xffffffff\n    # pytype: enable=module-attr\n\n    if self.cur_cmpr:\n      chunk = self.cur_cmpr.compress(chunk)\n      self.cur_compress_size += len(chunk)\n\n    self._stream.write(chunk)\n    return self._stream.GetValueAndReset()"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nwriting the file footer.", "response": "def WriteFileFooter(self):\n    \"\"\"Writes the file footer (finished the file).\"\"\"\n\n    if not self._stream:\n      raise ArchiveAlreadyClosedError(\n          \"Attempting to write to a ZIP archive that was already closed.\")\n\n    if self.cur_cmpr:\n      buf = self.cur_cmpr.flush()\n      self.cur_compress_size += len(buf)\n      self.cur_zinfo.compress_size = self.cur_compress_size\n\n      self._stream.write(buf)\n    else:\n      self.cur_zinfo.compress_size = self.cur_file_size\n\n    self.cur_zinfo.CRC = self.cur_crc\n    self.cur_zinfo.file_size = self.cur_file_size\n\n    # The zip footer has a 8 bytes limit for sizes so if we compress a\n    # file larger than 4 GB, the code below will not work. The ZIP64\n    # convention is to write 0xffffffff for compressed and\n    # uncompressed size in those cases. The actual size is written by\n    # the library for us anyways so those fields are redundant.\n    cur_file_size = min(0xffffffff, self.cur_file_size)\n    cur_compress_size = min(0xffffffff, self.cur_compress_size)\n\n    # Writing data descriptor ZIP64-way by default. We never know how large\n    # the archive may become as we're generating it dynamically.\n    #\n    # crc-32                          8 bytes (little endian)\n    # compressed size                 8 bytes (little endian)\n    # uncompressed size               8 bytes (little endian)\n    self._stream.write(\n        struct.pack(\"<LLL\", self.cur_crc, cur_compress_size, cur_file_size))\n\n    # Register the file in the zip file, so that central directory gets\n    # written correctly.\n    self._zip_fd.filelist.append(self.cur_zinfo)\n    self._zip_fd.NameToInfo[self.cur_zinfo.filename] = self.cur_zinfo\n\n    self._ResetState()\n\n    return self._stream.GetValueAndReset()"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nwrites a zip member from a file - like object.", "response": "def WriteFromFD(self, src_fd, arcname=None, compress_type=None, st=None):\n    \"\"\"Write a zip member from a file like object.\n\n    Args:\n      src_fd: A file like object, must support seek(), tell(), read().\n      arcname: The name in the archive this should take.\n      compress_type: Compression type (zipfile.ZIP_DEFLATED, or ZIP_STORED)\n      st: An optional stat object to be used for setting headers.\n\n    Raises:\n      ArchiveAlreadyClosedError: If the zip if already closed.\n\n    Yields:\n      Chunks of binary data.\n    \"\"\"\n    yield self.WriteFileHeader(\n        arcname=arcname, compress_type=compress_type, st=st)\n    while 1:\n      buf = src_fd.read(1024 * 1024)\n      if not buf:\n        break\n\n      yield self.WriteFileChunk(buf)\n\n    yield self.WriteFileFooter()"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nwrite a symlink into the archive.", "response": "def WriteSymlink(self, src_arcname, dst_arcname):\n    \"\"\"Writes a symlink into the archive.\"\"\"\n\n    info = self._tar_fd.tarinfo()\n    info.tarfile = self._tar_fd\n    info.name = SmartStr(dst_arcname)\n    info.size = 0\n    info.mtime = time.time()\n    info.type = tarfile.SYMTYPE\n    info.linkname = SmartStr(src_arcname)\n\n    self._tar_fd.addfile(info)\n    return self._stream.GetValueAndReset()"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nwrite a chunk to the tar file.", "response": "def WriteFileChunk(self, chunk):\n    \"\"\"Writes file chunk.\"\"\"\n\n    self._tar_fd.fileobj.write(chunk)\n    self.cur_file_size += len(chunk)\n    return self._stream.GetValueAndReset()"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef WriteFileFooter(self):\n\n    if self.cur_file_size != self.cur_info.size:\n      raise IOError(\"Incorrect file size: st_size=%d, but written %d bytes.\" %\n                    (self.cur_info.size, self.cur_file_size))\n\n    # TODO(user):pytype: BLOCKSIZE/NUL constants are not visible to type\n    # checker.\n    # pytype: disable=module-attr\n    blocks, remainder = divmod(self.cur_file_size, tarfile.BLOCKSIZE)\n    if remainder > 0:\n      self._tar_fd.fileobj.write(tarfile.NUL * (tarfile.BLOCKSIZE - remainder))\n      blocks += 1\n    self._tar_fd.offset += blocks * tarfile.BLOCKSIZE\n    # pytype: enable=module-attr\n\n    self._ResetState()\n\n    return self._stream.GetValueAndReset()", "response": "Writes the file footer."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef Get(self):\n\n    args = user_management_pb2.ApiGetGrrUserArgs(username=self.username)\n    data = self._context.SendRequest(\"GetGrrUser\", args)\n    return GrrUser(data=data, context=self._context)", "response": "Fetches user s data and returns it wrapped in a Grruser object."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef Modify(self, user_type=None, password=None):\n\n    args = user_management_pb2.ApiModifyGrrUserArgs(\n        username=self.username, user_type=user_type)\n\n    if user_type is not None:\n      args.user_type = user_type\n\n    if password is not None:\n      args.password = password\n\n    data = self._context.SendRequest(\"ModifyGrrUser\", args)\n    return GrrUser(data=data, context=self._context)", "response": "Modifies user s type and password."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nuploading data from a given stream and signs them with a given key.", "response": "def Upload(self, fd, sign_fn=None):\n    \"\"\"Uploads data from a given stream and signs them with a given key.\"\"\"\n\n    if not sign_fn:\n      raise ValueError(\"sign_fn can't be empty. \"\n                       \"See DefaultUploadSigner as a possible option.\")\n\n    args = binary_management_pb2.ApiUploadGrrBinaryArgs(\n        type=self.binary_type, path=self.path)\n\n    while True:\n      data = fd.read(self.__class__.CHUNK_SIZE)\n      if not data:\n        break\n\n      blob = args.blobs.add()\n\n      blob.signature = sign_fn(data)\n      blob.signature_type = blob.RSA_PKCS1v15\n\n      blob.digest = hashlib.sha256(data).digest()\n      blob.digest_type = blob.SHA256\n\n      blob.data = data\n\n    self._context.SendRequest(\"UploadGrrBinary\", args)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncreating a new GRR user of a given type with a given username and password.", "response": "def CreateGrrUser(self, username=None, user_type=None, password=None):\n    \"\"\"Creates a new GRR user of a given type with a given username/password.\"\"\"\n\n    if not username:\n      raise ValueError(\"Username can't be empty.\")\n\n    args = user_management_pb2.ApiCreateGrrUserArgs(username=username)\n\n    if user_type is not None:\n      args.user_type = user_type\n\n    if password is not None:\n      args.password = password\n\n    data = self._context.SendRequest(\"CreateGrrUser\", args)\n    return GrrUser(data=data, context=self._context)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nlisting all GRR users.", "response": "def ListGrrUsers(self):\n    \"\"\"Lists all registered GRR users.\"\"\"\n\n    args = user_management_pb2.ApiListGrrUsersArgs()\n\n    items = self._context.SendIteratorRequest(\"ListGrrUsers\", args)\n    return utils.MapItemsIterator(\n        lambda data: GrrUser(data=data, context=self._context), items)"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn all audit log entries between now - offset and now.", "response": "def GetAuditLogEntries(offset, now, token):\n  \"\"\"Return all audit log entries between now-offset and now.\n\n  Args:\n    offset: rdfvalue.Duration how far back to look in time\n    now: rdfvalue.RDFDatetime for current time\n    token: GRR access token\n  Yields:\n    AuditEvents created during the time range\n  \"\"\"\n  start_time = now - offset - audit.AUDIT_ROLLOVER_TIME\n\n  for fd in audit.LegacyAuditLogsForTimespan(start_time, now, token):\n    for event in fd.GenerateItems():\n      if now - offset < event.timestamp < now:\n        yield event"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nfetching all clients from the relational db.", "response": "def _IterateAllClients(recency_window=None):\n  \"\"\"Fetches client data from the relational db.\n\n  Args:\n    recency_window: An rdfvalue.Duration specifying a window of last-ping\n      timestamps to consider. Clients that haven't communicated with GRR servers\n      longer than the given period will be skipped. If recency_window is None,\n      all clients will be iterated.\n\n  Yields:\n    Batches (lists) of ClientFullInfo objects.\n  \"\"\"\n  if recency_window is None:\n    min_last_ping = None\n  else:\n    min_last_ping = rdfvalue.RDFDatetime.Now() - recency_window\n  client_ids = data_store.REL_DB.ReadAllClientIDs(min_last_ping=min_last_ping)\n  for client_id_batch in collection.Batch(client_ids, CLIENT_READ_BATCH_SIZE):\n    client_info_dict = data_store.REL_DB.MultiReadClientFullInfo(\n        client_id_batch)\n    yield list(itervalues(client_info_dict))"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef Add(self, category, label, age):\n    now = rdfvalue.RDFDatetime.Now()\n    category = utils.SmartUnicode(category)\n\n    for active_time in self.active_days:\n      self.categories[active_time].setdefault(label, {})\n      if (now - age).seconds < active_time * 24 * 60 * 60:\n        self.categories[active_time][label][\n            category] = self.categories[active_time][label].get(category, 0) + 1", "response": "Adds another instance of this category into the active_days counter."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef Save(self, token=None):\n    graph_series_by_label = {}\n    for active_time in self.active_days:\n      for label in self.categories[active_time]:\n        graphs_for_label = graph_series_by_label.setdefault(\n            label, rdf_stats.ClientGraphSeries(report_type=self._report_type))\n        graph = rdf_stats.Graph(title=\"%s day actives for %s label\" %\n                                (active_time, label))\n        for k, v in sorted(iteritems(self.categories[active_time][label])):\n          graph.Append(label=k, y_value=v)\n        graphs_for_label.graphs.Append(graph)\n\n    for label, graph_series in iteritems(graph_series_by_label):\n      client_report_utils.WriteGraphSeries(graph_series, label, token=token)", "response": "Save the histogram to the specified attribute."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef Run(self):\n    try:\n\n      self.stats = {}\n\n      self.BeginProcessing()\n\n      processed_count = 0\n\n      for client_info_batch in _IterateAllClients(\n          recency_window=self.recency_window):\n        for client_info in client_info_batch:\n          self.ProcessClientFullInfo(client_info)\n        processed_count += len(client_info_batch)\n        self.Log(\"Processed %d clients.\", processed_count)\n        self.HeartBeat()\n\n      self.FinishProcessing()\n      for fd in itervalues(self.stats):\n        fd.Close()\n\n      logging.info(\"%s: processed %d clients.\", self.__class__.__name__,\n                   processed_count)\n    except Exception as e:  # pylint: disable=broad-except\n      logging.exception(\"Error while calculating stats: %s\", e)\n      raise", "response": "Retrieve all the clients for the AbstractClientStatsCollectors."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef Start(self):\n    try:\n\n      self.stats = {}\n\n      self.BeginProcessing()\n\n      processed_count = 0\n\n      if data_store.RelationalDBEnabled():\n        for client_info_batch in _IterateAllClients(\n            recency_window=self.recency_window):\n          for client_info in client_info_batch:\n            self.ProcessClientFullInfo(client_info)\n          processed_count += len(client_info_batch)\n          self.Log(\"Processed %d clients.\", processed_count)\n          self.HeartBeat()\n      else:\n        root_children = aff4.FACTORY.Open(\n            aff4.ROOT_URN, token=self.token).OpenChildren(mode=\"r\")\n        for batch in collection.Batch(root_children, CLIENT_READ_BATCH_SIZE):\n          for child in batch:\n            if not isinstance(child, aff4_grr.VFSGRRClient):\n              continue\n\n            last_ping = child.Get(child.Schema.PING)\n\n            self.ProcessLegacyClient(last_ping, child)\n            processed_count += 1\n            # This flow is not dead: we don't want to run out of lease time.\n            self.HeartBeat()\n\n      self.FinishProcessing()\n      for fd in itervalues(self.stats):\n        fd.Close()\n\n      logging.info(\"%s: processed %d clients.\", self.__class__.__name__,\n                   processed_count)\n    except Exception as e:  # pylint: disable=broad-except\n      logging.exception(\"Error while calculating stats: %s\", e)\n      raise", "response": "Retrieve all the clients for the AbstractClientStatsCollectors."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nprocess a legacy client.", "response": "def ProcessLegacyClient(self, ping, client):\n    \"\"\"Update counters for system, version and release attributes.\"\"\"\n    labels = self._GetClientLabelsList(client)\n    system = client.Get(client.Schema.SYSTEM, \"Unknown\")\n    uname = client.Get(client.Schema.UNAME, \"Unknown\")\n\n    self._Process(labels, ping, system, uname)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nstart an interrogation hunt on all available clients.", "response": "def StartInterrogationHunt(self):\n    \"\"\"Starts an interrogation hunt on all available clients.\"\"\"\n    flow_name = compatibility.GetName(flows_discovery.Interrogate)\n    flow_args = flows_discovery.InterrogateArgs(lightweight=False)\n    description = \"Interrogate run by cron to keep host info fresh.\"\n\n    if data_store.RelationalDBEnabled():\n      hunt_id = hunt.CreateAndStartHunt(\n          flow_name,\n          flow_args,\n          self.token.username,\n          client_limit=0,\n          client_rate=50,\n          crash_limit=config.CONFIG[\"Cron.interrogate_crash_limit\"],\n          description=description,\n          duration=rdfvalue.Duration(\"1w\"),\n          output_plugins=self.GetOutputPlugins())\n      self.Log(\"Started hunt %s.\", hunt_id)\n    else:\n      with hunts_implementation.StartHunt(\n          hunt_name=hunts_standard.GenericHunt.__name__,\n          client_limit=0,\n          flow_runner_args=rdf_flow_runner.FlowRunnerArgs(flow_name=flow_name),\n          flow_args=flow_args,\n          output_plugins=self.GetOutputPlugins(),\n          crash_limit=config.CONFIG[\"Cron.interrogate_crash_limit\"],\n          client_rate=50,\n          expiry_time=rdfvalue.Duration(\"1w\"),\n          description=description,\n          token=self.token) as hunt_obj:\n\n        hunt_obj.GetRunner().Start()\n        self.Log(\"Started hunt %s.\", hunt_obj.urn)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nprocess the client responses.", "response": "def ProcessClients(self, responses):\n    \"\"\"Does the work.\"\"\"\n    del responses\n\n    end = rdfvalue.RDFDatetime.Now() - db.CLIENT_STATS_RETENTION\n    client_urns = export_utils.GetAllClients(token=self.token)\n\n    for batch in collection.Batch(client_urns, 10000):\n      with data_store.DB.GetMutationPool() as mutation_pool:\n        for client_urn in batch:\n          mutation_pool.DeleteAttributes(\n              client_urn.Add(\"stats\"), [u\"aff4:stats\"],\n              start=0,\n              end=end.AsMicrosecondsSinceEpoch())\n      self.HeartBeat()\n\n    if data_store.RelationalDBEnabled():\n      total_deleted_count = 0\n      for deleted_count in data_store.REL_DB.DeleteOldClientStats(\n          yield_after_count=_STATS_DELETION_BATCH_SIZE, retention_time=end):\n        self.HeartBeat()\n        total_deleted_count += deleted_count\n      self.Log(\"Deleted %d ClientStats that expired before %s\",\n               total_deleted_count, end)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef CheckApproversForLabel(self, token, client_urn, requester, approvers,\n                             label):\n    \"\"\"Checks if requester and approvers have approval privileges for labels.\n\n    Checks against list of approvers for each label defined in approvers.yaml to\n    determine if the list of approvers is sufficient.\n\n    Args:\n      token: user token\n      client_urn: ClientURN object of the client\n      requester: username string of person requesting approval.\n      approvers: list of username strings that have approved this client.\n      label: label strings to check approval privs for.\n    Returns:\n      True if access is allowed, raises otherwise.\n    \"\"\"\n    auth = self.reader.GetAuthorizationForSubject(label)\n    if not auth:\n      # This label isn't listed in approvers.yaml\n      return True\n\n    if auth.requester_must_be_authorized:\n      if not self.CheckPermissions(requester, label):\n        raise access_control.UnauthorizedAccess(\n            \"User %s not in %s or groups:%s for %s\" % (requester, auth.users,\n                                                       auth.groups, label),\n            subject=client_urn,\n            requested_access=token.requested_access)\n\n    approved_count = 0\n    for approver in approvers:\n      if self.CheckPermissions(approver, label) and approver != requester:\n        approved_count += 1\n\n    if approved_count < auth.num_approvers_required:\n      raise access_control.UnauthorizedAccess(\n          \"Found %s approvers for %s, needed %s\" %\n          (approved_count, label, auth.num_approvers_required),\n          subject=client_urn,\n          requested_access=token.requested_access)\n    return True", "response": "Checks if requester and approvers have approval privileges for label."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef EnsureTempDirIsSane(directory):\n\n  if not os.path.isabs(directory):\n    raise ErrorBadPath(\"Directory %s is not absolute\" % directory)\n\n  if os.path.isdir(directory):\n    # The temp dir already exists, we probably created it already but\n    # let's check to make sure.\n    if not client_utils.VerifyFileOwner(directory):\n      # Just delete it, it's only temp dirs and we don't own it. If\n      # this goes wrong we just raise.\n      shutil.rmtree(directory)\n\n  if not os.path.isdir(directory):\n    os.makedirs(directory)\n\n    # Make directory 700 before we write the file\n    if sys.platform == \"win32\":\n      from grr_response_client import client_utils_windows  # pylint: disable=g-import-not-at-top\n      client_utils_windows.WinChmod(directory,\n                                    [\"FILE_GENERIC_READ\", \"FILE_GENERIC_WRITE\"])\n    else:\n      os.chmod(directory, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR)", "response": "Checks that the directory exists and has the correct permissions set."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncreating a temporary file with GRR prefix in directory.", "response": "def CreateGRRTempFile(filename=None, lifetime=0, mode=\"w+b\", suffix=\"\"):\n  \"\"\"Open file with GRR prefix in directory to allow easy deletion.\n\n  Missing parent dirs will be created. If an existing directory is specified\n  its permissions won't be modified to avoid breaking system functionality.\n  Permissions on the destination file will be set to root/SYSTEM rw.\n\n  On windows the file is created, then permissions are set.  So there is\n  potentially a race condition where the file is readable by other users.  If\n  the caller doesn't specify a directory on windows we use the directory we are\n  executing from as a safe default.\n\n  If lifetime is specified a housekeeping thread is created to delete the file\n  after lifetime seconds.  Files won't be deleted by default.\n\n  Args:\n    filename: The name of the file to use. Note that setting both filename and\n       directory name is not allowed.\n\n    lifetime: time in seconds before we should delete this tempfile.\n\n    mode: The mode to open the file.\n\n    suffix: optional suffix to use for the temp file\n\n  Returns:\n    Python file object\n\n  Raises:\n    OSError: on permission denied\n    ErrorBadPath: if path is not absolute\n    ValueError: if Client.tempfile_prefix is undefined in the config.\n  \"\"\"\n  directory = GetDefaultGRRTempDirectory()\n\n  EnsureTempDirIsSane(directory)\n\n  prefix = config.CONFIG.Get(\"Client.tempfile_prefix\")\n  if filename is None:\n    outfile = tempfile.NamedTemporaryFile(\n        prefix=prefix, suffix=suffix, dir=directory, delete=False)\n  else:\n    if filename.startswith(\"/\") or filename.startswith(\"\\\\\"):\n      raise ValueError(\"Filename must be relative\")\n\n    if suffix:\n      filename = \"%s.%s\" % (filename, suffix)\n\n    outfile = open(os.path.join(directory, filename), mode)\n\n  if lifetime > 0:\n    cleanup = threading.Timer(lifetime, DeleteGRRTempFile, (outfile.name,))\n    cleanup.start()\n\n  # Fix perms on the file, since this code is used for writing executable blobs\n  # we apply RWX.\n  if sys.platform == \"win32\":\n    from grr_response_client import client_utils_windows  # pylint: disable=g-import-not-at-top\n    client_utils_windows.WinChmod(outfile.name, [\"FILE_ALL_ACCESS\"])\n  else:\n    os.chmod(outfile.name, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR)\n\n  return outfile"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncreates a GRR VFS temp file.", "response": "def CreateGRRTempFileVFS(filename=None, lifetime=0, mode=\"w+b\", suffix=\"\"):\n  \"\"\"Creates a GRR VFS temp file.\n\n  This function is analogous to CreateGRRTempFile but returns an open VFS handle\n  to the newly created file. Arguments are the same as for CreateGRRTempFile:\n\n  Args:\n    filename: The name of the file to use. Note that setting both filename and\n       directory name is not allowed.\n\n    lifetime: time in seconds before we should delete this tempfile.\n\n    mode: The mode to open the file.\n\n    suffix: optional suffix to use for the temp file\n\n  Returns:\n    An open file handle to the new file and the corresponding pathspec.\n  \"\"\"\n\n  fd = CreateGRRTempFile(\n      filename=filename, lifetime=lifetime, mode=mode, suffix=suffix)\n  pathspec = rdf_paths.PathSpec(\n      path=fd.name, pathtype=rdf_paths.PathSpec.PathType.TMPFILE)\n  return fd, pathspec"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncheck if given path is valid for deletion.", "response": "def _CheckIfPathIsValidForDeletion(path, prefix=None, directories=None):\n  \"\"\"Checks if given path is valid for deletion.\"\"\"\n  precondition.AssertType(path, Text)\n  precondition.AssertType(prefix, Text)\n\n  if prefix and os.path.basename(path).startswith(prefix):\n    return True\n\n  path = path.lower()\n  for directory in directories or []:\n    directory = directory.lower()\n    if os.path.commonprefix([directory, path]) == directory:\n      return True\n  return False"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ndelete a GRR temp file.", "response": "def DeleteGRRTempFile(path):\n  \"\"\"Delete a GRR temp file.\n\n  To limit possible damage the path must be absolute and either the\n  file must be within any of the Client.tempdir_roots or the file name\n  must begin with Client.tempfile_prefix.\n\n  Args:\n    path: path string to file to be deleted.\n\n  Raises:\n    OSError: Permission denied, or file not found.\n    ErrorBadPath: Path must be absolute.\n    ErrorNotTempFile: Filename must start with Client.tempfile_prefix.\n    ErrorNotAFile: File to delete does not exist.\n  \"\"\"\n  precondition.AssertType(path, Text)\n\n  if not os.path.isabs(path):\n    raise ErrorBadPath(\"Path must be absolute\")\n\n  prefix = config.CONFIG[\"Client.tempfile_prefix\"]\n  directories = [\n      GetTempDirForRoot(root) for root in config.CONFIG[\"Client.tempdir_roots\"]\n  ]\n  if not _CheckIfPathIsValidForDeletion(\n      path, prefix=prefix, directories=directories):\n    msg = (\"Can't delete temp file %s. Filename must start with %s \"\n           \"or lie within any of %s.\")\n    raise ErrorNotTempFile(msg % (path, prefix, \";\".join(directories)))\n\n  if os.path.exists(path):\n    # Clear our file handle cache so the file can be deleted.\n    files.FILE_HANDLE_CACHE.Flush()\n    os.remove(path)\n  else:\n    raise ErrorNotAFile(\"%s does not exist.\" % path)"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ndelete all GRR temp files in path.", "response": "def Run(self, args):\n    \"\"\"Delete all the GRR temp files in path.\n\n    If path is a directory, look in the top level for filenames beginning with\n    Client.tempfile_prefix, and delete them.\n\n    If path is a regular file and starts with Client.tempfile_prefix delete it.\n\n    Args:\n      args: pathspec pointing to directory containing temp files to be\n            deleted, or a single file to be deleted.\n    Returns:\n      deleted: array of filename strings that were deleted\n    Raises:\n      ErrorBadPath: if path doesn't exist or is not a regular file or directory\n    \"\"\"\n\n    allowed_temp_dirs = [\n        GetTempDirForRoot(root)\n        for root in config.CONFIG[\"Client.tempdir_roots\"]\n    ]\n\n    if args.path:\n      # Normalize the path, so DeleteGRRTempFile can correctly check if\n      # it is within Client.tempdir.\n      path = utils.NormalizePath(args.path)\n      if platform.system() == \"Windows\":\n        # TODO: On non-Windows systems `CanonicalPathToLocalPath`\n        # is equivalent to `SmartStr`, so it does nothing except for breaking\n        # the types. However, a lot of code actually depends on this behaviour\n        # so we cannot easily change it. As a workaround for now we simply do\n        # not call it on Linux and macOS but ideally we should get rid of this\n        # `SmartStr` call and not branch here.\n        path = client_utils.CanonicalPathToLocalPath(path)\n\n      paths = [path]\n    else:\n      paths = allowed_temp_dirs\n\n    deleted = []\n    errors = []\n    for path in paths:\n      if os.path.isdir(path):\n        for filename in os.listdir(path):\n          abs_filename = os.path.join(path, filename)\n\n          try:\n            DeleteGRRTempFile(abs_filename)\n            deleted.append(abs_filename)\n          except Exception as e:  # pylint: disable=broad-except\n            # The error we are most likely to get is ErrorNotTempFile but\n            # especially on Windows there might be locking issues that raise\n            # various WindowsErrors so we just catch them all and continue\n            # deleting all other temp files in this directory.\n            errors.append(e)\n\n      elif os.path.isfile(path):\n        DeleteGRRTempFile(path)\n        deleted = [path]\n\n      elif path not in allowed_temp_dirs:\n        if not os.path.exists(path):\n          raise ErrorBadPath(\"File %s does not exist\" % path)\n        else:\n          raise ErrorBadPath(\"Not a regular file or directory: %s\" % path)\n\n    reply = \"\"\n    if deleted:\n      reply = \"Deleted: %s.\" % deleted\n    else:\n      reply = \"Nothing deleted.\"\n    if errors:\n      reply += \"\\n%s\" % errors\n\n    self.SendReply(rdf_client.LogMessage(data=reply))"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nissuing a request to list the directory.", "response": "def Start(self):\n    \"\"\"Issue a request to list the directory.\"\"\"\n    self.CallClient(\n        server_stubs.PlistQuery,\n        request=self.args.request,\n        next_state=\"Receive\")"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef GetRequestFormatMode(request, method_metadata):\n  if request.path.startswith(\"/api/v2/\"):\n    return JsonMode.PROTO3_JSON_MODE\n\n  if request.args.get(\"strip_type_info\", \"\"):\n    return JsonMode.GRR_TYPE_STRIPPED_JSON_MODE\n\n  for http_method, unused_url, options in method_metadata.http_methods:\n    if (http_method == request.method and\n        options.get(\"strip_root_types\", False)):\n      return JsonMode.GRR_ROOT_TYPES_STRIPPED_JSON_MODE\n\n  return JsonMode.GRR_JSON_MODE", "response": "Returns the JSON format mode corresponding to a given request and method."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef RenderHttpResponse(request):\n\n  start_time = time.time()\n  response = HTTP_REQUEST_HANDLER.HandleRequest(request)\n  total_time = time.time() - start_time\n\n  method_name = response.headers.get(\"X-API-Method\", \"unknown\")\n  if response.status_code == 200:\n    status = \"SUCCESS\"\n  elif response.status_code == 403:\n    status = \"FORBIDDEN\"\n  elif response.status_code == 404:\n    status = \"NOT_FOUND\"\n  elif response.status_code == 501:\n    status = \"NOT_IMPLEMENTED\"\n  else:\n    status = \"SERVER_ERROR\"\n\n  if request.method == \"HEAD\":\n    metric_name = \"api_access_probe_latency\"\n  else:\n    metric_name = \"api_method_latency\"\n\n  stats_collector_instance.Get().RecordEvent(\n      metric_name, total_time, fields=(method_name, \"http\", status))\n\n  return response", "response": "Renders a HTTP response to a given HTTP request."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _BuildHttpRoutingMap(self, router_cls):\n\n    if not issubclass(router_cls, api_call_router.ApiCallRouter):\n      raise ValueError(\"Router has to be an instance of ApiCallRouter.\")\n\n    routing_map = routing.Map()\n    # Note: we traverse methods of the base class (ApiCallRouter) to avoid\n    # potential problems caused by child router classes using the @Http\n    # annotation (thus adding additional unforeseen HTTP paths/methods). We\n    # don't want the HTTP API to depend on a particular router implementation.\n    for _, metadata in iteritems(router_cls.GetAnnotatedMethods()):\n      for http_method, path, unused_options in metadata.http_methods:\n        routing_map.add(\n            routing.Rule(path, methods=[http_method], endpoint=metadata))\n        # This adds support for the next version of the API that uses\n        # standartized JSON protobuf serialization.\n        routing_map.add(\n            routing.Rule(\n                path.replace(\"/api/\", \"/api/v2/\"),\n                methods=[http_method],\n                endpoint=metadata))\n\n    return routing_map", "response": "Builds a werkzeug routing map out of a given router class."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning a routing map for a given router instance.", "response": "def _GetRoutingMap(self, router):\n    \"\"\"Returns a routing map for a given router instance.\"\"\"\n\n    try:\n      routing_map = self._routing_maps_cache.Get(router.__class__)\n    except KeyError:\n      routing_map = self._BuildHttpRoutingMap(router.__class__)\n      self._routing_maps_cache.Put(router.__class__, routing_map)\n\n    return routing_map"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _SetField(self, args, type_info, value):\n    if hasattr(type_info, \"enum\"):\n      try:\n        coerced_obj = type_info.enum[value.upper()]\n      except KeyError:\n        # A bool is an enum but serializes to \"1\" / \"0\" which are both not in\n        # enum or reverse_enum.\n        coerced_obj = type_info.type.FromHumanReadable(value)\n    else:\n      coerced_obj = type_info.type.FromHumanReadable(value)\n    args.Set(type_info.name, coerced_obj)", "response": "Sets the field on the arg rdfvalue object."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _GetArgsFromRequest(self, request, method_metadata, route_args):\n    format_mode = GetRequestFormatMode(request, method_metadata)\n\n    if request.method in [\"GET\", \"HEAD\"]:\n      if method_metadata.args_type:\n        unprocessed_request = request.args\n        if hasattr(unprocessed_request, \"dict\"):\n          unprocessed_request = unprocessed_request.dict()\n\n        args = method_metadata.args_type()\n        for type_info in args.type_infos:\n          if type_info.name in route_args:\n            self._SetField(args, type_info, route_args[type_info.name])\n          elif type_info.name in unprocessed_request:\n            self._SetField(args, type_info, unprocessed_request[type_info.name])\n\n      else:\n        args = None\n    elif request.method in [\"POST\", \"DELETE\", \"PATCH\"]:\n      try:\n        args = method_metadata.args_type()\n        for type_info in args.type_infos:\n          if type_info.name in route_args:\n            self._SetField(args, type_info, route_args[type_info.name])\n\n        if request.content_type and request.content_type.startswith(\n            \"multipart/form-data;\"):\n          payload = json.Parse(request.form[\"_params_\"].decode(\"utf-8\"))\n          args.FromDict(payload)\n\n          for name, fd in iteritems(request.files):\n            args.Set(name, fd.read())\n        elif format_mode == JsonMode.PROTO3_JSON_MODE:\n          # NOTE: Arguments rdfvalue has to be a protobuf-based RDFValue.\n          args_proto = args.protobuf()\n          json_format.Parse(request.get_data(as_text=True) or \"{}\", args_proto)\n          args.ParseFromString(args_proto.SerializeToString())\n        else:\n          json_data = request.get_data(as_text=True) or \"{}\"\n          payload = json.Parse(json_data)\n          if payload:\n            args.FromDict(payload)\n      except Exception as e:  # pylint: disable=broad-except\n        logging.exception(\"Error while parsing POST request %s (%s): %s\",\n                          request.path, request.method, e)\n        raise PostRequestParsingError(e)\n    else:\n      raise UnsupportedHttpMethod(\"Unsupported method: %s.\" % request.method)\n\n    return args", "response": "Builds the args struct from the request."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef MatchRouter(self, request):\n    router = api_auth_manager.API_AUTH_MGR.GetRouterForUser(request.user)\n    routing_map = self._GetRoutingMap(router)\n\n    matcher = routing_map.bind(\n        \"%s:%s\" %\n        (request.environ[\"SERVER_NAME\"], request.environ[\"SERVER_PORT\"]))\n    try:\n      match = matcher.match(request.path, request.method)\n    except werkzeug_exceptions.NotFound:\n      raise ApiCallRouterNotFoundError(\"No API router was found for (%s) %s\" %\n                                       (request.path, request.method))\n\n    router_method_metadata, route_args_dict = match\n    return (router, router_method_metadata,\n            self._GetArgsFromRequest(request, router_method_metadata,\n                                     route_args_dict))", "response": "Returns a router for a given HTTP request."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef BuildToken(request, execution_time):\n\n    # The request.args dictionary will also be filled on HEAD calls.\n    if request.method in [\"GET\", \"HEAD\"]:\n      reason = request.args.get(\"reason\", \"\")\n    elif request.method in [\"POST\", \"DELETE\", \"PATCH\"]:\n      # The header X-GRR-Reason is set in api-service.js.\n      reason = utils.SmartUnicode(\n          urlparse.unquote(request.headers.get(\"X-Grr-Reason\", \"\")))\n\n    # We assume that request.user contains the username that we can trust.\n    # No matter what authentication method is used, the WebAuthManager is\n    # responsible for authenticating the userand setting request.user to\n    # a correct value (see gui/webauth.py).\n    #\n    # The token that's built here will be later used to find an API router,\n    # get the ApiCallHandler from the router, and then to call the handler's\n    # Handle() method. API router will be responsible for all the ACL checks.\n    token = access_control.ACLToken(\n        username=request.user,\n        reason=reason,\n        process=\"GRRAdminUI\",\n        expiry=rdfvalue.RDFDatetime.Now() + execution_time)\n\n    for field in [\"Remote_Addr\", \"X-Forwarded-For\"]:\n      remote_addr = request.headers.get(field, \"\")\n      if remote_addr:\n        token.source_ips.append(remote_addr)\n    return token", "response": "Builds an ACLToken from the request."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nhandling an API call to a given handler with given args and token.", "response": "def CallApiHandler(handler, args, token=None):\n    \"\"\"Handles API call to a given handler with given args and token.\"\"\"\n\n    result = handler.Handle(args, token=token)\n\n    expected_type = handler.result_type\n    if expected_type is None:\n      expected_type = None.__class__\n\n    if result.__class__ != expected_type:\n      raise UnexpectedResultTypeError(\n          \"Expected %s, but got %s.\" %\n          (expected_type.__name__, result.__class__.__name__))\n\n    return result"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _BuildResponse(self,\n                     status,\n                     rendered_data,\n                     method_name=None,\n                     headers=None,\n                     content_length=None,\n                     token=None,\n                     no_audit_log=False):\n    \"\"\"Builds HTTPResponse object from rendered data and HTTP status.\"\"\"\n\n    # To avoid IE content sniffing problems, escape the tags. Otherwise somebody\n    # may send a link with malicious payload that will be opened in IE (which\n    # does content sniffing and doesn't respect Content-Disposition header) and\n    # IE will treat the document as html and executre arbitrary JS that was\n    # passed with the payload.\n    str_data = json.Dump(\n        rendered_data, encoder=JSONEncoderWithRDFPrimitivesSupport)\n    # XSSI protection and tags escaping\n    rendered_data = \")]}'\\n\" + str_data.replace(\"<\", r\"\\u003c\").replace(\n        \">\", r\"\\u003e\")\n\n    response = werkzeug_wrappers.Response(\n        rendered_data,\n        status=status,\n        content_type=\"application/json; charset=utf-8\")\n    response.headers[\n        \"Content-Disposition\"] = \"attachment; filename=response.json\"\n    response.headers[\"X-Content-Type-Options\"] = \"nosniff\"\n\n    if token and token.reason:\n      response.headers[\"X-GRR-Reason\"] = utils.SmartStr(token.reason)\n    if method_name:\n      response.headers[\"X-API-Method\"] = method_name\n    if no_audit_log:\n      response.headers[\"X-No-Log\"] = \"True\"\n\n    for key, value in iteritems(headers or {}):\n      response.headers[key] = value\n\n    if content_length is not None:\n      response.content_length = content_length\n\n    return response", "response": "Builds a HTTPResponse object from rendered data and HTTP status."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nbuilding HTTPResponse object for streaming.", "response": "def _BuildStreamingResponse(self, binary_stream, method_name=None):\n    \"\"\"Builds HTTPResponse object for streaming.\"\"\"\n    precondition.AssertType(method_name, Text)\n\n    # We get a first chunk of the output stream. This way the likelihood\n    # of catching an exception that may happen during response generation\n    # is much higher.\n    content = binary_stream.GenerateContent()\n    try:\n      peek = content.next()\n      stream = itertools.chain([peek], content)\n    except StopIteration:\n      stream = []\n\n    response = werkzeug_wrappers.Response(\n        response=stream,\n        content_type=\"binary/octet-stream\",\n        direct_passthrough=True)\n    response.headers[\"Content-Disposition\"] = ((\n        \"attachment; filename=%s\" % binary_stream.filename).encode(\"utf-8\"))\n    if method_name:\n      response.headers[\"X-API-Method\"] = method_name.encode(\"utf-8\")\n\n    if binary_stream.content_length:\n      response.content_length = binary_stream.content_length\n\n    return response"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef HandleRequest(self, request):\n    impersonated_username = config.CONFIG[\"AdminUI.debug_impersonate_user\"]\n    if impersonated_username:\n      logging.info(\"Overriding user as %s\", impersonated_username)\n      request.user = config.CONFIG[\"AdminUI.debug_impersonate_user\"]\n\n    if not aff4_users.GRRUser.IsValidUsername(request.user):\n      return self._BuildResponse(\n          403, dict(message=\"Invalid username: %s\" % request.user))\n\n    try:\n      router, method_metadata, args = self._router_matcher.MatchRouter(request)\n    except access_control.UnauthorizedAccess as e:\n      error_message = str(e)\n      logging.exception(\"Access denied to %s (%s): %s\", request.path,\n                        request.method, e)\n\n      additional_headers = {\n          \"X-GRR-Unauthorized-Access-Reason\": error_message.replace(\"\\n\", \"\"),\n          \"X-GRR-Unauthorized-Access-Subject\": utils.SmartStr(e.subject)\n      }\n      return self._BuildResponse(\n          403,\n          dict(\n              message=\"Access denied by ACL: %s\" % error_message,\n              subject=utils.SmartStr(e.subject)),\n          headers=additional_headers)\n\n    except ApiCallRouterNotFoundError as e:\n      error_message = str(e)\n      return self._BuildResponse(404, dict(message=error_message))\n    except werkzeug_exceptions.MethodNotAllowed as e:\n      error_message = str(e)\n      return self._BuildResponse(405, dict(message=error_message))\n    except Error as e:\n      logging.exception(\"Can't match URL to router/method: %s\", e)\n\n      return self._BuildResponse(\n          500, dict(message=str(e), traceBack=traceback.format_exc()))\n\n    request.method_metadata = method_metadata\n    request.parsed_args = args\n\n    # SetUID() is called here so that ACL checks done by the router do not\n    # clash with datastore ACL checks.\n    # TODO(user): increase token expiry time.\n    token = self.BuildToken(request, 60).SetUID()\n\n    if data_store.AFF4Enabled():\n      # TODO:\n      # AFF4 edge case: if a user is issuing a request, before they are created\n      # using CreateGRRUser (e.g. in E2E tests or with single sign-on),\n      # AFF4's ReadGRRUsers will NEVER contain the user, because the creation\n      # done in the following lines does not add the user to the /users/\n      # collection. Furthermore, subsequent CreateGrrUserHandler calls fail,\n      # because the user technically already exists.\n\n      # We send a blind-write request to ensure that the user object is created\n      # for a user specified by the username.\n      user_urn = rdfvalue.RDFURN(\"aff4:/users/\").Add(request.user)\n      # We can't use conventional AFF4 interface, since aff4.FACTORY.Create will\n      # create a new version of the object for every call.\n      with data_store.DB.GetMutationPool() as pool:\n        pool.MultiSet(\n            user_urn, {\n                aff4_users.GRRUser.SchemaCls.TYPE:\n                    [aff4_users.GRRUser.__name__],\n                aff4_users.GRRUser.SchemaCls.LAST:\n                    [rdfvalue.RDFDatetime.Now().SerializeToDataStore()]\n            },\n            replace=True)\n\n    if data_store.RelationalDBEnabled():\n      data_store.REL_DB.WriteGRRUser(request.user)\n\n    handler = None\n    try:\n      # ACL checks are done here by the router. If this method succeeds (i.e.\n      # does not raise), then handlers run without further ACL checks (they're\n      # free to do some in their own implementations, though).\n      handler = getattr(router, method_metadata.name)(args, token=token)\n\n      if handler.args_type != method_metadata.args_type:\n        raise RuntimeError(\"Handler args type doesn't match \"\n                           \"method args type: %s vs %s\" %\n                           (handler.args_type, method_metadata.args_type))\n\n      binary_result_type = (\n          api_call_router.RouterMethodMetadata.BINARY_STREAM_RESULT_TYPE)\n\n      if (handler.result_type != method_metadata.result_type and\n          not (handler.result_type is None and\n               method_metadata.result_type == binary_result_type)):\n        raise RuntimeError(\"Handler result type doesn't match \"\n                           \"method result type: %s vs %s\" %\n                           (handler.result_type, method_metadata.result_type))\n\n      # HEAD method is only used for checking the ACLs for particular API\n      # methods.\n      if request.method == \"HEAD\":\n        # If the request would return a stream, we add the Content-Length\n        # header to the response.\n        if (method_metadata.result_type ==\n            method_metadata.BINARY_STREAM_RESULT_TYPE):\n          binary_stream = handler.Handle(args, token=token)\n          return self._BuildResponse(\n              200, {\"status\": \"OK\"},\n              method_name=method_metadata.name,\n              no_audit_log=method_metadata.no_audit_log_required,\n              content_length=binary_stream.content_length,\n              token=token)\n        else:\n          return self._BuildResponse(\n              200, {\"status\": \"OK\"},\n              method_name=method_metadata.name,\n              no_audit_log=method_metadata.no_audit_log_required,\n              token=token)\n\n      if (method_metadata.result_type ==\n          method_metadata.BINARY_STREAM_RESULT_TYPE):\n        binary_stream = handler.Handle(args, token=token)\n        return self._BuildStreamingResponse(\n            binary_stream, method_name=method_metadata.name)\n      else:\n        format_mode = GetRequestFormatMode(request, method_metadata)\n        result = self.CallApiHandler(handler, args, token=token)\n        rendered_data = self._FormatResultAsJson(\n            result, format_mode=format_mode)\n\n        return self._BuildResponse(\n            200,\n            rendered_data,\n            method_name=method_metadata.name,\n            no_audit_log=method_metadata.no_audit_log_required,\n            token=token)\n    except access_control.UnauthorizedAccess as e:\n      error_message = str(e)\n      logging.warning(\"Access denied for %s (HTTP %s %s): %s\",\n                      method_metadata.name, request.method, request.path,\n                      error_message)\n\n      additional_headers = {\n          \"X-GRR-Unauthorized-Access-Reason\": error_message.replace(\"\\n\", \"\"),\n          \"X-GRR-Unauthorized-Access-Subject\": utils.SmartStr(e.subject)\n      }\n      return self._BuildResponse(\n          403,\n          dict(\n              message=\"Access denied by ACL: %s\" % error_message,\n              subject=utils.SmartStr(e.subject)),\n          headers=additional_headers,\n          method_name=method_metadata.name,\n          no_audit_log=method_metadata.no_audit_log_required,\n          token=token)\n    except api_call_handler_base.ResourceNotFoundError as e:\n      error_message = str(e)\n      return self._BuildResponse(\n          404,\n          dict(message=error_message),\n          method_name=method_metadata.name,\n          no_audit_log=method_metadata.no_audit_log_required,\n          token=token)\n    except NotImplementedError as e:\n      error_message = str(e)\n      return self._BuildResponse(\n          501,\n          dict(message=error_message),\n          method_name=method_metadata.name,\n          no_audit_log=method_metadata.no_audit_log_required,\n          token=token)\n    except Exception as e:  # pylint: disable=broad-except\n      error_message = str(e)\n      logging.exception(\"Error while processing %s (%s) with %s: %s\",\n                        request.path, request.method,\n                        handler.__class__.__name__, e)\n      return self._BuildResponse(\n          500,\n          dict(message=error_message, traceBack=traceback.format_exc()),\n          method_name=method_metadata.name,\n          no_audit_log=method_metadata.no_audit_log_required,\n          token=token)", "response": "Handles given HTTP request."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns the provided token or the default token.", "response": "def GetDefaultToken(token):\n  \"\"\"Returns the provided token or the default token.\n\n  Args:\n    token: A token or None.\n\n  Raises:\n    access_control.UnauthorizedAccess: no token was provided.\n  \"\"\"\n  if token is None:\n    token = default_token\n\n  if not isinstance(token, access_control.ACLToken):\n    raise access_control.UnauthorizedAccess(\n        \"Token is not properly specified. It should be an \"\n        \"instance of grr.lib.access_control.ACLToken()\")\n\n  return token"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nflushing all the operations in the pool.", "response": "def Flush(self):\n    \"\"\"Flushing actually applies all the operations in the pool.\"\"\"\n    DB.DeleteSubjects(self.delete_subject_requests, sync=False)\n\n    for req in self.delete_attributes_requests:\n      subject, attributes, start, end = req\n      DB.DeleteAttributes(subject, attributes, start=start, end=end, sync=False)\n\n    for req in self.set_requests:\n      subject, values, timestamp, replace, to_delete = req\n      DB.MultiSet(\n          subject,\n          values,\n          timestamp=timestamp,\n          replace=replace,\n          to_delete=to_delete,\n          sync=False)\n\n    if (self.delete_subject_requests or self.delete_attributes_requests or\n        self.set_requests):\n      DB.Flush()\n\n    for queue, notifications in self.new_notifications:\n      DB.CreateNotifications(queue, notifications)\n    self.new_notifications = []\n\n    self.delete_subject_requests = []\n    self.set_requests = []\n    self.delete_attributes_requests = []"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef QueueClaimRecords(self,\n                        queue_id,\n                        item_rdf_type,\n                        limit=10000,\n                        timeout=\"30m\",\n                        start_time=None,\n                        record_filter=lambda x: False,\n                        max_filtered=1000):\n    \"\"\"Claims records from a queue. See server/aff4_objects/queue.py.\"\"\"\n    now = rdfvalue.RDFDatetime.Now()\n    expiration = rdfvalue.RDFDatetime.Now() + rdfvalue.Duration(timeout)\n\n    after_urn = None\n    if start_time:\n      after_urn, _, _ = DataStore.CollectionMakeURN(\n          queue_id, start_time.AsMicrosecondsSinceEpoch(), 0, subpath=\"Records\")\n    results = []\n\n    filtered_count = 0\n\n    for subject, values in DB.ScanAttributes(\n        str(queue_id.Add(\"Records\")),\n        [DataStore.COLLECTION_ATTRIBUTE, DataStore.QUEUE_LOCK_ATTRIBUTE],\n        max_records=4 * limit,\n        after_urn=after_urn):\n      if DataStore.COLLECTION_ATTRIBUTE not in values:\n        # Unlikely case, but could happen if, say, a thread called RefreshClaims\n        # so late that another thread already deleted the record. Go ahead and\n        # clean this up.\n        self.DeleteAttributes(subject, [DataStore.QUEUE_LOCK_ATTRIBUTE])\n        continue\n      if DataStore.QUEUE_LOCK_ATTRIBUTE in values:\n        timestamp = rdfvalue.RDFDatetime.FromMicrosecondsSinceEpoch(\n            values[DataStore.QUEUE_LOCK_ATTRIBUTE][1])\n        if timestamp > now:\n          continue\n      rdf_value = item_rdf_type.FromSerializedString(\n          values[DataStore.COLLECTION_ATTRIBUTE][1])\n      if record_filter(rdf_value):\n        filtered_count += 1\n        if max_filtered and filtered_count >= max_filtered:\n          break\n        continue\n      results.append(\n          Record(\n              queue_id=queue_id,\n              timestamp=values[DataStore.COLLECTION_ATTRIBUTE][0],\n              suffix=int(subject[-6:], 16),\n              subpath=\"Records\",\n              value=rdf_value))\n      self.Set(subject, DataStore.QUEUE_LOCK_ATTRIBUTE, expiration)\n\n      filtered_count = 0\n      if len(results) >= limit:\n        break\n\n    return results", "response": "Claims records from a queue. See server / aff4_objects. py."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef QueueDeleteTasks(self, queue, tasks):\n    predicates = []\n    for task in tasks:\n      task_id = getattr(task, \"task_id\", None) or int(task)\n      predicates.append(DataStore.QueueTaskIdToColumn(task_id))\n    self.DeleteAttributes(queue, predicates)", "response": "Removes the given tasks from the given queue."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning a list of GrrMessage objects leased for a certain time.", "response": "def QueueQueryAndOwn(self, queue, lease_seconds, limit, timestamp):\n    \"\"\"Returns a list of Tasks leased for a certain time.\n\n    Args:\n      queue: The queue to query from.\n      lease_seconds: The tasks will be leased for this long.\n      limit: Number of values to fetch.\n      timestamp: Range of times for consideration.\n\n    Returns:\n        A list of GrrMessage() objects leased.\n    \"\"\"\n    # Do the real work in a transaction\n    try:\n      lock = DB.LockRetryWrapper(queue, lease_time=lease_seconds)\n      return self._QueueQueryAndOwn(\n          lock.subject,\n          lease_seconds=lease_seconds,\n          limit=limit,\n          timestamp=timestamp)\n    except DBSubjectLockError:\n      # This exception just means that we could not obtain the lock on the queue\n      # so we just return an empty list, let the worker sleep and come back to\n      # fetch more tasks.\n      return []\n    except Error as e:\n      logging.warning(\"Datastore exception: %s\", e)\n      return []"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _QueueQueryAndOwn(self,\n                        subject,\n                        lease_seconds=100,\n                        limit=1,\n                        timestamp=None):\n    \"\"\"Business logic helper for QueueQueryAndOwn().\"\"\"\n    tasks = []\n\n    lease = int(lease_seconds * 1e6)\n\n    # Only grab attributes with timestamps in the past.\n    delete_attrs = set()\n    serialized_tasks_dict = {}\n    for predicate, task, timestamp in DB.ResolvePrefix(\n        subject,\n        DataStore.QUEUE_TASK_PREDICATE_PREFIX,\n        timestamp=(0, timestamp or rdfvalue.RDFDatetime.Now())):\n      task = rdf_flows.GrrMessage.FromSerializedString(task)\n      task.leased_until = timestamp\n      task.leased_by = utils.ProcessIdString()\n      # Decrement the ttl\n      task.task_ttl -= 1\n      if task.task_ttl <= 0:\n        # Remove the task if ttl is exhausted.\n        delete_attrs.add(predicate)\n        stats_collector_instance.Get().IncrementCounter(\n            \"grr_task_ttl_expired_count\")\n      else:\n        if task.task_ttl != rdf_flows.GrrMessage.max_ttl - 1:\n          stats_collector_instance.Get().IncrementCounter(\n              \"grr_task_retransmission_count\")\n\n        serialized_tasks_dict.setdefault(predicate,\n                                         []).append(task.SerializeToString())\n        tasks.append(task)\n        if len(tasks) >= limit:\n          break\n\n    if delete_attrs or serialized_tasks_dict:\n      # Update the timestamp on claimed tasks to be in the future and decrement\n      # their TTLs, delete tasks with expired ttls.\n      self.MultiSet(\n          subject,\n          serialized_tasks_dict,\n          replace=True,\n          timestamp=int(time.time() * 1e6) + lease,\n          to_delete=delete_attrs)\n\n    if delete_attrs:\n      logging.info(\"TTL exceeded for %d messages on queue %s\",\n                   len(delete_attrs), subject)\n    return tasks", "response": "Internal helper for QueueQueryAndOwn."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nadd a child to the specified parent.", "response": "def AFF4AddChild(self, subject, child, extra_attributes=None):\n    \"\"\"Adds a child to the specified parent.\"\"\"\n    precondition.AssertType(child, Text)\n\n    attributes = {\n        DataStore.AFF4_INDEX_DIR_TEMPLATE %\n        child: [DataStore.EMPTY_DATA_PLACEHOLDER]\n    }\n    if extra_attributes:\n      attributes.update(extra_attributes)\n    self.MultiSet(subject, attributes)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef InitializeMonitorThread(self):\n    if self.monitor_thread:\n      return\n    self.monitor_thread = utils.InterruptableThread(\n        name=\"DataStore monitoring thread\",\n        target=self._RegisterSize,\n        sleep_time=60)\n    self.monitor_thread.start()", "response": "Start the monitor thread that registers the size of the DataStore."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef DeleteSubjects(self, subjects, sync=False):\n    for subject in subjects:\n      self.DeleteSubject(subject, sync=sync)", "response": "Delete multiple subjects at once."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef LockRetryWrapper(self,\n                       subject,\n                       retrywrap_timeout=1,\n                       retrywrap_max_timeout=10,\n                       blocking=True,\n                       lease_time=None):\n    \"\"\"Retry a DBSubjectLock until it succeeds.\n\n    Args:\n      subject: The subject which the lock applies to.\n      retrywrap_timeout: How long to wait before retrying the lock.\n      retrywrap_max_timeout: The maximum time to wait for a retry until we\n        raise.\n      blocking: If False, raise on first lock failure.\n      lease_time: lock lease time in seconds.\n\n    Returns:\n      The DBSubjectLock object\n\n    Raises:\n      DBSubjectLockError: If the maximum retry count has been reached.\n    \"\"\"\n    timeout = 0\n    while timeout < retrywrap_max_timeout:\n      try:\n        return self.DBSubjectLock(subject, lease_time=lease_time)\n      except DBSubjectLockError:\n        if not blocking:\n          raise\n        stats_collector_instance.Get().IncrementCounter(\"datastore_retries\")\n        time.sleep(retrywrap_timeout)\n        timeout += retrywrap_timeout\n\n    raise DBSubjectLockError(\"Retry number exceeded.\")", "response": "Retry a DBSubjectLock until it succeeds."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nsetting multiple attributes values for this subject in one operation.", "response": "def MultiSet(self,\n               subject,\n               values,\n               timestamp=None,\n               replace=True,\n               sync=True,\n               to_delete=None):\n    \"\"\"Set multiple attributes' values for this subject in one operation.\n\n    Args:\n      subject: The subject this applies to.\n      values: A dict with keys containing attributes and values, serializations\n        to be set. values can be a tuple of (value, timestamp). Value must be\n        one of the supported types.\n      timestamp: The timestamp for this entry in microseconds since the epoch.\n        None means now.\n      replace: Bool whether or not to overwrite current records.\n      sync: If true we block until the operation completes.\n      to_delete: An array of attributes to clear prior to setting.\n    \"\"\""}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef MultiDeleteAttributes(self,\n                            subjects,\n                            attributes,\n                            start=None,\n                            end=None,\n                            sync=True):\n    \"\"\"Remove all specified attributes from a list of subjects.\n\n    Args:\n      subjects: The list of subjects that will have these attributes removed.\n      attributes: A list of attributes.\n      start: A timestamp, attributes older than start will not be deleted.\n      end: A timestamp, attributes newer than end will not be deleted.\n      sync: If true we block until the operation completes.\n    \"\"\"\n    for subject in subjects:\n      self.DeleteAttributes(\n          subject, attributes, start=start, end=end, sync=sync)", "response": "Deletes attributes from a list of subjects."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nresolving a subject s attribute and returns the value and timestamp.", "response": "def Resolve(self, subject, attribute):\n    \"\"\"Retrieve a value set for a subject's attribute.\n\n    This method is easy to use but always gets the latest version of the\n    attribute. It is more flexible and efficient to use the other Resolve\n    methods.\n\n    Args:\n      subject: The subject URN.\n      attribute: The attribute.\n\n    Returns:\n      A (value, timestamp in microseconds) stored in the datastore cell, or\n      (None, 0). Value will be the same type as originally stored with Set().\n\n    Raises:\n      AccessError: if anything goes wrong.\n    \"\"\"\n    for _, value, timestamp in self.ResolveMulti(\n        subject, [attribute], timestamp=self.NEWEST_TIMESTAMP):\n\n      # Just return the first one.\n      return value, timestamp\n\n    return (None, 0)"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns a set of values matching the given subject s attribute with the given prefix.", "response": "def ResolvePrefix(self, subject, attribute_prefix, timestamp=None,\n                    limit=None):\n    \"\"\"Retrieve a set of value matching for this subject's attribute.\n\n    Args:\n      subject: The subject that we will search.\n      attribute_prefix: The attribute prefix.\n      timestamp: A range of times for consideration (In microseconds). Can be a\n        constant such as ALL_TIMESTAMPS or NEWEST_TIMESTAMP or a tuple of ints\n        (start, end).\n      limit: The number of results to fetch.\n\n    Returns:\n       A list of (attribute, value string, timestamp).\n\n       Values with the same attribute (happens when timestamp is not\n       NEWEST_TIMESTAMP, but ALL_TIMESTAMPS or time range) are guaranteed\n       to be ordered in the decreasing timestamp order.\n\n    Raises:\n      AccessError: if anything goes wrong.\n    \"\"\"\n    for _, values in self.MultiResolvePrefix([subject],\n                                             attribute_prefix,\n                                             timestamp=timestamp,\n                                             limit=limit):\n      values.sort(key=lambda a: a[0])\n      return values\n\n    return []"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef ReadRequestsAndResponses(self,\n                               session_id,\n                               timestamp=None,\n                               request_limit=None,\n                               response_limit=None):\n    \"\"\"Fetches all Requests and Responses for a given session_id.\"\"\"\n    subject = session_id.Add(\"state\")\n    requests = {}\n\n    # Get some requests.\n    for predicate, serialized, _ in self.ResolvePrefix(\n        subject,\n        self.FLOW_REQUEST_PREFIX,\n        limit=request_limit,\n        timestamp=timestamp):\n\n      request_id = predicate.split(\":\", 1)[1]\n      requests[str(subject.Add(request_id))] = serialized\n\n    # And the responses for them.\n    response_data = dict(\n        self.MultiResolvePrefix(\n            list(iterkeys(requests)),\n            self.FLOW_RESPONSE_PREFIX,\n            limit=response_limit,\n            timestamp=timestamp))\n\n    for urn, request_data in sorted(iteritems(requests)):\n      request = rdf_flow_runner.RequestState.FromSerializedString(request_data)\n      responses = []\n      for _, serialized, timestamp in response_data.get(urn, []):\n        msg = rdf_flows.GrrMessage.FromSerializedString(serialized)\n        msg.timestamp = timestamp\n        responses.append(msg)\n\n      yield (request, sorted(responses, key=lambda msg: msg.response_id))", "response": "Reads all Requests and Responses for a given session_id."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef ReadCompletedRequests(self, session_id, timestamp=None, limit=None):\n    subject = session_id.Add(\"state\")\n    requests = {}\n    status = {}\n\n    for predicate, serialized, _ in self.ResolvePrefix(\n        subject, [self.FLOW_REQUEST_PREFIX, self.FLOW_STATUS_PREFIX],\n        limit=limit,\n        timestamp=timestamp):\n\n      parts = predicate.split(\":\", 3)\n      request_id = parts[2]\n      if parts[1] == \"status\":\n        status[request_id] = serialized\n      else:\n        requests[request_id] = serialized\n\n    for request_id, serialized in sorted(iteritems(requests)):\n      if request_id in status:\n        yield (rdf_flow_runner.RequestState.FromSerializedString(serialized),\n               rdf_flows.GrrMessage.FromSerializedString(status[request_id]))", "response": "Reads all the requests with a status message queued for them."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef ReadResponsesForRequestId(self, session_id, request_id, timestamp=None):\n    request = rdf_flow_runner.RequestState(id=request_id, session_id=session_id)\n    for _, responses in self.ReadResponses([request], timestamp=timestamp):\n      return responses", "response": "Reads responses for a request."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreading responses for multiple requests at the same time.", "response": "def ReadResponses(self, request_list, timestamp=None):\n    \"\"\"Reads responses for multiple requests at the same time.\n\n    Args:\n      request_list: The list of requests the responses should be fetched for.\n      timestamp: A timestamp as used in the data store.\n\n    Yields:\n      tuples (request, lists of fetched responses for the request)\n    \"\"\"\n\n    response_subjects = {}\n    for request in request_list:\n      response_subject = self.GetFlowResponseSubject(request.session_id,\n                                                     request.id)\n      response_subjects[response_subject] = request\n\n    response_data = dict(\n        self.MultiResolvePrefix(\n            response_subjects, self.FLOW_RESPONSE_PREFIX, timestamp=timestamp))\n\n    for response_urn, request in sorted(iteritems(response_subjects)):\n      responses = []\n      for _, serialized, timestamp in response_data.get(response_urn, []):\n        msg = rdf_flows.GrrMessage.FromSerializedString(serialized)\n        msg.timestamp = timestamp\n        responses.append(msg)\n\n      yield (request, sorted(responses, key=lambda msg: msg.response_id))"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef StoreRequestsAndResponses(self,\n                                new_requests=None,\n                                new_responses=None,\n                                requests_to_delete=None):\n    \"\"\"Stores new flow requests and responses to the data store.\n\n    Args:\n      new_requests: A list of tuples (request, timestamp) to store in the data\n        store.\n      new_responses: A list of tuples (response, timestamp) to store in the data\n        store.\n      requests_to_delete: A list of requests that should be deleted from the\n        data store.\n    \"\"\"\n    to_write = {}\n    if new_requests is not None:\n      for request, timestamp in new_requests:\n        subject = request.session_id.Add(\"state\")\n        queue = to_write.setdefault(subject, {})\n        queue.setdefault(self.FLOW_REQUEST_TEMPLATE % request.id, []).append(\n            (request.SerializeToString(), timestamp))\n\n    if new_responses is not None:\n      for response, timestamp in new_responses:\n        # Status messages cause their requests to be marked as complete. This\n        # allows us to quickly enumerate all the completed requests - it is\n        # essentially an index for completed requests.\n        if response.type == rdf_flows.GrrMessage.Type.STATUS:\n          subject = response.session_id.Add(\"state\")\n          attribute = self.FLOW_STATUS_TEMPLATE % response.request_id\n          to_write.setdefault(subject, {}).setdefault(attribute, []).append(\n              (response.SerializeToString(), timestamp))\n\n        subject = self.GetFlowResponseSubject(response.session_id,\n                                              response.request_id)\n        attribute = self.FLOW_RESPONSE_TEMPLATE % (response.request_id,\n                                                   response.response_id)\n        to_write.setdefault(subject, {}).setdefault(attribute, []).append(\n            (response.SerializeToString(), timestamp))\n\n    to_delete = {}\n    if requests_to_delete is not None:\n      for request in requests_to_delete:\n        queue = to_delete.setdefault(request.session_id.Add(\"state\"), [])\n        queue.append(self.FLOW_REQUEST_TEMPLATE % request.id)\n        queue.append(self.FLOW_STATUS_TEMPLATE % request.id)\n\n    for subject in set(to_write) | set(to_delete):\n      self.MultiSet(\n          subject,\n          to_write.get(subject, {}),\n          to_delete=to_delete.get(subject, []),\n          sync=True)", "response": "Stores new flow requests and responses in the data store."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncheck if there are any status messages queued for a number of requests.", "response": "def CheckRequestsForCompletion(self, requests):\n    \"\"\"Checks if there is a status message queued for a number of requests.\"\"\"\n\n    subjects = [r.session_id.Add(\"state\") for r in requests]\n\n    statuses_found = {}\n\n    for subject, result in self.MultiResolvePrefix(subjects,\n                                                   self.FLOW_STATUS_PREFIX):\n      for predicate, _, _ in result:\n        request_nr = int(predicate.split(\":\")[-1], 16)\n        statuses_found.setdefault(subject, set()).add(request_nr)\n\n    status_available = set()\n    for r in requests:\n      if r.request_id in statuses_found.get(r.session_id.Add(\"state\"), set()):\n        status_available.add(r)\n\n    return status_available"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef MultiDestroyFlowStates(self, session_ids, request_limit=None):\n\n    subjects = [session_id.Add(\"state\") for session_id in session_ids]\n    to_delete = []\n    deleted_requests = []\n\n    for subject, values in self.MultiResolvePrefix(\n        subjects, self.FLOW_REQUEST_PREFIX, limit=request_limit):\n      for _, serialized, _ in values:\n\n        request = rdf_flow_runner.RequestState.FromSerializedString(serialized)\n        deleted_requests.append(request)\n\n        # Drop all responses to this request.\n        response_subject = self.GetFlowResponseSubject(request.session_id,\n                                                       request.id)\n        to_delete.append(response_subject)\n\n      # Mark the request itself for deletion.\n      to_delete.append(subject)\n\n    # Drop them all at once.\n    self.DeleteSubjects(to_delete, sync=True)\n    return deleted_requests", "response": "Deletes all requests and responses for the given flows."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nfind all objects associated with any of the keywords in the index.", "response": "def IndexReadPostingLists(self,\n                            index_urn,\n                            keywords,\n                            start_time,\n                            end_time,\n                            last_seen_map=None):\n    \"\"\"Finds all objects associated with any of the keywords.\n\n    Args:\n      index_urn: The base urn of the index.\n      keywords: A collection of keywords that we are interested in.\n      start_time: Only considers keywords added at or after this point in time.\n      end_time: Only considers keywords at or before this point in time.\n      last_seen_map: If present, is treated as a dict and populated to map pairs\n        (keyword, name) to the timestamp of the latest connection found.\n\n    Returns:\n      A dict mapping each keyword to a set of relevant names.\n    \"\"\"\n    keyword_urns = {self._KeywordToURN(index_urn, k): k for k in keywords}\n    result = {}\n    for kw in keywords:\n      result[kw] = set()\n\n    for keyword_urn, value in self.MultiResolvePrefix(\n        list(iterkeys(keyword_urns)),\n        self._INDEX_PREFIX,\n        timestamp=(start_time, end_time + 1)):\n      for column, _, ts in value:\n        kw = keyword_urns[keyword_urn]\n        name = column[self._INDEX_PREFIX_LEN:]\n        result[kw].add(name)\n        if last_seen_map is not None:\n          last_seen_map[(kw, name)] = max(last_seen_map.get((kw, name), -1), ts)\n\n    return result"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef CollectionReadIndex(self, collection_id):\n    for (attr, value,\n         ts) in self.ResolvePrefix(collection_id,\n                                   self.COLLECTION_INDEX_ATTRIBUTE_PREFIX):\n      i = int(attr[len(self.COLLECTION_INDEX_ATTRIBUTE_PREFIX):], 16)\n      yield (i, ts, int(value, 16))", "response": "Reads all index entries for the given collection."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef QueueQueryTasks(self, queue, limit=1):\n    prefix = DataStore.QUEUE_TASK_PREDICATE_PREFIX\n    all_tasks = []\n\n    for _, serialized, ts in self.ResolvePrefix(\n        queue, prefix, timestamp=DataStore.ALL_TIMESTAMPS):\n      task = rdf_flows.GrrMessage.FromSerializedString(serialized)\n      task.leased_until = ts\n      all_tasks.append(task)\n\n    return all_tasks[:limit]", "response": "Retrieves all tasks from a queue without leasing them."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nsearch the index for matches starting with target_prefix.", "response": "def FileHashIndexQuery(self, subject, target_prefix, limit=100):\n    \"\"\"Search the index for matches starting with target_prefix.\n\n    Args:\n       subject: The index to use. Should be a urn that points to the sha256\n         namespace.\n       target_prefix: The prefix to match against the index.\n       limit: Either a tuple of (start, limit) or a maximum number of results to\n         return.\n\n    Yields:\n      URNs of files which have the same data as this file - as read from the\n      index.\n    \"\"\"\n    if isinstance(limit, (tuple, list)):\n      start, length = limit  # pylint: disable=unpacking-non-sequence\n    else:\n      start = 0\n      length = limit\n\n    prefix = (DataStore.FILE_HASH_TEMPLATE % target_prefix).lower()\n    results = self.ResolvePrefix(subject, prefix, limit=limit)\n\n    for i, (_, hit, _) in enumerate(results):\n      if i < start:\n        continue\n      if i >= start + length:\n        break\n      yield rdfvalue.RDFURN(hit)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef DownloadFile(file_obj, target_path, buffer_size=BUFFER_SIZE):\n  logging.info(u\"Downloading: %s to: %s\", file_obj.urn, target_path)\n\n  target_file = open(target_path, \"wb\")\n  file_obj.Seek(0)\n  count = 0\n\n  data_buffer = file_obj.Read(buffer_size)\n  while data_buffer:\n    target_file.write(data_buffer)\n    data_buffer = file_obj.Read(buffer_size)\n    count += 1\n    if not count % 3:\n      logging.debug(u\"Downloading: %s: %s done\", file_obj.urn,\n                    utils.FormatNumberAsString(count * buffer_size))\n  target_file.close()", "response": "Downloads an aff4 file to the local filesystem overwriting it if it exists."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _OpenCollectionPath(coll_path):\n  hunt_collection = results.HuntResultCollection(coll_path)\n  if hunt_collection and hunt_collection[0].payload:\n    return hunt_collection\n\n  indexed_collection = sequential_collection.GeneralIndexedCollection(coll_path)\n  if indexed_collection:\n    return indexed_collection", "response": "Tries to open various types of collections at the given path."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ndownloads all files from a collection.", "response": "def DownloadCollection(coll_path,\n                       target_path,\n                       token=None,\n                       overwrite=False,\n                       dump_client_info=False,\n                       flatten=False,\n                       max_threads=10):\n  \"\"\"Iterate through a Collection object downloading all files.\n\n  Args:\n    coll_path: Path to an AFF4 collection.\n    target_path: Base directory to write to.\n    token: Token for access.\n    overwrite: If True, overwrite existing files.\n    dump_client_info: If True, this will detect client paths, and dump a yaml\n      version of the client object to the root path. This is useful for seeing\n      the hostname/users of the machine the client id refers to.\n    flatten: If True, produce a \"files\" flat folder with links to all the found\n      files.\n    max_threads: Use this many threads to do the downloads.\n  \"\"\"\n  completed_clients = set()\n  coll = _OpenCollectionPath(coll_path)\n  if coll is None:\n    logging.error(\n        \"%s is not a valid collection. Typo? \"\n        \"Are you sure something was written to it?\", coll_path)\n    return\n\n  thread_pool = threadpool.ThreadPool.Factory(\"Downloader\", max_threads)\n  thread_pool.Start()\n\n  # Extract the client id from the source urn. This code makes me\n  # quite sad but there is just no concept of passing a client id in\n  # the export tool (and this would be unsafe anyways since the user\n  # could then download files from arbitrary machines easily). The\n  # export tool is on the way to deprecation so we decided to do this\n  # instead of fixing the obsolete code.\n  try:\n    collection_urn = coll.collection_id\n  except AttributeError:\n    collection_urn = coll.urn\n\n  try:\n    original_client_id = rdf_client.ClientURN(collection_urn.Split()[0])\n  except IOError:\n    original_client_id = None\n\n  logging.info(\"Expecting to download %s files\", len(coll))\n\n  # Collections can include anything they want, but we only handle RDFURN and\n  # StatEntry entries in this function.\n  for grr_message in coll:\n    source = None\n    # If a raw message, work out the type.\n    if isinstance(grr_message, rdf_flows.GrrMessage):\n      source = grr_message.source\n      grr_message = grr_message.payload\n\n    if isinstance(grr_message, rdfvalue.RDFURN):\n      urn = grr_message\n    elif isinstance(grr_message, rdf_client_fs.StatEntry):\n      urn = rdfvalue.RDFURN(grr_message.AFF4Path(source or original_client_id))\n    elif isinstance(grr_message, rdf_file_finder.FileFinderResult):\n      urn = rdfvalue.RDFURN(\n          grr_message.stat_entry.AFF4Path(source or original_client_id))\n    elif isinstance(grr_message, collectors.ArtifactFilesDownloaderResult):\n      if grr_message.HasField(\"downloaded_file\"):\n        urn = grr_message.downloaded_file.AFF4Path(source or original_client_id)\n      else:\n        continue\n    elif isinstance(grr_message, rdfvalue.RDFBytes):\n      try:\n        os.makedirs(target_path)\n      except OSError:\n        pass\n      try:\n        # We just dump out bytes and carry on.\n        client_id = source.Split()[0]\n        with open(os.path.join(target_path, client_id), \"wb\") as fd:\n          fd.write(str(grr_message))\n      except AttributeError:\n        pass\n      continue\n    else:\n      continue\n\n    # Handle dumping client info, but only once per client.\n    if dump_client_info:\n      client_id = urn.Split()[0]\n      re_match = aff4_grr.VFSGRRClient.CLIENT_ID_RE.match(client_id)\n      if re_match and client_id not in completed_clients:\n        args = (rdf_client.ClientURN(client_id), target_path, token, overwrite)\n        thread_pool.AddTask(\n            target=DumpClientYaml, args=args, name=\"ClientYamlDownloader\")\n        completed_clients.add(client_id)\n\n    # Now queue downloading the actual files.\n    args = (urn, target_path, token, overwrite)\n    if flatten:\n      target = CopyAndSymlinkAFF4ToLocal\n    else:\n      target = CopyAFF4ToLocal\n    thread_pool.AddTask(target=target, args=args, name=\"Downloader\")\n\n  # Join and stop the threadpool.\n  thread_pool.Stop(join_timeout=THREADPOOL_JOIN_TIMEOUT)"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncopies an AFF4 object to a local filesystem.", "response": "def CopyAFF4ToLocal(aff4_urn, target_dir, token=None, overwrite=False):\n  \"\"\"Copy an AFF4 object that supports a read interface to local filesystem.\n\n  Args:\n    aff4_urn: URN of thing to copy.\n    target_dir: Directory to copy the file to.\n    token: Auth token.\n    overwrite: If True overwrite the file if it exists.\n\n  Returns:\n    If aff4_urn points to a file, returns path to the downloaded file.\n    Otherwise returns None.\n\n  By default file will only be overwritten if file size differs.\n  \"\"\"\n  try:\n    fd = aff4.FACTORY.Open(aff4_urn, token=token)\n    filepath = os.path.join(target_dir, fd.urn.Path()[1:])\n\n    # If urn points to a directory, just create it.\n    if isinstance(fd, standard.VFSDirectory):\n      try:\n        os.makedirs(filepath)\n      except OSError:\n        pass\n\n      return None\n    # If urn points to a file, download it.\n    elif isinstance(fd, aff4.AFF4Stream):\n      if not os.path.isfile(filepath):\n        try:\n          # Ensure directory exists.\n          os.makedirs(os.path.dirname(filepath))\n        except OSError:\n          pass\n        DownloadFile(fd, filepath)\n      elif (os.stat(filepath)[stat.ST_SIZE] != fd.Get(fd.Schema.SIZE) or\n            overwrite):\n        # We should overwrite because user said, or file sizes differ.\n        DownloadFile(fd, filepath)\n      else:\n        logging.info(\"File %s exists, skipping\", filepath)\n\n      return filepath\n    else:\n      raise ValueError(\"Opened urn is neither a downloaded file nor a \"\n                       \"directory: %s\" % aff4_urn)\n\n  except IOError as e:\n    logging.exception(\"Failed to read %s due to %s\", aff4_urn, e)\n    raise"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef DumpClientYaml(client_urn, target_dir, token=None, overwrite=False):\n  fd = aff4.FACTORY.Open(client_urn, aff4_grr.VFSGRRClient, token=token)\n  dirpath = os.path.join(target_dir, fd.urn.Split()[0])\n  try:\n    # Due to threading this can actually be created by another thread.\n    os.makedirs(dirpath)\n  except OSError:\n    pass\n  filepath = os.path.join(dirpath, \"client_info.yaml\")\n  if not os.path.isfile(filepath) or overwrite:\n    with open(filepath, \"wb\") as out_file:\n      out_file.write(serialize.YamlDumper(fd))", "response": "Dump a yaml file containing client info."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nyielding all client urns in a batch.", "response": "def GetInput(self):\n    \"\"\"Yield client urns.\"\"\"\n    client_list = GetAllClients(token=self.token)\n    logging.debug(\"Got %d clients\", len(client_list))\n    for client_group in collection.Batch(client_list, self.client_chunksize):\n      for fd in aff4.FACTORY.MultiOpen(\n          client_group,\n          mode=\"r\",\n          aff4_type=aff4_grr.VFSGRRClient,\n          token=self.token):\n        if isinstance(fd, aff4_grr.VFSGRRClient):\n          # Skip if older than max_age\n          oldest_time = (time.time() - self.max_age) * 1e6\n        if fd.Get(aff4_grr.VFSGRRClient.SchemaCls.PING) >= oldest_time:\n          yield fd"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef InterpolateKbAttributes(pattern, knowledge_base, ignore_errors=False):\n  # TODO(hanuszczak): Control flow feels a bit awkward here because of error\n  # handling that tries not to break any functionality. With the new utilities\n  # it should be possible to improve the code, changing the behaviour to a more\n  # sane one.\n  missing_attr_names = set()\n\n  interpolator = interpolation.Interpolator(pattern)\n\n  for vid in interpolator.Vars():\n    attr_name = str(vid).lower()\n\n    value = getattr(knowledge_base, attr_name, None)\n    if not value:\n      missing_attr_names.add(attr_name)\n      continue\n\n    interpolator.BindVar(attr_name, value)\n\n  for scope_id in interpolator.Scopes():\n    scope_name = str(scope_id).lower()\n\n    kb_structs = getattr(knowledge_base, scope_name, None)\n    if not kb_structs:\n      missing_attr_names.add(scope_name)\n      continue\n\n    bound = False\n\n    for kb_struct in kb_structs:\n      bindings = {}\n\n      for var_id in interpolator.ScopeVars(scope_id):\n        attr_name = str(var_id).lower()\n\n        value = getattr(kb_struct, attr_name, None)\n        if value:\n          bindings[var_id] = value\n\n      if len(bindings) == len(interpolator.ScopeVars(scope_id)):\n        interpolator.BindScope(scope_id, bindings)\n        bound = True\n\n    if not bound:\n      missing_attr_names.add(scope_name)\n\n  if missing_attr_names:\n    message = \"Some attributes could not be located in the knowledgebase: {}\"\n    message = message.format(\", \".join(missing_attr_names))\n    if not ignore_errors:\n      raise KnowledgeBaseInterpolationError(message)\n    else:\n      logging.error(message)\n      return\n\n  for result in interpolator.Interpolate():\n    yield result", "response": "Interpolate all attributes in a pattern."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef GetWindowsEnvironmentVariablesMap(knowledge_base):\n\n  environ_vars = {}\n\n  if knowledge_base.environ_path:\n    environ_vars[\"path\"] = knowledge_base.environ_path\n\n  if knowledge_base.environ_temp:\n    environ_vars[\"temp\"] = knowledge_base.environ_temp\n\n  if knowledge_base.environ_systemroot:\n    environ_vars[\"systemroot\"] = knowledge_base.environ_systemroot\n\n  if knowledge_base.environ_windir:\n    environ_vars[\"windir\"] = knowledge_base.environ_windir\n\n  if knowledge_base.environ_programfiles:\n    environ_vars[\"programfiles\"] = knowledge_base.environ_programfiles\n    environ_vars[\"programw6432\"] = knowledge_base.environ_programfiles\n\n  if knowledge_base.environ_programfilesx86:\n    environ_vars[\"programfiles(x86)\"] = knowledge_base.environ_programfilesx86\n\n  if knowledge_base.environ_systemdrive:\n    environ_vars[\"systemdrive\"] = knowledge_base.environ_systemdrive\n\n  if knowledge_base.environ_allusersprofile:\n    environ_vars[\"allusersprofile\"] = knowledge_base.environ_allusersprofile\n    environ_vars[\"programdata\"] = knowledge_base.environ_allusersprofile\n\n  if knowledge_base.environ_allusersappdata:\n    environ_vars[\"allusersappdata\"] = knowledge_base.environ_allusersappdata\n\n  for user in knowledge_base.users:\n    if user.appdata:\n      environ_vars.setdefault(\"appdata\", []).append(user.appdata)\n\n    if user.localappdata:\n      environ_vars.setdefault(\"localappdata\", []).append(user.localappdata)\n\n    if user.userdomain:\n      environ_vars.setdefault(\"userdomain\", []).append(user.userdomain)\n\n    if user.userprofile:\n      environ_vars.setdefault(\"userprofile\", []).append(user.userprofile)\n\n  return environ_vars", "response": "Returns a dictionary of environment variables and values."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef ExpandWindowsEnvironmentVariables(data_string, knowledge_base):\n  win_environ_regex = re.compile(r\"%([^%]+?)%\")\n  components = []\n  offset = 0\n  for match in win_environ_regex.finditer(data_string):\n    components.append(data_string[offset:match.start()])\n\n    # KB environment variables are prefixed with environ_.\n    kb_value = getattr(knowledge_base, \"environ_%s\" % match.group(1).lower(),\n                       None)\n    if isinstance(kb_value, string_types) and kb_value:\n      components.append(kb_value)\n    else:\n      # Failed to expand, leave the variable as it was.\n      components.append(\"%%%s%%\" % match.group(1))\n    offset = match.end()\n  components.append(data_string[offset:])  # Append the final chunk.\n  return \"\".join(components)", "response": "Take a string and expand any windows environment variables."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nchecking if a condition matches an object.", "response": "def CheckCondition(condition, check_object):\n  \"\"\"Check if a condition matches an object.\n\n  Args:\n    condition: A string condition e.g. \"os == 'Windows'\"\n    check_object: Object to validate, e.g. an rdf_client.KnowledgeBase()\n\n  Returns:\n    True or False depending on whether the condition matches.\n\n  Raises:\n    ConditionError: If condition is bad.\n  \"\"\"\n  try:\n    of = objectfilter.Parser(condition).Parse()\n    compiled_filter = of.Compile(objectfilter.BaseFilterImplementation)\n    return compiled_filter.Matches(check_object)\n  except objectfilter.Error as e:\n    raise ConditionError(e)"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ntake a string and expand windows user environment variables based.", "response": "def ExpandWindowsUserEnvironmentVariables(data_string,\n                                          knowledge_base,\n                                          sid=None,\n                                          username=None):\n  r\"\"\"Take a string and expand windows user environment variables based.\n\n  Args:\n    data_string: A string, e.g. \"%TEMP%\\\\LogFiles\"\n    knowledge_base: A knowledgebase object.\n    sid: A Windows SID for a user to expand for.\n    username: A Windows user name to expand for.\n\n  Returns:\n    A string with available environment variables expanded.\n  \"\"\"\n  win_environ_regex = re.compile(r\"%([^%]+?)%\")\n  components = []\n  offset = 0\n  for match in win_environ_regex.finditer(data_string):\n    components.append(data_string[offset:match.start()])\n    kb_user = knowledge_base.GetUser(sid=sid, username=username)\n    kb_value = None\n    if kb_user:\n      kb_value = getattr(kb_user, match.group(1).lower(), None)\n    if isinstance(kb_value, string_types) and kb_value:\n      components.append(kb_value)\n    else:\n      components.append(\"%%%s%%\" % match.group(1))\n    offset = match.end()\n\n  components.append(data_string[offset:])  # Append the final chunk.\n  return \"\".join(components)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef GetExtAttrs(filepath):\n  path = CanonicalPathToLocalPath(filepath)\n\n  try:\n    attr_names = xattr.listxattr(path)\n  except (IOError, OSError, UnicodeDecodeError) as error:\n    msg = \"Failed to retrieve extended attributes for '%s': %s\"\n    logging.error(msg, path, error)\n    return\n\n  # `xattr` (version 0.9.2) decodes names as UTF-8. Since we (and the system)\n  # allows for names and values to be arbitrary byte strings, we use `bytes`\n  # rather than `unicode` objects here. Therefore we have to re-encode what\n  # `xattr` has decoded. Additionally, because the decoding that `xattr` does\n  # may fail, we additionally guard against such exceptions.\n  def EncodeUtf8(attr_name):\n    if isinstance(attr_name, Text):\n      return attr_name.encode(\"utf-8\")\n    if isinstance(attr_name, bytes):\n      return attr_name\n    raise TypeError(\"Unexpected type `%s`\" % type(attr_name))\n\n  for attr_name in attr_names:\n    attr_name = EncodeUtf8(attr_name)\n    try:\n      attr_value = xattr.getxattr(path, attr_name)\n    except (IOError, OSError) as error:\n      msg = \"Failed to retrieve attribute '%s' for '%s': %s\"\n      logging.error(msg, attr_name, path, error)\n      continue\n\n    yield rdf_client_fs.ExtAttr(name=attr_name, value=attr_value)", "response": "Retrieves extended file attributes."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nsleep a given number of seconds.", "response": "def Sleep(self, seconds):\n    \"\"\"Sleep a given time in 1 second intervals.\n\n    When a machine is suspended during a time.sleep(n) call for more\n    than n seconds, sometimes the sleep is interrupted and all threads\n    wake up at the same time. This leads to race conditions between\n    the threads issuing the heartbeat and the one checking for it. By\n    sleeping in small intervals, we make sure that even if one sleep\n    call is interrupted, we do not check for the heartbeat too early.\n\n    Args:\n      seconds: Number of seconds to sleep.\n\n    Raises:\n      MemoryError: if the process exceeds memory quota.\n    \"\"\"\n    time.sleep(seconds - int(seconds))\n    for _ in range(int(seconds)):\n      time.sleep(1)\n      # Check that we do not exceeded our memory allowance.\n      if self.GetMemoryUsage() > self.memory_quota:\n        raise MemoryError(\"Exceeded memory allowance.\")\n      if not self.running:\n        break"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nwriting the message into the transaction log.", "response": "def Write(self, grr_message):\n    \"\"\"Write the message into the transaction log.\"\"\"\n    grr_message = grr_message.SerializeToString()\n\n    try:\n      with io.open(self.logfile, \"wb\") as fd:\n        fd.write(grr_message)\n    except (IOError, OSError):\n      # Check if we're missing directories and try to create them.\n      if not os.path.isdir(os.path.dirname(self.logfile)):\n        try:\n          os.makedirs(os.path.dirname(self.logfile))\n          with io.open(self.logfile, \"wb\") as fd:\n            fd.write(grr_message)\n        except (IOError, OSError):\n          logging.exception(\"Couldn't write nanny transaction log to %s\",\n                            self.logfile)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nwipe the transaction log.", "response": "def Clear(self):\n    \"\"\"Wipes the transaction log.\"\"\"\n    try:\n      with io.open(self.logfile, \"wb\") as fd:\n        fd.write(b\"\")\n    except (IOError, OSError):\n      pass"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef Get(self):\n    try:\n      with io.open(self.logfile, \"rb\") as fd:\n        data = fd.read(self.max_log_size)\n    except (IOError, OSError):\n      return\n\n    try:\n      if data:\n        return rdf_flows.GrrMessage.FromSerializedString(data)\n    except (message.Error, rdfvalue.Error):\n      return", "response": "Returns a GrrMessage instance from the transaction log or None."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nremove a specific symbol from a fn_table.", "response": "def FilterFnTable(fn_table, symbol):\n  \"\"\"Remove a specific symbol from a fn_table.\"\"\"\n  new_table = list()\n  for entry in fn_table:\n    # symbol[0] is a str with the symbol name\n    if entry[0] != symbol:\n      new_table.append(entry)\n  return new_table"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef SetCTypesForLibrary(libname, fn_table):\n  libpath = ctypes.util.find_library(libname)\n  if not libpath:\n    raise ErrorLibNotFound('Library %s not found' % libname)\n\n  lib = ctypes.cdll.LoadLibrary(libpath)\n\n  # We need to define input / output parameters for all functions we use\n  for (function, args, result) in fn_table:\n    f = getattr(lib, function)\n    f.argtypes = args\n    f.restype = result\n\n  return lib", "response": "Sets function argument types and return types for an ObjC library."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef WrapCFTypeInPython(self, obj):\n    obj_type = self.dll.CFGetTypeID(obj)\n    if obj_type == self.dll.CFBooleanGetTypeID():\n      return CFBoolean(obj)\n    elif obj_type == self.dll.CFNumberGetTypeID():\n      return CFNumber(obj)\n    elif obj_type == self.dll.CFStringGetTypeID():\n      return CFString(obj)\n    elif obj_type == self.dll.CFDictionaryGetTypeID():\n      return CFDictionary(obj)\n    elif obj_type == self.dll.CFArrayGetTypeID():\n      return CFArray(obj)\n    else:\n      raise TypeError('Unknown type for object: {0}'.format(obj))", "response": "Package a CoreFoundation object in a Python wrapper."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef SMGetJobDictionaries(self, domain='kSMDomainSystemLaunchd'):\n    cfstring_launchd = ctypes.c_void_p.in_dll(self.dll, domain)\n    return CFArray(self.dll.SMCopyAllJobDictionaries(cfstring_launchd))", "response": "Copy all Job Dictionaries from the ServiceManagement."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn dictionary values or default.", "response": "def get(self, key, default='', stringify=True):\n    \"\"\"Returns dictionary values or default.\n\n    Args:\n      key: string. Dictionary key to look up.\n      default: string. Return this value if key not found.\n      stringify: bool. Force all return values to string for compatibility\n                 reasons.\n    Returns:\n      python-wrapped CF object or default if not found.\n    \"\"\"\n    obj = self.__getitem__(key)\n    if obj is None:\n      obj = default\n    elif stringify:\n      obj = str(obj)\n    return obj"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nloading the kextmanager with the given name.", "response": "def SafeLoadKextManager(self, fn_table):\n    \"\"\"Load the kextmanager, replacing unavailable symbols.\"\"\"\n    dll = None\n    try:\n      dll = SetCTypesForLibrary('IOKit', fn_table)\n    except AttributeError as ae:\n      if 'KextManagerUnloadKextWithIdentifier' in str(ae):\n        # Try without this symbol, as it is not available on 10.6\n        logging.debug('Using legacy kextunload')\n        dll = self.SafeLoadKextManager(\n            FilterFnTable(fn_table, 'KextManagerUnloadKextWithIdentifier'))\n        dll.KextManagerUnloadKextWithIdentifier = self.LegacyKextunload\n      elif 'KextManagerLoadKextWithURL' in str(ae):\n        logging.debug('Using legacy kextload')\n        dll = self.SafeLoadKextManager(\n            FilterFnTable(fn_table, 'KextManagerLoadKextWithURL'))\n        dll.KextManagerLoadKextWithURL = self.LegacyKextload\n      else:\n        raise OSError('Can\\'t resolve KextManager symbols:{0}'.format(str(ae)))\n\n    return dll"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef LegacyKextload(self, cf_bundle_url, dependency_kext):\n    _ = dependency_kext\n    error_code = OS_SUCCESS\n    cf_path = self.dll.CFURLCopyFileSystemPath(cf_bundle_url, POSIX_PATH_STYLE)\n    path = self.CFStringToPystring(cf_path)\n    self.dll.CFRelease(cf_path)\n    try:\n      subprocess.check_call(['/sbin/kextload', path])\n    except subprocess.CalledProcessError as cpe:\n      logging.debug('failed to load %s:%s', path, str(cpe))\n      error_code = -1\n    return error_code", "response": "Load a kext by forking into kextload."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nunloading a kext by forking into kextunload.", "response": "def LegacyKextunload(self, cf_bundle_identifier):\n    \"\"\"Unload a kext by forking into kextunload.\"\"\"\n    error_code = OS_SUCCESS\n    bundle_identifier = self.CFStringToPystring(cf_bundle_identifier)\n    try:\n      subprocess.check_call(['/sbin/kextunload', '-b', bundle_identifier])\n    except subprocess.CalledProcessError as cpe:\n      logging.debug('failed to unload %s:%s', bundle_identifier, str(cpe))\n      error_code = -1\n    return error_code"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef ReadPathInfo(self,\n                   client_id,\n                   path_type,\n                   components,\n                   timestamp=None,\n                   cursor=None):\n    \"\"\"Retrieves a path info record for a given path.\"\"\"\n    if timestamp is None:\n      path_infos = self.ReadPathInfos(client_id, path_type, [components])\n\n      path_info = path_infos[components]\n      if path_info is None:\n        raise db.UnknownPathError(\n            client_id=client_id, path_type=path_type, components=components)\n\n      return path_info\n\n    query = \"\"\"\n    SELECT directory, UNIX_TIMESTAMP(p.timestamp),\n           stat_entry, UNIX_TIMESTAMP(last_stat_entry_timestamp),\n           hash_entry, UNIX_TIMESTAMP(last_hash_entry_timestamp)\n      FROM client_paths as p\n LEFT JOIN (SELECT client_id, path_type, path_id, stat_entry\n              FROM client_path_stat_entries\n             WHERE client_id = %(client_id)s\n               AND path_type = %(path_type)s\n               AND path_id = %(path_id)s\n               AND UNIX_TIMESTAMP(timestamp) <= %(timestamp)s\n          ORDER BY timestamp DESC\n             LIMIT 1) AS s\n        ON p.client_id = s.client_id\n       AND p.path_type = s.path_type\n       AND p.path_id = s.path_id\n LEFT JOIN (SELECT client_id, path_type, path_id, hash_entry\n              FROM client_path_hash_entries\n             WHERE client_id = %(client_id)s\n               AND path_type = %(path_type)s\n               AND path_id = %(path_id)s\n               AND UNIX_TIMESTAMP(timestamp) <= %(timestamp)s\n          ORDER BY timestamp DESC\n             LIMIT 1) AS h\n        ON p.client_id = h.client_id\n       AND p.path_type = h.path_type\n       AND p.path_id = h.path_id\n     WHERE p.client_id = %(client_id)s\n       AND p.path_type = %(path_type)s\n       AND p.path_id = %(path_id)s\n    \"\"\"\n    values = {\n        \"client_id\": db_utils.ClientIDToInt(client_id),\n        \"path_type\": int(path_type),\n        \"path_id\": rdf_objects.PathID.FromComponents(components).AsBytes(),\n        \"timestamp\": mysql_utils.RDFDatetimeToTimestamp(timestamp),\n    }\n\n    cursor.execute(query, values)\n    row = cursor.fetchone()\n    if row is None:\n      raise db.UnknownPathError(\n          client_id=client_id, path_type=path_type, components=components)\n\n    # pyformat: disable\n    (directory, timestamp,\n     stat_entry_bytes, last_stat_entry_timestamp,\n     hash_entry_bytes, last_hash_entry_timestamp) = row\n    # pyformat: enable\n\n    if stat_entry_bytes is not None:\n      stat_entry = rdf_client_fs.StatEntry.FromSerializedString(\n          stat_entry_bytes)\n    else:\n      stat_entry = None\n\n    if hash_entry_bytes is not None:\n      hash_entry = rdf_crypto.Hash.FromSerializedString(hash_entry_bytes)\n    else:\n      hash_entry = None\n\n    datetime = mysql_utils.TimestampToRDFDatetime\n    return rdf_objects.PathInfo(\n        path_type=path_type,\n        components=components,\n        timestamp=datetime(timestamp),\n        last_stat_entry_timestamp=datetime(last_stat_entry_timestamp),\n        last_hash_entry_timestamp=datetime(last_hash_entry_timestamp),\n        directory=directory,\n        stat_entry=stat_entry,\n        hash_entry=hash_entry)", "response": "Reads a path info record for a given path."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nread path info records for given paths.", "response": "def ReadPathInfos(self, client_id, path_type, components_list, cursor=None):\n    \"\"\"Retrieves path info records for given paths.\"\"\"\n\n    if not components_list:\n      return {}\n\n    path_ids = list(map(rdf_objects.PathID.FromComponents, components_list))\n\n    path_infos = {components: None for components in components_list}\n\n    query = \"\"\"\n    SELECT path, directory, UNIX_TIMESTAMP(client_paths.timestamp),\n           stat_entry, UNIX_TIMESTAMP(last_stat_entry_timestamp),\n           hash_entry, UNIX_TIMESTAMP(last_hash_entry_timestamp)\n      FROM client_paths\n LEFT JOIN client_path_stat_entries ON\n           (client_paths.client_id = client_path_stat_entries.client_id AND\n            client_paths.path_type = client_path_stat_entries.path_type AND\n            client_paths.path_id = client_path_stat_entries.path_id AND\n            client_paths.last_stat_entry_timestamp = client_path_stat_entries.timestamp)\n LEFT JOIN client_path_hash_entries ON\n           (client_paths.client_id = client_path_hash_entries.client_id AND\n            client_paths.path_type = client_path_hash_entries.path_type AND\n            client_paths.path_id = client_path_hash_entries.path_id AND\n            client_paths.last_hash_entry_timestamp = client_path_hash_entries.timestamp)\n     WHERE client_paths.client_id = %(client_id)s\n       AND client_paths.path_type = %(path_type)s\n       AND client_paths.path_id IN %(path_ids)s\n    \"\"\"\n    values = {\n        \"client_id\": db_utils.ClientIDToInt(client_id),\n        \"path_type\": int(path_type),\n        \"path_ids\": [path_id.AsBytes() for path_id in path_ids]\n    }\n\n    cursor.execute(query, values)\n    for row in cursor.fetchall():\n      # pyformat: disable\n      (path, directory, timestamp,\n       stat_entry_bytes, last_stat_entry_timestamp,\n       hash_entry_bytes, last_hash_entry_timestamp) = row\n      # pyformat: enable\n      components = mysql_utils.PathToComponents(path)\n\n      if stat_entry_bytes is not None:\n        stat_entry = rdf_client_fs.StatEntry.FromSerializedString(\n            stat_entry_bytes)\n      else:\n        stat_entry = None\n\n      if hash_entry_bytes is not None:\n        hash_entry = rdf_crypto.Hash.FromSerializedString(hash_entry_bytes)\n      else:\n        hash_entry = None\n\n      datetime = mysql_utils.TimestampToRDFDatetime\n      path_info = rdf_objects.PathInfo(\n          path_type=path_type,\n          components=components,\n          timestamp=datetime(timestamp),\n          last_stat_entry_timestamp=datetime(last_stat_entry_timestamp),\n          last_hash_entry_timestamp=datetime(last_hash_entry_timestamp),\n          directory=directory,\n          stat_entry=stat_entry,\n          hash_entry=hash_entry)\n\n      path_infos[components] = path_info\n\n    return path_infos"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nwrites a collection of path_info records for a client.", "response": "def WritePathInfos(self, client_id, path_infos):\n    \"\"\"Writes a collection of path_info records for a client.\"\"\"\n    try:\n      self._MultiWritePathInfos({client_id: path_infos})\n    except MySQLdb.IntegrityError as error:\n      raise db.UnknownClientError(client_id=client_id, cause=error)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nwriting a collection of path info records for specified clients.", "response": "def MultiWritePathInfos(self, path_infos):\n    \"\"\"Writes a collection of path info records for specified clients.\"\"\"\n    try:\n      self._MultiWritePathInfos(path_infos)\n    except MySQLdb.IntegrityError as error:\n      client_ids = list(iterkeys(path_infos))\n      raise db.AtLeastOneUnknownClientError(client_ids=client_ids, cause=error)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nwrite a collection of path info records for specified clients.", "response": "def _MultiWritePathInfos(self, path_infos, connection=None):\n    \"\"\"Writes a collection of path info records for specified clients.\"\"\"\n    query = \"\"\n\n    path_info_count = 0\n    path_info_values = []\n\n    parent_path_info_count = 0\n    parent_path_info_values = []\n\n    has_stat_entries = False\n    has_hash_entries = False\n\n    for client_id, client_path_infos in iteritems(path_infos):\n      for path_info in client_path_infos:\n        path = mysql_utils.ComponentsToPath(path_info.components)\n\n        path_info_values.append(db_utils.ClientIDToInt(client_id))\n        path_info_values.append(int(path_info.path_type))\n        path_info_values.append(path_info.GetPathID().AsBytes())\n        path_info_values.append(path)\n        path_info_values.append(bool(path_info.directory))\n        path_info_values.append(len(path_info.components))\n\n        if path_info.HasField(\"stat_entry\"):\n          path_info_values.append(path_info.stat_entry.SerializeToString())\n          has_stat_entries = True\n        else:\n          path_info_values.append(None)\n        if path_info.HasField(\"hash_entry\"):\n          path_info_values.append(path_info.hash_entry.SerializeToString())\n          path_info_values.append(path_info.hash_entry.sha256.AsBytes())\n          has_hash_entries = True\n        else:\n          path_info_values.append(None)\n          path_info_values.append(None)\n\n        path_info_count += 1\n\n        # TODO(hanuszczak): Implement a trie in order to avoid inserting\n        # duplicated records.\n        for parent_path_info in path_info.GetAncestors():\n          path = mysql_utils.ComponentsToPath(parent_path_info.components)\n\n          parent_path_info_values.append(db_utils.ClientIDToInt(client_id))\n          parent_path_info_values.append(int(parent_path_info.path_type))\n          parent_path_info_values.append(parent_path_info.GetPathID().AsBytes())\n          parent_path_info_values.append(path)\n          parent_path_info_values.append(len(parent_path_info.components))\n\n          parent_path_info_count += 1\n\n    query += \"\"\"\n    CREATE TEMPORARY TABLE client_path_infos(\n      client_id BIGINT UNSIGNED NOT NULL,\n      path_type INT UNSIGNED NOT NULL,\n      path_id BINARY(32) NOT NULL,\n      path TEXT NOT NULL,\n      directory BOOLEAN NOT NULL,\n      depth INT NOT NULL,\n      stat_entry MEDIUMBLOB NULL,\n      hash_entry MEDIUMBLOB NULL,\n      sha256 BINARY(32) NULL,\n      timestamp TIMESTAMP(6) NOT NULL DEFAULT now(6)\n    );\"\"\"\n\n    if path_info_count > 0:\n      query += \"\"\"\n      INSERT INTO client_path_infos(client_id, path_type, path_id,\n                                    path, directory, depth,\n                                    stat_entry, hash_entry, sha256)\n      VALUES {};\n      \"\"\".format(mysql_utils.Placeholders(num=9, values=path_info_count))\n\n      query += \"\"\"\n      INSERT INTO client_paths(client_id, path_type, path_id, path,\n                               directory, depth)\n           SELECT client_id, path_type, path_id, path, directory, depth\n             FROM client_path_infos\n      ON DUPLICATE KEY UPDATE\n        client_paths.directory = client_paths.directory OR VALUES(client_paths.directory),\n        client_paths.timestamp = now(6);\n      \"\"\"\n\n    if parent_path_info_count > 0:\n      placeholders = [\"(%s, %s, %s, %s, TRUE, %s)\"] * parent_path_info_count\n\n      query += \"\"\"\n      INSERT INTO client_paths(client_id, path_type, path_id, path,\n                               directory, depth)\n      VALUES {}\n      ON DUPLICATE KEY UPDATE\n        directory = TRUE,\n        timestamp = now();\n      \"\"\".format(\", \".join(placeholders))\n\n    if has_stat_entries:\n      query += \"\"\"\n      INSERT INTO client_path_stat_entries(client_id, path_type, path_id,\n                                           stat_entry, timestamp)\n           SELECT client_id, path_type, path_id, stat_entry, timestamp\n             FROM client_path_infos\n            WHERE stat_entry IS NOT NULL;\n      \"\"\"\n\n      query += \"\"\"\n      UPDATE client_paths, client_path_infos\n         SET client_paths.last_stat_entry_timestamp = client_path_infos.timestamp\n       WHERE client_paths.client_id = client_path_infos.client_id\n         AND client_paths.path_type = client_path_infos.path_type\n         AND client_paths.path_id = client_path_infos.path_id\n         AND client_path_infos.stat_entry IS NOT NULL;\n      \"\"\"\n\n    if has_hash_entries:\n      query += \"\"\"\n      INSERT INTO client_path_hash_entries(client_id, path_type, path_id,\n                                           hash_entry, sha256, timestamp)\n           SELECT client_id, path_type, path_id, hash_entry, sha256, timestamp\n             FROM client_path_infos\n            WHERE hash_entry IS NOT NULL;\n      \"\"\"\n\n      query += \"\"\"\n      UPDATE client_paths, client_path_infos\n         SET client_paths.last_hash_entry_timestamp = client_path_infos.timestamp\n       WHERE client_paths.client_id = client_path_infos.client_id\n         AND client_paths.path_type = client_path_infos.path_type\n         AND client_paths.path_id = client_path_infos.path_id\n         AND client_path_infos.hash_entry IS NOT NULL;\n      \"\"\"\n\n    try:\n      with contextlib.closing(connection.cursor()) as cursor:\n        cursor.execute(query, path_info_values + parent_path_info_values)\n    finally:\n      # Drop the temporary table in a separate cursor. This ensures that\n      # even if the previous cursor.execute fails mid-way leaving the\n      # temporary table created (as table creation can't be rolled back), the\n      # table would still be correctly dropped.\n      #\n      # This is important since connections are reused in the MySQL connection\n      # pool.\n      with contextlib.closing(connection.cursor()) as cursor:\n        cursor.execute(\"DROP TEMPORARY TABLE IF EXISTS client_path_infos\")"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nlists all path info records that correspond to descendants of given path.", "response": "def ListDescendentPathInfos(self,\n                              client_id,\n                              path_type,\n                              components,\n                              timestamp=None,\n                              max_depth=None,\n                              cursor=None):\n    \"\"\"Lists path info records that correspond to descendants of given path.\"\"\"\n    path_infos = []\n\n    query = \"\"\n\n    path = mysql_utils.ComponentsToPath(components)\n    values = {\n        \"client_id\": db_utils.ClientIDToInt(client_id),\n        \"path_type\": int(path_type),\n        \"path\": db_utils.EscapeWildcards(path),\n    }\n\n    query += \"\"\"\n    SELECT path, directory, UNIX_TIMESTAMP(p.timestamp),\n           stat_entry, UNIX_TIMESTAMP(last_stat_entry_timestamp),\n           hash_entry, UNIX_TIMESTAMP(last_hash_entry_timestamp)\n      FROM client_paths AS p\n    \"\"\"\n    if timestamp is None:\n      query += \"\"\"\n      LEFT JOIN client_path_stat_entries AS s ON\n                (p.client_id = s.client_id AND\n                 p.path_type = s.path_type AND\n                 p.path_id = s.path_id AND\n                 p.last_stat_entry_timestamp = s.timestamp)\n      LEFT JOIN client_path_hash_entries AS h ON\n                (p.client_id = h.client_id AND\n                 p.path_type = h.path_type AND\n                 p.path_id = h.path_id AND\n                 p.last_hash_entry_timestamp = h.timestamp)\n      \"\"\"\n      only_explicit = False\n    else:\n      query += \"\"\"\n      LEFT JOIN (SELECT sr.client_id, sr.path_type, sr.path_id, sr.stat_entry\n                   FROM client_path_stat_entries AS sr\n             INNER JOIN (SELECT client_id, path_type, path_id,\n                                MAX(timestamp) AS max_timestamp\n                           FROM client_path_stat_entries\n                          WHERE UNIX_TIMESTAMP(timestamp) <= %(timestamp)s\n                       GROUP BY client_id, path_type, path_id) AS st\n                     ON sr.client_id = st.client_id\n                    AND sr.path_type = st.path_type\n                    AND sr.path_id = st.path_id\n                    AND sr.timestamp = st.max_timestamp) AS s\n             ON (p.client_id = s.client_id AND\n                 p.path_type = s.path_type AND\n                 p.path_id = s.path_id)\n      LEFT JOIN (SELECT hr.client_id, hr.path_type, hr.path_id, hr.hash_entry\n                   FROM client_path_hash_entries AS hr\n             INNER JOIN (SELECT client_id, path_type, path_id,\n                                MAX(timestamp) AS max_timestamp\n                           FROM client_path_hash_entries\n                          WHERE UNIX_TIMESTAMP(timestamp) <= %(timestamp)s\n                       GROUP BY client_id, path_type, path_id) AS ht\n                     ON hr.client_id = ht.client_id\n                    AND hr.path_type = ht.path_type\n                    AND hr.path_id = ht.path_id\n                    AND hr.timestamp = ht.max_timestamp) AS h\n             ON (p.client_id = h.client_id AND\n                 p.path_type = h.path_type AND\n                 p.path_id = h.path_id)\n      \"\"\"\n      values[\"timestamp\"] = mysql_utils.RDFDatetimeToTimestamp(timestamp)\n      only_explicit = True\n\n    query += \"\"\"\n    WHERE p.client_id = %(client_id)s\n      AND p.path_type = %(path_type)s\n      AND path LIKE concat(%(path)s, '/%%')\n    \"\"\"\n\n    if max_depth is not None:\n      query += \"\"\"\n      AND depth <= %(depth)s\n      \"\"\"\n      values[\"depth\"] = len(components) + max_depth\n\n    cursor.execute(query, values)\n    for row in cursor.fetchall():\n      # pyformat: disable\n      (path, directory, timestamp,\n       stat_entry_bytes, last_stat_entry_timestamp,\n       hash_entry_bytes, last_hash_entry_timestamp) = row\n      # pyformat: enable\n\n      components = mysql_utils.PathToComponents(path)\n\n      if stat_entry_bytes is not None:\n        stat_entry = rdf_client_fs.StatEntry.FromSerializedString(\n            stat_entry_bytes)\n      else:\n        stat_entry = None\n\n      if hash_entry_bytes is not None:\n        hash_entry = rdf_crypto.Hash.FromSerializedString(hash_entry_bytes)\n      else:\n        hash_entry = None\n\n      datetime = mysql_utils.TimestampToRDFDatetime\n      path_info = rdf_objects.PathInfo(\n          path_type=path_type,\n          components=components,\n          timestamp=datetime(timestamp),\n          last_stat_entry_timestamp=datetime(last_stat_entry_timestamp),\n          last_hash_entry_timestamp=datetime(last_hash_entry_timestamp),\n          directory=directory,\n          stat_entry=stat_entry,\n          hash_entry=hash_entry)\n\n      path_infos.append(path_info)\n\n    path_infos.sort(key=lambda _: tuple(_.components))\n\n    # For specific timestamp, we return information only about explicit paths\n    # (paths that have associated stat or hash entry or have an ancestor that is\n    # explicit).\n    if not only_explicit:\n      return path_infos\n\n    explicit_path_infos = []\n    has_explicit_ancestor = set()\n\n    # This list is sorted according to the keys component, so by traversing it\n    # in the reverse order we make sure that we process deeper paths first.\n    for path_info in reversed(path_infos):\n      components = tuple(path_info.components)\n\n      if (path_info.HasField(\"stat_entry\") or\n          path_info.HasField(\"hash_entry\") or\n          components in has_explicit_ancestor):\n        explicit_path_infos.append(path_info)\n        has_explicit_ancestor.add(components[:-1])\n\n    # Since we collected explicit paths in reverse order, we need to reverse it\n    # again to conform to the interface.\n    return list(reversed(explicit_path_infos))"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreading a collection of hash and stat entries for given paths.", "response": "def ReadPathInfosHistories(self,\n                             client_id,\n                             path_type,\n                             components_list,\n                             cursor=None):\n    \"\"\"Reads a collection of hash and stat entries for given paths.\"\"\"\n    # MySQL does not handle well empty `IN` clauses so we guard against that.\n    if not components_list:\n      return {}\n\n    path_infos = {components: [] for components in components_list}\n\n    path_id_components = {}\n    for components in components_list:\n      path_id = rdf_objects.PathID.FromComponents(components)\n      path_id_components[path_id] = components\n\n    # MySQL does not support full outer joins, so we emulate them with a union.\n    query = \"\"\"\n    SELECT s.path_id, s.stat_entry, UNIX_TIMESTAMP(s.timestamp),\n           h.path_id, h.hash_entry, UNIX_TIMESTAMP(h.timestamp)\n      FROM client_path_stat_entries AS s\n LEFT JOIN client_path_hash_entries AS h\n        ON s.client_id = h.client_id\n       AND s.path_type = h.path_type\n       AND s.path_id = h.path_id\n       AND s.timestamp = h.timestamp\n     WHERE s.client_id = %(client_id)s\n       AND s.path_type = %(path_type)s\n       AND s.path_id IN %(path_ids)s\n     UNION\n    SELECT s.path_id, s.stat_entry, UNIX_TIMESTAMP(s.timestamp),\n           h.path_id, h.hash_entry, UNIX_TIMESTAMP(h.timestamp)\n      FROM client_path_hash_entries AS h\n LEFT JOIN client_path_stat_entries AS s\n        ON h.client_id = s.client_id\n       AND h.path_type = s.path_type\n       AND h.path_id = s.path_id\n       AND h.timestamp = s.timestamp\n     WHERE h.client_id = %(client_id)s\n       AND h.path_type = %(path_type)s\n       AND h.path_id IN %(path_ids)s\n    \"\"\"\n\n    params = {\n        \"client_id\": db_utils.ClientIDToInt(client_id),\n        \"path_type\": int(path_type),\n        \"path_ids\": [path_id.AsBytes() for path_id in path_id_components]\n    }\n\n    cursor.execute(query, params)\n    for row in cursor.fetchall():\n      # pyformat: disable\n      (stat_entry_path_id_bytes, stat_entry_bytes, stat_entry_timestamp,\n       hash_entry_path_id_bytes, hash_entry_bytes, hash_entry_timestamp) = row\n      # pyformat: enable\n\n      path_id_bytes = stat_entry_path_id_bytes or hash_entry_path_id_bytes\n      path_id = rdf_objects.PathID.FromBytes(path_id_bytes)\n      components = path_id_components[path_id]\n\n      timestamp = stat_entry_timestamp or hash_entry_timestamp\n\n      if stat_entry_bytes is not None:\n        stat_entry = rdf_client_fs.StatEntry.FromSerializedString(\n            stat_entry_bytes)\n      else:\n        stat_entry = None\n\n      if hash_entry_bytes is not None:\n        hash_entry = rdf_crypto.Hash.FromSerializedString(hash_entry_bytes)\n      else:\n        hash_entry = None\n\n      path_info = rdf_objects.PathInfo(\n          path_type=path_type,\n          components=components,\n          stat_entry=stat_entry,\n          hash_entry=hash_entry,\n          timestamp=mysql_utils.TimestampToRDFDatetime(timestamp))\n\n      path_infos[components].append(path_info)\n\n    for components in components_list:\n      path_infos[components].sort(key=lambda path_info: path_info.timestamp)\n\n    return path_infos"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef EnumerateInterfacesFromClient(args):\n  del args  # Unused\n\n  libc = ctypes.cdll.LoadLibrary(ctypes.util.find_library(\"c\"))\n  ifa = Ifaddrs()\n  p_ifa = ctypes.pointer(ifa)\n  libc.getifaddrs(ctypes.pointer(p_ifa))\n\n  addresses = {}\n  macs = {}\n  ifs = set()\n\n  m = p_ifa\n  while m:\n    ifname = ctypes.string_at(m.contents.ifa_name)\n    ifs.add(ifname)\n    try:\n      iffamily = ord(m.contents.ifa_addr[0])\n\n      if iffamily == 0x2:  # AF_INET\n        data = ctypes.cast(m.contents.ifa_addr, ctypes.POINTER(Sockaddrin))\n        ip4 = bytes(list(data.contents.sin_addr))\n        address_type = rdf_client_network.NetworkAddress.Family.INET\n        address = rdf_client_network.NetworkAddress(\n            address_type=address_type, packed_bytes=ip4)\n        addresses.setdefault(ifname, []).append(address)\n\n      if iffamily == 0x11:  # AF_PACKET\n        data = ctypes.cast(m.contents.ifa_addr, ctypes.POINTER(Sockaddrll))\n        addlen = data.contents.sll_halen\n        macs[ifname] = bytes(list(data.contents.sll_addr[:addlen]))\n\n      if iffamily == 0xA:  # AF_INET6\n        data = ctypes.cast(m.contents.ifa_addr, ctypes.POINTER(Sockaddrin6))\n        ip6 = bytes(list(data.contents.sin6_addr))\n        address_type = rdf_client_network.NetworkAddress.Family.INET6\n        address = rdf_client_network.NetworkAddress(\n            address_type=address_type, packed_bytes=ip6)\n        addresses.setdefault(ifname, []).append(address)\n    except ValueError:\n      # Some interfaces don't have a iffamily and will raise a null pointer\n      # exception. We still want to send back the name.\n      pass\n\n    m = m.contents.ifa_next\n\n  libc.freeifaddrs(p_ifa)\n\n  for interface in ifs:\n    mac = macs.setdefault(interface, b\"\")\n    address_list = addresses.setdefault(interface, b\"\")\n    args = {\"ifname\": interface}\n    if mac:\n      args[\"mac_address\"] = mac\n    if addresses:\n      args[\"addresses\"] = address_list\n    yield rdf_client_network.Interface(**args)", "response": "Enumerate all interfaces and collect their MAC addresses."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nenumerating all the users on this system.", "response": "def EnumerateUsersFromClient(args):\n  \"\"\"Enumerates all the users on this system.\"\"\"\n\n  del args  # Unused\n\n  users = _ParseWtmp()\n  for user, last_login in iteritems(users):\n\n    # Lose the null termination\n    username, _ = user.split(b\"\\x00\", 1)\n    username = username.decode(\"utf-8\")\n\n    if username:\n      # Somehow the last login time can be < 0. There is no documentation\n      # what this means so we just set it to 0 (the rdfvalue field is\n      # unsigned so we can't send negative values).\n      if last_login < 0:\n        last_login = 0\n\n      result = rdf_client.User(\n          username=username, last_logon=last_login * 1000000)\n\n      try:\n        pwdict = pwd.getpwnam(username)\n        result.homedir = utils.SmartUnicode(pwdict.pw_dir)\n        result.full_name = utils.SmartUnicode(pwdict.pw_gecos)\n        result.uid = pwdict.pw_uid\n        result.gid = pwdict.pw_gid\n        result.shell = utils.SmartUnicode(pwdict.pw_shell)\n      except KeyError:\n        pass\n\n      yield result"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef CheckMounts(filename):\n  with io.open(filename, \"r\") as fd:\n    for line in fd:\n      try:\n        device, mnt_point, fs_type, _ = line.split(\" \", 3)\n      except ValueError:\n        continue\n      if fs_type in ACCEPTABLE_FILESYSTEMS:\n        if os.path.exists(device):\n          yield device, fs_type, mnt_point", "response": "Parses the currently mounted devices."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef EnumerateFilesystemsFromClient(args):\n  del args  # Unused.\n\n  filenames = [\"/proc/mounts\", \"/etc/mtab\"]\n\n  for filename in filenames:\n    for device, fs_type, mnt_point in CheckMounts(filename):\n      yield rdf_client_fs.Filesystem(\n          mount_point=mnt_point, type=fs_type, device=device)", "response": "Enumerate all the filesystems mounted on the system."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _ParseWtmp():\n  users = {}\n\n  wtmp_struct_size = UtmpStruct.GetSize()\n  filenames = glob.glob(\"/var/log/wtmp*\") + [\"/var/run/utmp\"]\n\n  for filename in filenames:\n    try:\n      wtmp = open(filename, \"rb\").read()\n    except IOError:\n      continue\n\n    for offset in range(0, len(wtmp), wtmp_struct_size):\n      try:\n        record = UtmpStruct(wtmp[offset:offset + wtmp_struct_size])\n      except utils.ParsingError:\n        break\n\n      # Users only appear for USER_PROCESS events, others are system.\n      if record.ut_type != 7:\n        continue\n\n      try:\n        if users[record.ut_user] < record.tv_sec:\n          users[record.ut_user] = record.tv_sec\n      except KeyError:\n        users[record.ut_user] = record.tv_sec\n\n  return users", "response": "Parse wtmp and utmp and extract the last logon time."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _InstallRpm(self, path):\n\n    pid = os.fork()\n    if pid == 0:\n      # This is the child that will become the installer process.\n\n      cmd = \"/bin/rpm\"\n      cmd_args = [cmd, \"-U\", \"--replacepkgs\", \"--replacefiles\", path]\n\n      # We need to clean the environment or rpm will fail - similar to the\n      # use_client_context=False parameter.\n      env = os.environ.copy()\n      env.pop(\"LD_LIBRARY_PATH\", None)\n      env.pop(\"PYTHON_PATH\", None)\n\n      # This call doesn't return.\n      os.execve(cmd, cmd_args, env)\n\n    else:\n      # The installer will run in the background and kill the main process\n      # so we just wait. If something goes wrong, the nanny will restart the\n      # service after a short while and the client will come back to life.\n      time.sleep(1000)", "response": "Installs the rpm file at path."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _TempRootPath():\n  # `FLAGS.test_tmpdir` is defined only in test environment, so we can't expect\n  # for it to be always defined.\n  test_tmpdir = (\n      compatibility.Environ(\"TEST_TMPDIR\", default=None) or\n      FLAGS.get_flag_value(\"test_tmpdir\", default=None))\n\n  if not os.path.exists(test_tmpdir):\n    # TODO: We add a try-catch block to avoid rare race condition.\n    # In Python 3 the exception being thrown is way more specific\n    # (`FileExistsError`) but in Python 2 `OSError` is the best we can do. Once\n    # support for Python 2 is dropped we can switch to catching that and remove\n    # the conditional (EAFP).\n    try:\n      os.makedirs(test_tmpdir)\n    except OSError as err:\n      logging.error(err)\n\n  # TODO(hanuszczak): Investigate whether this check still makes sense.\n  if platform.system() == \"Windows\":\n    return None\n\n  return test_tmpdir", "response": "Returns a default root path for storing temporary files."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef TempDirPath(suffix = \"\", prefix = \"tmp\"):\n  precondition.AssertType(suffix, Text)\n  precondition.AssertType(prefix, Text)\n\n  return tempfile.mkdtemp(suffix=suffix, prefix=prefix, dir=_TempRootPath())", "response": "Creates a temporary directory based on the environment configuration."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef TempFilePath(suffix = \"\", prefix = \"tmp\",\n                 dir = None):  # pylint: disable=redefined-builtin\n  \"\"\"Creates a temporary file based on the environment configuration.\n\n  If no directory is specified the file will be placed in folder as specified by\n  the `TEST_TMPDIR` environment variable if available or fallback to\n  `Test.tmpdir` of the current configuration if not.\n\n  If directory is specified it must be part of the default test temporary\n  directory.\n\n  Args:\n    suffix: A suffix to end the file name with.\n    prefix: A prefix to begin the file name with.\n    dir: A directory to place the file in.\n\n  Returns:\n    An absolute path to the created file.\n\n  Raises:\n    ValueError: If the specified directory is not part of the default test\n        temporary directory.\n  \"\"\"\n  precondition.AssertType(suffix, Text)\n  precondition.AssertType(prefix, Text)\n  precondition.AssertOptionalType(dir, Text)\n\n  root = _TempRootPath()\n  if not dir:\n    dir = root\n  elif root and not os.path.commonprefix([dir, root]):\n    raise ValueError(\"path '%s' must start with '%s'\" % (dir, root))\n\n  # `mkstemp` returns an open descriptor for the file. We don't care about it as\n  # we are only concerned with the path, but we need to close it first or else\n  # the file will remain open until the garbage collectors steps in.\n  desc, path = tempfile.mkstemp(suffix=suffix, prefix=prefix, dir=dir)\n  os.close(desc)\n  return path", "response": "Creates a temporary file based on the environment configuration."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef Name(self):\n    return (self.name or self.device_path or self.windowsvolume.drive_letter or\n            self.unixvolume.mount_point or None)", "response": "Return the best available name for this volume."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef Validate(self):\n    self.pathspec.Validate()\n\n    if (self.HasField(\"start_time\") and self.HasField(\"end_time\") and\n        self.start_time > self.end_time):\n      raise ValueError(\"Start time must be before end time.\")\n\n    if not self.path_regex and not self.data_regex and not self.path_glob:\n      raise ValueError(\"A Find specification can not contain both an empty \"\n                       \"path regex and an empty data regex\")", "response": "Ensure the pathspec is valid."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nturn Exported * protos with embedded metadata into a nested dict.", "response": "def _GetNestedDict(self, value):\n    \"\"\"Turn Exported* protos with embedded metadata into a nested dict.\"\"\"\n    row = {}\n    for type_info in value.__class__.type_infos:\n      # We only expect the metadata proto to be included as ProtoEmbedded.\n      if type_info.__class__.__name__ == \"ProtoEmbedded\":\n        row[type_info.name] = self._GetNestedDict(value.Get(type_info.name))\n      else:\n        row[type_info.name] = utils.SmartStr(value.Get(type_info.name))\n\n    return row"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _CreateOutputFileHandles(self, output_type):\n    gzip_filehandle_parent = tempfile.NamedTemporaryFile(suffix=output_type)\n    gzip_filehandle = gzip.GzipFile(gzip_filehandle_parent.name, \"wb\",\n                                    self.GZIP_COMPRESSION_LEVEL,\n                                    gzip_filehandle_parent)\n    self.temp_output_trackers[output_type] = TempOutputTracker(\n        output_type=output_type,\n        gzip_filehandle=gzip_filehandle,\n        gzip_filehandle_parent=gzip_filehandle_parent)\n    return self.temp_output_trackers[output_type]", "response": "Creates a new gzipped output tempfile for the output type."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn the tracker for a given value type.", "response": "def _GetTempOutputFileHandles(self, value_type):\n    \"\"\"Returns the tracker for a given value type.\"\"\"\n    try:\n      return self.temp_output_trackers[value_type], False\n    except KeyError:\n      return self._CreateOutputFileHandles(value_type), True"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef Flush(self, state):\n    self.bigquery = bigquery.GetBigQueryClient()\n    # BigQuery job ids must be alphanum plus dash and underscore.\n    urn_str = self.source_urn.RelativeName(\"aff4:/\").replace(\"/\", \"_\").replace(\n        \":\", \"\").replace(\".\", \"-\")\n\n    for tracker in itervalues(self.temp_output_trackers):\n      # Close out the gzip handle and pass the original file handle to the\n      # bigquery client so it sees the gzip'd content.\n      tracker.gzip_filehandle.write(\"\\n\")\n      tracker.gzip_filehandle.close()\n      tracker.gzip_filehandle_parent.seek(0)\n\n      # e.g. job_id: hunts_HFFE1D044_Results_ExportedFile_1446056474\n      job_id = \"{0}_{1}_{2}\".format(\n          urn_str, tracker.output_type,\n          rdfvalue.RDFDatetime.Now().AsSecondsSinceEpoch())\n\n      # If we have a job id stored, that means we failed last time. Re-use the\n      # job id and append to the same file if it continues to fail. This avoids\n      # writing many files on failure.\n      if tracker.output_type in self.output_jobids:\n        job_id = self.output_jobids[tracker.output_type]\n      else:\n        self.output_jobids[tracker.output_type] = job_id\n\n      if (state.failure_count + self.failure_count >=\n          config.CONFIG[\"BigQuery.max_upload_failures\"]):\n        logging.error(\n            \"Exceeded BigQuery.max_upload_failures for %s, giving up.\",\n            self.source_urn)\n      else:\n        try:\n          self.bigquery.InsertData(tracker.output_type,\n                                   tracker.gzip_filehandle_parent,\n                                   tracker.schema, job_id)\n          self.failure_count = max(0, self.failure_count - 1)\n          del self.output_jobids[tracker.output_type]\n        except bigquery.BigQueryJobUploadError:\n          self.failure_count += 1\n\n    # Now that everything is in bigquery we can remove the output streams\n    self.temp_output_trackers = {}", "response": "Finish writing JSON files upload to Cloud Storage and BigQuery."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nconvert exported rdfvalue into a BigQuery schema.", "response": "def RDFValueToBigQuerySchema(self, value):\n    \"\"\"Convert Exported* rdfvalue into a BigQuery schema.\"\"\"\n    fields_array = []\n    for type_info in value.__class__.type_infos:\n      # Nested structures are indicated by setting type \"RECORD\"\n      if type_info.__class__.__name__ == \"ProtoEmbedded\":\n        fields_array.append({\n            \"name\": type_info.name,\n            \"type\": \"RECORD\",\n            \"description\": type_info.description,\n            \"fields\": self.RDFValueToBigQuerySchema(value.Get(type_info.name))\n        })\n      else:\n        # If we don't have a specific map use string.\n        bq_type = self.RDF_BIGQUERY_TYPE_MAP.get(type_info.proto_type_name,\n                                                 None) or \"STRING\"\n\n        # For protos with RDF types we need to do some more checking to properly\n        # covert types.\n        if hasattr(type_info, \"original_proto_type_name\"):\n          if type_info.original_proto_type_name in [\n              \"RDFDatetime\", \"RDFDatetimeSeconds\"\n          ]:\n            bq_type = \"TIMESTAMP\"\n          elif type_info.proto_type_name == \"uint64\":\n            # This is to catch fields like st_mode which are stored as ints but\n            # exported as more useful strings. Things which are just plain ints\n            # won't have an RDF type specified and so will be exported as\n            # INTEGER\n            bq_type = \"STRING\"\n\n        fields_array.append({\n            \"name\": type_info.name,\n            \"type\": bq_type,\n            \"description\": type_info.description\n        })\n    return fields_array"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef WriteValuesToJSONFile(self, state, values):\n    value_counters = {}\n    max_post_size = config.CONFIG[\"BigQuery.max_file_post_size\"]\n    for value in values:\n      class_name = value.__class__.__name__\n      output_tracker, created = self._GetTempOutputFileHandles(class_name)\n\n      # If our output stream is getting huge we should flush everything now and\n      # set up new output files. Only start checking when we are getting within\n      # range of the limit because we need to flush the stream to check the\n      # size. Start counting at 0 so we check each file the first time.\n      value_counters[class_name] = value_counters.get(class_name, -1) + 1\n      if not value_counters[class_name] % max_post_size // 1000:\n\n        # Flush our temp gzip handle so we can stat it to see how big it is.\n        output_tracker.gzip_filehandle.flush()\n        if os.path.getsize(output_tracker.gzip_filehandle.name) > max_post_size:\n          # Flush what we have and get new temp output handles.\n          self.Flush(state)\n          value_counters[class_name] = 0\n          output_tracker, created = self._GetTempOutputFileHandles(class_name)\n\n      if not output_tracker.schema:\n        output_tracker.schema = self.RDFValueToBigQuerySchema(value)\n\n      if created:\n        # Omit the leading newline for the first entry in the file.\n        self._WriteJSONValue(output_tracker.gzip_filehandle, value)\n      else:\n        self._WriteJSONValue(\n            output_tracker.gzip_filehandle, value, delimiter=\"\\n\")\n\n    for output_tracker in itervalues(self.temp_output_trackers):\n      output_tracker.gzip_filehandle.flush()", "response": "Writes RDF values to a JSON file."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef FilterArgsFromSemanticProtobuf(protobuf, kwargs):\n  for descriptor in protobuf.type_infos:\n    value = kwargs.pop(descriptor.name, None)\n    if value is not None:\n      setattr(protobuf, descriptor.name, value)", "response": "Assign kwargs to the protobuf and remove them from the kwargs dict."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef GetOutputPluginStates(output_plugins, source=None, token=None):\n  output_plugins_states = []\n  for plugin_descriptor in output_plugins:\n    plugin_class = plugin_descriptor.GetPluginClass()\n    try:\n      _, plugin_state = plugin_class.CreatePluginAndDefaultState(\n          source_urn=source, args=plugin_descriptor.plugin_args, token=token)\n    except Exception as e:  # pylint: disable=broad-except\n      raise ValueError(\"Plugin %s failed to initialize (%s)\" %\n                       (plugin_class, e))\n\n    # TODO(amoser): Those do not need to be inside the state, they\n    # could be part of the plugin descriptor.\n    plugin_state[\"logs\"] = []\n    plugin_state[\"errors\"] = []\n\n    output_plugins_states.append(\n        rdf_flow_runner.OutputPluginState(\n            plugin_state=plugin_state, plugin_descriptor=plugin_descriptor))\n\n  return output_plugins_states", "response": "Initializes state for a list of output plugins."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef Flush(self):\n    self._CheckLeaseAndFlush()\n    self.Load()\n    super(FlowBase, self).Flush()\n    # Writing the messages queued in the queue_manager of the runner always has\n    # to be the last thing that happens or we will have a race condition.\n    self.FlushMessages()", "response": "Flushes the flow and all its requests to the data_store."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nflushing the flow and all its requests to the data_store.", "response": "def Close(self):\n    \"\"\"Flushes the flow and all its requests to the data_store.\"\"\"\n    self._CheckLeaseAndFlush()\n    super(FlowBase, self).Close()\n    # Writing the messages queued in the queue_manager of the runner always has\n    # to be the last thing that happens or we will have a race condition.\n    self.FlushMessages()"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef CreateRunner(self, **kw):\n    self.runner = flow_runner.FlowRunner(self, token=self.token, **kw)\n    return self.runner", "response": "Create a new runner."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef NotifyAboutEnd(self):\n    flow_ref = None\n    if self.runner_args.client_id:\n      flow_ref = rdf_objects.FlowReference(\n          client_id=self.client_id, flow_id=self.urn.Basename())\n\n    num_results = len(self.ResultCollection())\n    notification_lib.Notify(\n        self.creator, rdf_objects.UserNotification.Type.TYPE_FLOW_RUN_COMPLETED,\n        \"Flow %s completed with %d %s\" %\n        (self.__class__.__name__, num_results, num_results == 1 and \"result\" or\n         \"results\"),\n        rdf_objects.ObjectReference(\n            reference_type=rdf_objects.ObjectReference.Type.FLOW,\n            flow=flow_ref))", "response": "Send out a final notification about the end of this flow."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef MarkForTermination(cls, flow_urn, reason=None, mutation_pool=None):\n    # Doing a blind write here using low-level data store API. Accessing\n    # the flow via AFF4 is not really possible here, because it forces a state\n    # to be written in Close() method.\n    if mutation_pool is None:\n      raise ValueError(\"Mutation pool can't be none.\")\n\n    mutation_pool.Set(\n        flow_urn,\n        cls.SchemaCls.PENDING_TERMINATION.predicate,\n        rdf_flow_objects.PendingFlowTermination(reason=reason),\n        replace=False)", "response": "Marks the flow as termination."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nterminating a flow. Args: flow_id: The flow session_id to terminate. reason: A reason to log. status: Status code used in the generated status message. token: The access token to be used for this request. Raises: FlowError: If the flow can not be found.", "response": "def TerminateAFF4Flow(cls, flow_id, reason=None, status=None, token=None):\n    \"\"\"Terminate a flow.\n\n    Args:\n      flow_id: The flow session_id to terminate.\n      reason: A reason to log.\n      status: Status code used in the generated status message.\n      token: The access token to be used for this request.\n\n    Raises:\n      FlowError: If the flow can not be found.\n    \"\"\"\n    flow_obj = aff4.FACTORY.Open(\n        flow_id, aff4_type=GRRFlow, mode=\"rw\", token=token)\n\n    if not flow_obj:\n      raise FlowError(\"Could not terminate flow %s\" % flow_id)\n\n    with flow_obj:\n      runner = flow_obj.GetRunner()\n      if not runner.IsRunning():\n        return\n\n      if token is None:\n        token = access_control.ACLToken()\n\n      if reason is None:\n        reason = \"Manual termination by console.\"\n\n      # This calls runner.Terminate to kill the flow\n      runner.Error(reason, status_code=status)\n\n      flow_obj.Log(\"Terminated by user {0}. Reason: {1}\".format(\n          token.username, reason))\n\n      # From now on we run with supervisor access\n      super_token = token.SetUID()\n\n      # Also terminate its children\n      children_to_kill = aff4.FACTORY.MultiOpen(\n          flow_obj.ListChildren(), token=super_token, aff4_type=GRRFlow)\n\n      for child_obj in children_to_kill:\n        cls.TerminateAFF4Flow(\n            child_obj.urn, reason=\"Parent flow terminated.\", token=super_token)"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef ResultCollectionForFID(cls, flow_id):\n    # TODO: Disallow/remove URNs after migration.\n    if not isinstance(flow_id, rdfvalue.RDFURN):\n      flow_id = rdfvalue.RDFURN(flow_id)\n\n    return sequential_collection.GeneralIndexedCollection(\n        flow_id.Add(RESULTS_SUFFIX))", "response": "Returns the ResultCollection for a given flow_id."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nget all well known flows.", "response": "def GetAllWellKnownFlows(cls, token=None):\n    \"\"\"Get instances of all well known flows.\"\"\"\n    well_known_flows = {}\n    for cls in itervalues(registry.AFF4FlowRegistry.FLOW_REGISTRY):\n      if issubclass(cls, WellKnownFlow) and cls.well_known_session_id:\n        well_known_flow = cls(cls.well_known_session_id, mode=\"rw\", token=token)\n        well_known_flows[cls.well_known_session_id.FlowName()] = well_known_flow\n\n    return well_known_flows"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nremoving all requests and responses from the queue and returns them.", "response": "def FetchAndRemoveRequestsAndResponses(self, session_id):\n    \"\"\"Removes WellKnownFlow messages from the queue and returns them.\"\"\"\n    messages = []\n    with queue_manager.WellKnownQueueManager(token=self.token) as manager:\n      for response in manager.FetchResponses(session_id):\n        messages.append(response)\n      manager.DeleteWellKnownFlowResponses(session_id, messages)\n\n    return messages"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef ProcessResponses(self, responses, thread_pool):\n    for response in responses:\n      thread_pool.AddTask(\n          target=self._SafeProcessMessage,\n          args=(response,),\n          name=self.__class__.__name__)", "response": "For WellKnownFlows we receive these messages directly."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef AsIPAddr(self):\n    if self.packed_bytes is None:\n      return None\n\n    packed_bytes = self.packed_bytes.AsBytes()\n\n    if self.address_type == NetworkAddress.Family.INET:\n      return ipaddress.IPv4Address(packed_bytes)\n    if self.address_type == NetworkAddress.Family.INET6:\n      return ipaddress.IPv6Address(packed_bytes)\n\n    message = \"IP address has invalid type: {}\".format(self.address_type)\n    raise ValueError(message)", "response": "Returns the IP as an IPv4Address object ( if packed bytes are defined )."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn a list of IP addresses.", "response": "def GetIPAddresses(self):\n    \"\"\"Return a list of IP addresses.\"\"\"\n    results = []\n    for address in self.addresses:\n      human_readable_address = address.human_readable_address\n      if human_readable_address is not None:\n        results.append(human_readable_address)\n\n    return results"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _CheckConditionsShortCircuit(content_conditions, pathspec):\n  matches = []\n  for cond in content_conditions:\n    with vfs.VFSOpen(pathspec) as vfs_file:\n      cur_matches = list(cond.Search(vfs_file))\n    if cur_matches:\n      matches.extend(cur_matches)\n    else:  # As soon as one condition does not match, we skip the file.\n      return []  # Return no matches to indicate skipping this file.\n  return matches", "response": "Checks all content_conditions until one yields no matches."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nyield all possible expansions from given path patterns.", "response": "def _GetExpandedPaths(args):\n  \"\"\"Yields all possible expansions from given path patterns.\"\"\"\n  opts = globbing.PathOpts(\n      follow_links=args.follow_links, pathtype=args.pathtype)\n\n  for path in args.paths:\n    for expanded_path in globbing.ExpandPath(str(path), opts):\n      yield expanded_path"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef RegistryKeyFromClient(args):\n  for path in _GetExpandedPaths(args):\n    pathspec = rdf_paths.PathSpec(\n        path=path, pathtype=rdf_paths.PathSpec.PathType.REGISTRY)\n    with vfs.VFSOpen(pathspec) as file_obj:\n      yield file_obj.Stat()", "response": "This function expands paths from the args and returns registry keys."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef GetClientVersion(client_id, token=None):\n  if data_store.RelationalDBEnabled():\n    sinfo = data_store.REL_DB.ReadClientStartupInfo(client_id=client_id)\n    if sinfo is not None:\n      return sinfo.client_info.client_version\n    else:\n      return config.CONFIG[\"Source.version_numeric\"]\n  else:\n    with aff4.FACTORY.Open(client_id, token=token) as client:\n      cinfo = client.Get(client.Schema.CLIENT_INFO)\n      if cinfo is not None:\n        return cinfo.client_version\n      else:\n        return config.CONFIG[\"Source.version_numeric\"]", "response": "Retrieves the GRR version that the client used."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef GetClientOs(client_id, token=None):\n  if data_store.RelationalDBEnabled():\n    kb = data_store.REL_DB.ReadClientSnapshot(client_id).knowledge_base\n  else:\n    with aff4.FACTORY.Open(client_id, token=token) as client:\n      kb = client.Get(client.Schema.KNOWLEDGE_BASE)\n\n  return kb.os", "response": "Retrieves the last known operating system name that the client used."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef GetFileHashEntry(fd):\n  # Hash file store is not migrated to RELDB just yet, hence the first check.\n  if (not fd.urn.Path().startswith(\"/files/hash/generic\") and\n      data_store.RelationalDBEnabled()):\n    return GetUrnHashEntry(fd.urn)\n  else:\n    return fd.Get(fd.Schema.HASH)", "response": "Returns an rdf_crypto. Hash instance for given AFF4 file descriptor."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn an rdf_crypto. Hash instance for given URN of an AFF4 file.", "response": "def GetUrnHashEntry(urn, token=None):\n  \"\"\"Returns an `rdf_crypto.Hash` instance for given URN of an AFF4 file.\"\"\"\n  if data_store.RelationalDBEnabled():\n    client_id, vfs_path = urn.Split(2)\n    path_type, components = rdf_objects.ParseCategorizedPath(vfs_path)\n\n    path_info = data_store.REL_DB.ReadPathInfo(client_id, path_type, components)\n    return path_info.hash_entry\n  else:\n    with aff4.FACTORY.Open(urn, token=token) as fd:\n      return GetFileHashEntry(fd)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef EnumMissingModules():\n  module_handle = ctypes.c_ulong()\n  count = ctypes.c_ulong()\n  process_handle = ctypes.windll.kernel32.OpenProcess(\n      PROCESS_QUERY_INFORMATION\n      | PROCESS_VM_READ, 0, os.getpid())\n  ctypes.windll.psapi.EnumProcessModules(process_handle,\n                                         ctypes.byref(module_handle),\n                                         ctypes.sizeof(module_handle),\n                                         ctypes.byref(count))\n\n  # The size of a handle is pointer size (i.e. 64 bit of amd64 and 32 bit on\n  # i386).\n  if sys.maxsize > 2**32:\n    handle_type = ctypes.c_ulonglong\n  else:\n    handle_type = ctypes.c_ulong\n\n  module_list = (handle_type * (count.value // ctypes.sizeof(handle_type)))()\n\n  ctypes.windll.psapi.EnumProcessModulesEx(process_handle,\n                                           ctypes.byref(module_list),\n                                           ctypes.sizeof(module_list),\n                                           ctypes.byref(count), 2)\n\n  for x in module_list:\n    module_filename = win32process.GetModuleFileNameEx(process_handle, x)\n    for pattern in MODULE_PATTERNS:\n      if pattern.match(os.path.basename(module_filename)):\n        yield module_filename\n\n  for venv_file in FILES_FROM_VIRTUALENV:\n    yield os.path.join(sys.prefix, venv_file)", "response": "Enumerate all modules which do not match the patterns MODULE_PATTERNS."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef CopyFileInZip(from_zip, from_name, to_zip, to_name=None):\n  data = from_zip.read(from_name)\n  if to_name is None:\n    to_name = from_name\n  to_zip.writestr(to_name, data)", "response": "Copy a file from a ZipFile and write it to a new ZipFile."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef BuildNanny(self):\n    # When running under cygwin, the following environment variables are not set\n    # (since they contain invalid chars). Visual Studio requires these or it\n    # will fail.\n    os.environ[\"ProgramFiles(x86)\"] = r\"C:\\Program Files (x86)\"\n    self.nanny_dir = os.path.join(self.build_dir, \"grr\", \"client\",\n                                  \"grr_response_client\", \"nanny\")\n    nanny_src_dir = config.CONFIG.Get(\n        \"ClientBuilder.nanny_source_dir\", context=self.context)\n    logging.info(\"Copying Nanny build files from %s to %s\", nanny_src_dir,\n                 self.nanny_dir)\n\n    shutil.copytree(\n        config.CONFIG.Get(\n            \"ClientBuilder.nanny_source_dir\", context=self.context),\n        self.nanny_dir)\n\n    build_type = config.CONFIG.Get(\n        \"ClientBuilder.build_type\", context=self.context)\n\n    vs_arch = config.CONFIG.Get(\n        \"ClientBuilder.vs_arch\", default=None, context=self.context)\n\n    # We have to set up the Visual Studio environment first and then call\n    # msbuild.\n    env_script = config.CONFIG.Get(\n        \"ClientBuilder.vs_env_script\", default=None, context=self.context)\n\n    if vs_arch is None or env_script is None or not os.path.exists(env_script):\n      # Visual Studio is not installed. We just use pre-built binaries in that\n      # case.\n      logging.warning(\n          \"Visual Studio does not appear to be installed, \"\n          \"Falling back to prebuilt GRRNanny binaries.\"\n          \"If you want to build it you must have VS 2012 installed.\")\n\n      binaries_dir = config.CONFIG.Get(\n          \"ClientBuilder.nanny_prebuilt_binaries\", context=self.context)\n\n      shutil.copy(\n          os.path.join(binaries_dir, \"GRRNanny_%s.exe\" % vs_arch),\n          os.path.join(self.output_dir, \"GRRservice.exe\"))\n\n    else:\n      # Lets build the nanny with the VS env script.\n      subprocess.check_call(\n          \"cmd /c \\\"\\\"%s\\\" && msbuild /p:Configuration=%s;Platform=%s\\\"\" %\n          (env_script, build_type, vs_arch),\n          cwd=self.nanny_dir)\n\n      # The templates always contain the same filenames - the repack step might\n      # rename them later.\n      shutil.copy(\n          os.path.join(self.nanny_dir, vs_arch, build_type, \"GRRNanny.exe\"),\n          os.path.join(self.output_dir, \"GRRservice.exe\"))", "response": "Use VS 2012 to build the Windows Nanny service."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ngenerates an executable template.", "response": "def MakeExecutableTemplate(self, output_file=None):\n    \"\"\"Windows templates also include the nanny.\"\"\"\n    super(WindowsClientBuilder,\n          self).MakeExecutableTemplate(output_file=output_file)\n\n    self.MakeBuildDirectory()\n    self.BuildWithPyInstaller()\n\n    # Get any dll's that pyinstaller forgot:\n    for module in EnumMissingModules():\n      logging.info(\"Copying additional dll %s.\", module)\n      shutil.copy(module, self.output_dir)\n\n    self.BuildNanny()\n\n    # Generate a prod and a debug version of nanny executable.\n    shutil.copy(\n        os.path.join(self.output_dir, \"GRRservice.exe\"),\n        os.path.join(self.output_dir, \"dbg_GRRservice.exe\"))\n    with open(os.path.join(self.output_dir, \"GRRservice.exe\"), \"r+\") as fd:\n      build.SetPeSubsystem(fd, console=False)\n    with open(os.path.join(self.output_dir, \"dbg_GRRservice.exe\"), \"r+\") as fd:\n      build.SetPeSubsystem(fd, console=True)\n\n    # Generate a prod and a debug version of client executable.\n    shutil.copy(\n        os.path.join(self.output_dir, \"grr-client.exe\"),\n        os.path.join(self.output_dir, \"dbg_grr-client.exe\"))\n    with open(os.path.join(self.output_dir, \"grr-client.exe\"), \"r+\") as fd:\n      build.SetPeSubsystem(fd, console=False)\n    with open(os.path.join(self.output_dir, \"dbg_grr-client.exe\"), \"r+\") as fd:\n      build.SetPeSubsystem(fd, console=True)\n\n    self.MakeZip(self.output_dir, self.template_file)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncheck password with legacy crypt based method.", "response": "def _CheckLegacyPassword(self, password):\n    \"\"\"Check password with legacy crypt based method.\"\"\"\n    # This import will fail on Windows.\n    import crypt  # pylint: disable=g-import-not-at-top\n    salt = self._value[:2]\n    return crypt.crypt(password, salt) == self._value"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nsending an AFF4 - based notification to the user.", "response": "def Notify(self, message_type, subject, msg, source):\n    \"\"\"Send an AFF4-based notification to the user in the UI.\n\n    Args:\n      message_type: One of aff4_grr.Notification.notification_types e.g.\n        \"ViewObject\", \"HostInformation\", \"GrantAccess\" or\n        the same with an added \":[new-style notification type] suffix, e.g.\n        \"ViewObject:TYPE_CLIENT_INTERROGATED\".\n      subject: The subject to use, normally a URN.\n      msg: The message to display.\n      source: The class doing the notification.\n\n    Raises:\n      TypeError: On invalid message_type.\n    \"\"\"\n    pending = self.Get(self.Schema.PENDING_NOTIFICATIONS)\n    if pending is None:\n      pending = self.Schema.PENDING_NOTIFICATIONS()\n\n    # This is a legacy code that should go away after RELDB migration is done.\n    # RELDB notifications have an explicitly stored notification type (that\n    # is then mapped to ApiNotification.notification_type). However, legacy\n    # AFF4-based data model notification types are different from the newly\n    # introduced ones (new types are very concrete, i.e. \"vfs file download\n    # completed\", legacy ones are very generic, i.e. \"ViewObject\").\n    #\n    # In order to temporarily keep the legacy code working and keep the tests\n    # passing, we \"pack\" new-style notification type into an old style\n    # \"message_type\". I.e. what was ViewObject before, will become\n    # ViewObject:TYPE_CLIENT_INTERROGATED.\n    #\n    # Notifications without the \":[notification type]\" suffix are also still\n    # supported for backwards compatibility reasonds. They will be treated\n    # as notifications with an unknown new-style type.\n    if message_type.split(\n        \":\", 2)[0] not in rdf_flows.Notification.notification_types:\n      raise TypeError(\"Invalid notification type %s\" % message_type)\n\n    pending.Append(\n        type=message_type,\n        subject=subject,\n        message=msg,\n        source=source,\n        timestamp=int(time.time() * 1e6))\n\n    # Limit the notification to 50, expiring older notifications.\n    while len(pending) > 50:\n      pending.Pop(0)\n\n    self.Set(self.Schema.PENDING_NOTIFICATIONS, pending)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef DeletePendingNotification(self, timestamp):\n    shown_notifications = self.Get(self.Schema.SHOWN_NOTIFICATIONS)\n    if not shown_notifications:\n      shown_notifications = self.Schema.SHOWN_NOTIFICATIONS()\n\n    pending = self.Get(self.Schema.PENDING_NOTIFICATIONS)\n    if not pending:\n      return\n\n    # Remove all notifications with the given timestamp from pending\n    # and add them to the shown notifications.\n    delete_count = 0\n    for idx in reversed(range(0, len(pending))):\n      if pending[idx].timestamp == timestamp:\n        shown_notifications.Append(pending[idx])\n        pending.Pop(idx)\n        delete_count += 1\n\n    if delete_count > 1:\n      raise UniqueKeyError(\"Multiple notifications at %s\" % timestamp)\n\n    self.Set(self.Schema.PENDING_NOTIFICATIONS, pending)\n    self.Set(self.Schema.SHOWN_NOTIFICATIONS, shown_notifications)", "response": "Deletes the pending notification with the given timestamp."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef ShowNotifications(self, reset=True):\n    shown_notifications = self.Schema.SHOWN_NOTIFICATIONS()\n\n    # Pending notifications first\n    pending = self.Get(self.Schema.PENDING_NOTIFICATIONS, [])\n    for notification in pending:\n      shown_notifications.Append(notification)\n\n    notifications = self.Get(self.Schema.SHOWN_NOTIFICATIONS, [])\n    for notification in notifications:\n      shown_notifications.Append(notification)\n\n    # Shall we reset the pending notification state?\n    if reset:\n      self.Set(shown_notifications)\n      self.Set(self.Schema.PENDING_NOTIFICATIONS())\n      self.Flush()\n\n    return shown_notifications", "response": "A generator of current notifications."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn a description of this user.", "response": "def Describe(self):\n    \"\"\"Return a description of this user.\"\"\"\n    result = [\"\\nUsername: %s\" % self.urn.Basename()]\n    labels = [l.name for l in self.GetLabels()]\n    result.append(\"Labels: %s\" % \",\".join(labels))\n\n    if self.Get(self.Schema.PASSWORD) is None:\n      result.append(\"Password: not set\")\n    else:\n      result.append(\"Password: set\")\n\n    return \"\\n\".join(result)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ngetting the metric value corresponding to the given field values.", "response": "def Get(self, fields=None):\n    \"\"\"Gets the metric value corresponding to the given field values.\"\"\"\n    if not self._field_defs and fields:\n      raise ValueError(\"Metric was registered without fields, \"\n                       \"but following fields were provided: %s.\" % (fields,))\n\n    if self._field_defs and not fields:\n      raise ValueError(\"Metric was registered with fields (%s), \"\n                       \"but no fields were provided.\" % self._field_defs)\n\n    if self._field_defs and fields and len(self._field_defs) != len(fields):\n      raise ValueError(\n          \"Metric was registered with %d fields (%s), but \"\n          \"%d fields were provided (%s).\" % (len(\n              self._field_defs), self._field_defs, len(fields), fields))\n\n    metric_value = self._metric_values.get(_FieldsToKey(fields))\n    return self._DefaultValue() if metric_value is None else metric_value"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef Increment(self, delta, fields=None):\n    if delta < 0:\n      raise ValueError(\n          \"Counter increment should not be < 0 (received: %d)\" % delta)\n\n    self._metric_values[_FieldsToKey(fields)] = self.Get(fields=fields) + delta", "response": "Increments the value of the counter by a given delta."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nrecording the given observation in a distribution.", "response": "def Record(self, value, fields=None):\n    \"\"\"Records the given observation in a distribution.\"\"\"\n    key = _FieldsToKey(fields)\n    metric_value = self._metric_values.get(key)\n    if metric_value is None:\n      metric_value = self._DefaultValue()\n      self._metric_values[key] = metric_value\n    metric_value.Record(value)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef Set(self, value, fields=None):\n    self._metric_values[_FieldsToKey(fields)] = self._value_type(value)", "response": "Sets the metric s current value."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef Get(self, fields=None):\n    result = super(_GaugeMetric, self).Get(fields=fields)\n    if callable(result):\n      return result()\n    else:\n      return result", "response": "Returns current metric s value."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef IncrementCounter(self, metric_name, delta=1, fields=None):\n    if delta < 0:\n      raise ValueError(\"Invalid increment for counter: %d.\" % delta)\n    self._counter_metrics[metric_name].Increment(delta, fields)", "response": "Increment the value of a counter."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nset the value of a gauge.", "response": "def SetGaugeValue(self, metric_name, value, fields=None):\n    \"\"\"See base class.\"\"\"\n    self._gauge_metrics[metric_name].Set(value, fields)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef SetGaugeCallback(self, metric_name, callback, fields=None):\n    self._gauge_metrics[metric_name].SetCallback(callback, fields)", "response": "Sets a callback for a gauge metric."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nfetch the metric object corresponding to the given name.", "response": "def _GetMetric(self, metric_name):\n    \"\"\"Fetches the metric object corresponding to the given name.\"\"\"\n    if metric_name in self._counter_metrics:\n      return self._counter_metrics[metric_name]\n    elif metric_name in self._event_metrics:\n      return self._event_metrics[metric_name]\n    elif metric_name in self._gauge_metrics:\n      return self._gauge_metrics[metric_name]\n    else:\n      raise ValueError(\"Metric %s is not registered.\" % metric_name)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef Parse(self, cmd, args, stdout, stderr, return_val, time_taken,\n            knowledge_base):\n    \"\"\"Parse the yum output.\"\"\"\n    _ = stderr, time_taken, args, knowledge_base  # Unused.\n    self.CheckReturn(cmd, return_val)\n    packages = []\n    for line in stdout.decode(\"utf-8\").splitlines()[1:]:  # Ignore first line\n      cols = line.split()\n      name_arch, version, source = cols\n      name, arch = name_arch.split(\".\")\n\n      status = rdf_client.SoftwarePackage.InstallState.INSTALLED\n      packages.append(\n          rdf_client.SoftwarePackage(\n              name=name,\n              publisher=source,\n              version=version,\n              architecture=arch,\n              install_state=status))\n\n    if packages:\n      yield rdf_client.SoftwarePackages(packages=packages)", "response": "Parse the yum output."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef Parse(self, cmd, args, stdout, stderr, return_val, time_taken,\n            knowledge_base):\n    \"\"\"Parse the yum repolist output.\"\"\"\n    _ = stderr, time_taken, args, knowledge_base  # Unused.\n    self.CheckReturn(cmd, return_val)\n    output = iter(stdout.decode(\"utf-8\").splitlines())\n\n    repo_regexes = {\n        \"name\": self._re_compile(\"Repo-name\"),\n        \"revision\": self._re_compile(\"Repo-revision\"),\n        \"last_update\": self._re_compile(\"Repo-updated\"),\n        \"num_packages\": self._re_compile(\"Repo-pkgs\"),\n        \"size\": self._re_compile(\"Repo-size\"),\n        \"baseurl\": self._re_compile(\"Repo-baseurl\"),\n        \"timeout\": self._re_compile(\"Repo-expire\")\n    }\n\n    repo_id_re = self._re_compile(\"Repo-id\")\n\n    for line in output:\n      match = repo_id_re.match(line)\n      if match:\n        repo_info = rdf_client.PackageRepository()\n        setattr(repo_info, \"id\", match.group(1).strip())\n        while line:\n          for attr, regex in iteritems(repo_regexes):\n            match = regex.match(line)\n            if match:\n              setattr(repo_info, attr, match.group(1).strip())\n              break\n          line = next(output)\n        yield repo_info", "response": "Parse the yum repolist output."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nparse the rpm - qa output.", "response": "def Parse(self, cmd, args, stdout, stderr, return_val, time_taken,\n            knowledge_base):\n    \"\"\"Parse the rpm -qa output.\"\"\"\n    _ = time_taken, args, knowledge_base  # Unused.\n    rpm_re = re.compile(r\"^(\\w[-\\w\\+]+?)-(\\d.*)$\")\n    self.CheckReturn(cmd, return_val)\n    packages = []\n    for line in stdout.splitlines():\n      pkg_match = rpm_re.match(line.strip())\n      if pkg_match:\n        name, version = pkg_match.groups()\n        status = rdf_client.SoftwarePackage.InstallState.INSTALLED\n        packages.append(\n            rdf_client.SoftwarePackage(\n                name=name, version=version, install_state=status))\n    if packages:\n      yield rdf_client.SoftwarePackages(packages=packages)\n\n    for line in stderr.splitlines():\n      if \"error: rpmdbNextIterator: skipping h#\" in line:\n        yield rdf_anomaly.Anomaly(\n            type=\"PARSER_ANOMALY\", symptom=\"Broken rpm database.\")\n        break"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nparses the dpkg output.", "response": "def Parse(self, cmd, args, stdout, stderr, return_val, time_taken,\n            knowledge_base):\n    \"\"\"Parse the dpkg output.\"\"\"\n    _ = stderr, time_taken, args, knowledge_base  # Unused.\n    self.CheckReturn(cmd, return_val)\n    column_lengths = []\n    i = 0\n    for i, line in enumerate(stdout.decode(\"utf-8\").splitlines()):\n      if line.startswith(\"+++-\"):\n        # This is a special header line that determines column size.\n        for col in line.split(\"-\")[1:]:\n          if not re.match(\"=*\", col):\n            raise parser.ParseError(\"Invalid header parsing for %s at line \"\n                                    \"%s\" % (cmd, i))\n          column_lengths.append(len(col))\n        break\n\n    if not column_lengths:\n      return\n\n    packages = []\n\n    remaining_lines = stdout.splitlines()[i + 1:]\n    for i, line in enumerate(remaining_lines):\n      cols = line.split(None, len(column_lengths))\n\n      # The status column is ignored in column_lengths.\n      if len(column_lengths) == 4:\n        # Installed, Name, Version, Architecture, Description\n        status, name, version, arch, desc = cols\n      elif len(column_lengths) == 3:\n        # Older versions of dpkg don't print Architecture\n        status, name, version, desc = cols\n        arch = None\n      else:\n        raise ValueError(\"Bad number of columns in dpkg --list output: %s\" %\n                         len(column_lengths))\n\n      # Status is potentially 3 columns, but always at least two, desired and\n      # actual state. We only care about actual state.\n      if status[1] == \"i\":\n        status = rdf_client.SoftwarePackage.InstallState.INSTALLED\n      else:\n        status = rdf_client.SoftwarePackage.InstallState.UNKNOWN\n      packages.append(\n          rdf_client.SoftwarePackage(\n              name=name,\n              description=desc,\n              version=version,\n              architecture=arch,\n              install_state=status))\n\n    if packages:\n      yield rdf_client.SoftwarePackages(packages=packages)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nparsing the dmidecode output into a dictionary.", "response": "def Parse(self, cmd, args, stdout, stderr, return_val, time_taken,\n            knowledge_base):\n    \"\"\"Parse the dmidecode output. All data is parsed into a dictionary.\"\"\"\n    _ = stderr, time_taken, args, knowledge_base  # Unused.\n    self.CheckReturn(cmd, return_val)\n    output = iter(stdout.decode(\"utf-8\").splitlines())\n\n    # Compile all regexes in advance.\n    sys_info_re = re.compile(r\"\\s*System Information\")\n    sys_regexes = {\n        \"system_manufacturer\": self._re_compile(\"Manufacturer\"),\n        \"serial_number\": self._re_compile(\"Serial Number\"),\n        \"system_product_name\": self._re_compile(\"Product Name\"),\n        \"system_uuid\": self._re_compile(\"UUID\"),\n        \"system_sku_number\": self._re_compile(\"SKU Number\"),\n        \"system_family\": self._re_compile(\"Family\"),\n        \"system_assettag\": self._re_compile(\"Asset Tag\")\n    }\n\n    bios_info_re = re.compile(r\"\\s*BIOS Information\")\n    bios_regexes = {\n        \"bios_vendor\": self._re_compile(\"Vendor\"),\n        \"bios_version\": self._re_compile(\"Version\"),\n        \"bios_release_date\": self._re_compile(\"Release Date\"),\n        \"bios_rom_size\": self._re_compile(\"ROM Size\"),\n        \"bios_revision\": self._re_compile(\"BIOS Revision\")\n    }\n\n    # Initialize RDF.\n    dmi_info = rdf_client.HardwareInfo()\n\n    for line in output:\n      if sys_info_re.match(line):\n        # Collect all System Information until we hit a blank line.\n        while line:\n          for attr, regex in iteritems(sys_regexes):\n            match = regex.match(line)\n            if match:\n              setattr(dmi_info, attr, match.group(1).strip())\n              break\n          line = next(output)\n\n      elif bios_info_re.match(line):\n        # Collect all BIOS Information until we hit a blank line.\n        while line:\n          for attr, regex in iteritems(bios_regexes):\n            match = regex.match(line)\n            if match:\n              setattr(dmi_info, attr, match.group(1).strip())\n              break\n          line = next(output)\n\n    yield dmi_info"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nparses the ps output.", "response": "def Parse(self, cmd, args, stdout, stderr, return_val, time_taken,\n            knowledge_base):\n    \"\"\"Parse the ps output.\n\n    Note that cmdline consumes every field up to the end of line\n    and as it is string, we can't perfectly see what the arguments\n    on the command line really were. We just assume a space is the arg\n    seperator. It's imperfect, but it's better than nothing.\n    Obviously, if cmd/cmdline is specified, it must be the last\n    column of output.\n\n    Args:\n      cmd: A string containing the base command that was run.\n      args: A list of strings containing the commandline args for the command.\n      stdout: A string containing the stdout of the command run.\n      stderr: A string containing the stderr of the command run. (Unused)\n      return_val: The return code following command execution.\n      time_taken: The time taken to run the process. (Unused)\n      knowledge_base: An RDF KnowledgeBase. (Unused)\n\n    Yields:\n      RDF Process objects.\n    \"\"\"\n\n    _ = stderr, time_taken, knowledge_base  # Unused.\n    self.CheckReturn(cmd, return_val)\n\n    lines = stdout.splitlines()[1:]  # First line is just a header.\n    for line in lines:\n      try:\n        uid, pid, ppid, c, _, tty, _, cmd = line.split(None, 7)\n\n        rdf_process = rdf_client.Process()\n        rdf_process.username = uid\n        rdf_process.pid = int(pid)\n        rdf_process.ppid = int(ppid)\n        rdf_process.cpu_percent = float(c)\n        rdf_process.terminal = tty\n        rdf_process.cmdline = cmd.split()\n        yield rdf_process\n      except ValueError as error:\n        message = \"Error while parsing `ps` output line '%s': %s\"\n        logging.warning(message, line, error)"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nvalidate an IAP JWT for your Compute Engine service.", "response": "def ValidateIapJwtFromComputeEngine(iap_jwt, cloud_project_number,\n                                    backend_service_id):\n  \"\"\"Validates an IAP JWT for your (Compute|Container) Engine service.\n\n  Args:\n    iap_jwt: The contents of the X-Goog-IAP-JWT-Assertion header.\n    cloud_project_number: The project *number* for your Google Cloud project.\n      This is returned by 'gcloud projects describe $PROJECT_ID', or in the\n      Project Info card in Cloud Console.\n    backend_service_id: The ID of the backend service used to access the\n      application. See https://cloud.google.com/iap/docs/signed-headers-howto\n        for details on how to get this value.\n\n  Returns:\n    A tuple of (user_id, user_email).\n\n  Raises:\n    IAPValidationFailedError: if the validation has failed.\n  \"\"\"\n  expected_audience = \"/projects/{}/global/backendServices/{}\".format(\n      cloud_project_number, backend_service_id)\n  return ValidateIapJwt(iap_jwt, expected_audience)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef ValidateIapJwt(iap_jwt, expected_audience):\n\n  try:\n    key_id = jwt.get_unverified_header(iap_jwt).get(\"kid\")\n    if not key_id:\n      raise IAPValidationFailedError(\"No key ID\")\n    key = GetIapKey(key_id)\n    decoded_jwt = jwt.decode(\n        iap_jwt, key, algorithms=[\"ES256\"], audience=expected_audience)\n    return (decoded_jwt[\"sub\"], decoded_jwt[\"email\"])\n  except (jwt.exceptions.InvalidTokenError,\n          requests.exceptions.RequestException) as e:\n    raise IAPValidationFailedError(e)", "response": "Validates an IAP JWT."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nretrieving a public key from the list published by Identity - aware Proxy.", "response": "def GetIapKey(key_id):\n  \"\"\"Retrieves a public key from the list published by Identity-Aware Proxy.\n\n  The key file is re-fetched if necessary.\n\n  Args:\n    key_id: Key id.\n\n  Returns:\n    String with a key.\n\n  Raises:\n    KeyNotFoundError: if the key is not found in the key file.\n    KeysCanNotBeFetchedError: if the key file can't be fetched.\n  \"\"\"\n  global _KEY_CACHE\n  key = _KEY_CACHE.get(key_id)\n  if not key:\n    # Re-fetch the key file.\n    resp = requests.get(\"https://www.gstatic.com/iap/verify/public_key\")\n    if resp.status_code != 200:\n      raise KeysCanNotBeFetchedError(\n          \"Unable to fetch IAP keys: {} / {} / {}\".format(\n              resp.status_code, resp.headers, resp.text))\n    _KEY_CACHE = resp.json()\n    key = _KEY_CACHE.get(key_id)\n    if not key:\n      raise KeyNotFoundError(\"Key {!r} not found\".format(key_id))\n  return key"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef InitHttp(api_endpoint=None,\n             page_size=None,\n             auth=None,\n             proxies=None,\n             verify=None,\n             cert=None,\n             trust_env=True):\n  \"\"\"Inits an GRR API object with a HTTP connector.\"\"\"\n\n  connector = http_connector.HttpConnector(\n      api_endpoint=api_endpoint,\n      page_size=page_size,\n      auth=auth,\n      proxies=proxies,\n      verify=verify,\n      cert=cert,\n      trust_env=trust_env)\n\n  return GrrApi(connector=connector)", "response": "Initializes an GRR API object with a HTTP connector."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef NetshStaticIp(interface,\n                  ip=u'127.0.0.9',\n                  subnet=u'255.255.255.255',\n                  gw=u'127.0.0.1'):\n  \"\"\"Changes interface to a staticly set IP.\n\n  Sets IP configs to local if no paramaters passed.\n\n  Args:\n    interface: Name of the interface.\n    ip: IP address.\n    subnet: Subnet mask.\n    gw: IP address of the default gateway.\n\n  Returns:\n    A tuple of stdout, stderr, exit_status.\n  \"\"\"\n  args = [\n      '/c', 'netsh', 'interface', 'ip', 'set', 'address', interface, 'static',\n      ip, subnet, gw, '1'\n  ]\n  # pylint: disable=undefined-variable\n  res = client_utils_common.Execute(\n      'cmd', args, time_limit=-1, bypass_whitelist=True)\n  return res", "response": "Changes the interface to a staticly set IP."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ndisables an interface. Only works on Vista and 7.", "response": "def DisableInterfaces(interface):\n  \"\"\"Tries to disable an interface.  Only works on Vista and 7.\n\n  Args:\n    interface: Name of the interface to disable.\n\n  Returns:\n    res which is a tuple of (stdout, stderr, exit_status, time_taken).\n  \"\"\"\n  set_tested_versions = ['vista', '2008']\n  set_args = ['/c', 'netsh', 'set', 'interface', interface, 'DISABLED']\n  host_version = platform.platform().lower()\n  for version in set_tested_versions:\n    if host_version.find(version) != -1:\n      # pylint: disable=undefined-variable\n      res = client_utils_common.Execute(\n          'cmd', set_args, time_limit=-1, bypass_whitelist=True)\n      return res\n  return ('', 'Command not available for this version.', 99, '')"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef GetEnabledInterfaces():\n  interfaces = []\n  show_args = ['/c', 'netsh', 'show', 'interface']\n  # pylint: disable=undefined-variable\n  res = client_utils_common.Execute(\n      'cmd', show_args, time_limit=-1, bypass_whitelist=True)\n  pattern = re.compile(r'\\s*')\n  for line in res[0].split('\\r\\n'):  # res[0] is stdout.\n    interface_info = pattern.split(line)\n    if 'Enabled' in interface_info:\n      interfaces.extend(interface_info[-1:])\n  return interfaces", "response": "Gives a list of enabled interfaces. Should work on all windows versions."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef MsgUser(msg):\n  msg_tested_versions = ['xp', 'vista', '2008', '2003']\n  msg_args = ['/c', '%SystemRoot%\\\\System32\\\\msg.exe', '*', '/TIME:0']\n  host_version = platform.platform().lower()\n  if not msg:\n    return ('Command not ran.', 'Empty message.', -1)\n  else:\n    msg_args.extend([msg])\n  for version in msg_tested_versions:\n    if host_version.find(version) != -1:\n      # pylint: disable=undefined-variable\n      res = client_utils_common.Execute(\n          'cmd', msg_args, time_limit=-1, bypass_whitelist=True)\n      return res\n  return ('', 'Command not available for this version.', -1)", "response": "Sends a message to a user."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\napplies build. yaml settings from the template.", "response": "def GetConfigFromTemplate(self, template_path):\n    \"\"\"Apply build.yaml settings from the template.\"\"\"\n    with zipfile.ZipFile(template_path) as template_zip:\n      build_yaml = None\n      for name in template_zip.namelist():\n        if name.endswith(\"build.yaml\"):\n          build_yaml = name\n          break\n      if not build_yaml:\n        raise RuntimeError(\"Couldn't find build.yaml in %s\" % template_path)\n      with template_zip.open(build_yaml) as buildfile:\n        repack_config = config.CONFIG.CopyConfig()\n        utf8_buildfile = io.TextIOWrapper(buildfile, encoding=\"utf-8\")\n        parser = config_lib.YamlParser(fd=utf8_buildfile)\n        config_data = parser.RawData()\n        self.Validate(config_data, template_path)\n        repack_config.MergeData(config_data)\n    return repack_config"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning the appropriate client deployer based on the selected flags.", "response": "def GetRepacker(self, context, signer=None):\n    \"\"\"Get the appropriate client deployer based on the selected flags.\"\"\"\n    if \"Target:Darwin\" in context:\n      deployer_class = build.DarwinClientRepacker\n    elif \"Target:Windows\" in context:\n      deployer_class = build.WindowsClientRepacker\n    elif \"Target:LinuxDeb\" in context:\n      deployer_class = build.LinuxClientRepacker\n    elif \"Target:LinuxRpm\" in context:\n      deployer_class = build.CentosClientRepacker\n    else:\n      raise RuntimeError(\"Bad build context: %s\" % context)\n\n    return deployer_class(context=context, signer=signer)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef RepackTemplate(self,\n                     template_path,\n                     output_dir,\n                     upload=False,\n                     token=None,\n                     sign=False,\n                     context=None,\n                     signed_template=False):\n    \"\"\"Repack binaries based on the configuration.\n\n    We repack all templates in the templates directory. We expect to find only\n    functioning templates, all other files should be removed. Each template\n    contains a build.yaml that specifies how it was built and how it should be\n    repacked.\n\n    Args:\n      template_path: template path string\n      output_dir: Output files will be put in this directory.\n      upload: If specified we also upload the repacked binary into the\n      token: Token to use when uploading to the datastore.\n      sign: If true, we want to digitally sign the installer.\n      context: Array of context strings\n      signed_template: If true, the libraries in the template are already\n        signed. This is only used for windows when repacking the template\n        multiple times.\n\n    Returns:\n      A list of output installers generated.\n    \"\"\"\n    orig_config = config.CONFIG\n    repack_config = RepackConfig()\n    print(\"Repacking template: %s\" % template_path)\n    config.CONFIG = repack_config.GetConfigFromTemplate(template_path)\n\n    result_path = None\n    try:\n      repack_context = config.CONFIG[\"Template.build_context\"]\n      if context:\n        repack_context.extend(context)\n\n      output_path = os.path.join(\n          output_dir,\n          config.CONFIG.Get(\n              \"ClientRepacker.output_filename\", context=repack_context))\n\n      print(\"Using context: %s and labels: %s\" %\n            (repack_context,\n             config.CONFIG.Get(\"Client.labels\", context=repack_context)))\n      try:\n        signer = None\n        if sign:\n          signer = self.GetSigner(repack_context)\n        builder_obj = self.GetRepacker(context=repack_context, signer=signer)\n        builder_obj.signed_template = signed_template\n        result_path = builder_obj.MakeDeployableBinary(template_path,\n                                                       output_path)\n      except Exception:  # pylint: disable=broad-except\n        logging.exception(\"Repacking template %s failed:\", template_path)\n\n      if result_path:\n        print(\"Repacked into %s\" % result_path)\n        if upload:\n          # We delay import here so we don't have to import the entire server\n          # codebase and do full server init if we're just building and\n          # repacking clients. This codepath is used by config_updater\n          # initialize\n          # pylint: disable=g-import-not-at-top\n          from grr_response_server import maintenance_utils\n          # pylint: enable=g-import-not-at-top\n          client_platform = config.CONFIG.Get(\n              \"Client.platform\", context=repack_context)\n          repack_basename = config.CONFIG.Get(\n              \"ClientRepacker.output_basename\", context=repack_context)\n          repack_extension = config.CONFIG.Get(\n              \"ClientBuilder.output_extension\", context=repack_context)\n          repack_filename = repack_basename + repack_extension\n          # TODO(user): Use signed_binary_utils.GetAFF4ExecutablesRoot()\n          # here instead of hardcoding the root once repacking logic has been\n          # moved to grr-response-server.\n          binary_urn = rdfvalue.RDFURN(\"aff4:/config/executables\").Add(\n              client_platform).Add(\"installers\").Add(repack_filename)\n          maintenance_utils.UploadSignedConfigBlob(\n              open(result_path, \"rb\").read(100 * 1024 * 1024),\n              binary_urn,\n              client_context=repack_context,\n              token=token)\n      else:\n        print(\"Failed to repack %s.\" % template_path)\n    finally:\n      config.CONFIG = orig_config\n\n    return result_path", "response": "Repacks a template into a directory."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nbuild the list of CloudMetadataRequest objects from collection maps.", "response": "def _MakeArgs(amazon_collection_map, google_collection_map):\n  \"\"\"Build metadata requests list from collection maps.\"\"\"\n  request_list = []\n  for url, label in iteritems(amazon_collection_map):\n    request_list.append(\n        CloudMetadataRequest(\n            bios_version_regex=AMAZON_BIOS_REGEX,\n            service_name_regex=AMAZON_SERVICE_REGEX,\n            instance_type=\"AMAZON\",\n            timeout=1.0,\n            url=url,\n            label=label))\n  for url, label in iteritems(google_collection_map):\n    request_list.append(\n        CloudMetadataRequest(\n            bios_version_regex=GOOGLE_BIOS_REGEX,\n            service_name_regex=GOOGLE_SERVICE_REGEX,\n            headers={\"Metadata-Flavor\": \"Google\"},\n            instance_type=\"GOOGLE\",\n            timeout=1.0,\n            url=url,\n            label=label))\n  return request_list"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nmake the google unique ID of zone and project.", "response": "def MakeGoogleUniqueID(cloud_instance):\n  \"\"\"Make the google unique ID of zone/project/id.\"\"\"\n  if not (cloud_instance.zone and cloud_instance.project_id and\n          cloud_instance.instance_id):\n    raise ValueError(\"Bad zone/project_id/id: '%s/%s/%s'\" %\n                     (cloud_instance.zone, cloud_instance.project_id,\n                      cloud_instance.instance_id))\n  return \"/\".join([\n      cloud_instance.zone.split(\"/\")[-1], cloud_instance.project_id,\n      cloud_instance.instance_id\n  ])"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nbuilds the standard set of cloud metadata to collect during interrogate.", "response": "def BuildCloudMetadataRequests():\n  \"\"\"Build the standard set of cloud metadata to collect during interrogate.\"\"\"\n  amazon_collection_map = {\n      \"/\".join((AMAZON_URL_BASE, \"instance-id\")): \"instance_id\",\n      \"/\".join((AMAZON_URL_BASE, \"ami-id\")): \"ami_id\",\n      \"/\".join((AMAZON_URL_BASE, \"hostname\")): \"hostname\",\n      \"/\".join((AMAZON_URL_BASE, \"public-hostname\")): \"public_hostname\",\n      \"/\".join((AMAZON_URL_BASE, \"instance-type\")): \"instance_type\",\n  }\n  google_collection_map = {\n      \"/\".join((GOOGLE_URL_BASE, \"instance/id\")): \"instance_id\",\n      \"/\".join((GOOGLE_URL_BASE, \"instance/zone\")): \"zone\",\n      \"/\".join((GOOGLE_URL_BASE, \"project/project-id\")): \"project_id\",\n      \"/\".join((GOOGLE_URL_BASE, \"instance/hostname\")): \"hostname\",\n      \"/\".join((GOOGLE_URL_BASE, \"instance/machine-type\")): \"machine_type\",\n  }\n\n  return CloudMetadataRequests(requests=_MakeArgs(amazon_collection_map,\n                                                  google_collection_map))"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nconvert CloudMetadataResponses to CloudInstance proto.", "response": "def ConvertCloudMetadataResponsesToCloudInstance(metadata_responses):\n  \"\"\"Convert CloudMetadataResponses to CloudInstance proto.\n\n  Ideally we'd just get the client to fill out a CloudInstance proto, but we\n  need to keep the flexibility of collecting new metadata and creating new\n  fields without a client push. So instead we bring back essentially a dict of\n  results and fill the proto on the server side.\n\n  Args:\n    metadata_responses: CloudMetadataResponses object from the client.\n  Returns:\n    CloudInstance object\n  Raises:\n    ValueError: if client passes bad or unset cloud type.\n  \"\"\"\n  if metadata_responses.instance_type == \"GOOGLE\":\n    cloud_instance = GoogleCloudInstance()\n    result = CloudInstance(cloud_type=\"GOOGLE\", google=cloud_instance)\n  elif metadata_responses.instance_type == \"AMAZON\":\n    cloud_instance = AmazonCloudInstance()\n    result = CloudInstance(cloud_type=\"AMAZON\", amazon=cloud_instance)\n  else:\n    raise ValueError(\n        \"Unknown cloud instance type: %s\" % metadata_responses.instance_type)\n\n  for cloud_metadata in metadata_responses.responses:\n    setattr(cloud_instance, cloud_metadata.label, cloud_metadata.text)\n\n  if result.cloud_type == \"GOOGLE\":\n    cloud_instance.unique_id = MakeGoogleUniqueID(cloud_instance)\n\n  return result"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef RegisterValue(self, value):\n    if self.bins:\n      for b in self.bins:\n        if b.range_max_value > value:\n          b.num += 1\n          return\n\n      self.bins[-1].num += 1", "response": "Registers a given value into an appropriate bin."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nupdates stats with info about resources consumed by a single client.", "response": "def RegisterResources(self, client_resources):\n    \"\"\"Update stats with info about resources consumed by a single client.\"\"\"\n    self.user_cpu_stats.RegisterValue(client_resources.cpu_usage.user_cpu_time)\n    self.system_cpu_stats.RegisterValue(\n        client_resources.cpu_usage.system_cpu_time)\n    self.network_bytes_sent_stats.RegisterValue(\n        client_resources.network_bytes_sent)\n\n    self.worst_performers.Append(client_resources)\n    new_worst_performers = sorted(\n        self.worst_performers,\n        key=lambda s: s.cpu_usage.user_cpu_time + s.cpu_usage.system_cpu_time,\n        reverse=True)[:self.NUM_WORST_PERFORMERS]\n    self.worst_performers = new_worst_performers"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef payload(self):\n    try:\n      rdf_cls = self.classes.get(self.name)\n      if rdf_cls:\n        value = rdf_cls.FromSerializedString(self.data)\n        value.age = self.embedded_age\n\n        return value\n    except TypeError:\n      return None", "response": "Extracts and returns the serialized object."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef SetValue(self, value, raise_on_error=True):\n    type_mappings = [(Text, \"string\"), (bytes, \"data\"), (bool, \"boolean\"),\n                     (int, \"integer\"), (long, \"integer\"), (dict, \"dict\"),\n                     (float, \"float\")]\n\n    if value is None:\n      self.none = \"None\"\n\n    elif isinstance(value, rdfvalue.RDFValue):\n      self.rdf_value.data = value.SerializeToString()\n      self.rdf_value.age = int(value.age)\n      self.rdf_value.name = value.__class__.__name__\n\n    elif isinstance(value, (list, tuple)):\n      self.list.content.Extend([\n          DataBlob().SetValue(v, raise_on_error=raise_on_error) for v in value\n      ])\n\n    elif isinstance(value, set):\n      self.set.content.Extend([\n          DataBlob().SetValue(v, raise_on_error=raise_on_error) for v in value\n      ])\n\n    elif isinstance(value, dict):\n      self.dict.FromDict(value, raise_on_error=raise_on_error)\n\n    else:\n      for type_mapping, member in type_mappings:\n        if isinstance(value, type_mapping):\n          setattr(self, member, value)\n\n          return self\n\n      message = \"Unsupported type for ProtoDict: %s\" % type(value)\n      if raise_on_error:\n        raise TypeError(message)\n\n      setattr(self, \"string\", message)\n\n    return self", "response": "Receives a value and fills it into a DataBlob."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef GetValue(self, ignore_error=True):\n    if self.HasField(\"none\"):\n      return None\n\n    field_names = [\n        \"integer\", \"string\", \"data\", \"boolean\", \"list\", \"dict\", \"rdf_value\",\n        \"float\", \"set\"\n    ]\n\n    values = [getattr(self, x) for x in field_names if self.HasField(x)]\n\n    if len(values) != 1:\n      return None\n\n    if self.HasField(\"boolean\"):\n      return bool(values[0])\n\n    # Unpack RDFValues.\n    if self.HasField(\"rdf_value\"):\n      try:\n        rdf_class = rdfvalue.RDFValue.classes[self.rdf_value.name]\n        return rdf_class.FromSerializedString(\n            self.rdf_value.data, age=self.rdf_value.age)\n      except (ValueError, KeyError) as e:\n        if ignore_error:\n          return e\n\n        raise\n\n    elif self.HasField(\"list\"):\n      return [x.GetValue() for x in self.list.content]\n\n    elif self.HasField(\"set\"):\n      return set([x.GetValue() for x in self.set.content])\n\n    else:\n      return values[0]", "response": "Extracts and returns a single value from a DataBlob."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef Append(self, value=None, **kwarg):\n    if self.rdf_type is not None:\n      if (isinstance(value, rdfvalue.RDFValue) and\n          value.__class__ != self.rdf_type):\n        raise ValueError(\"Can only accept %s\" % self.rdf_type)\n\n      try:\n        # Try to coerce the value.\n        value = self.rdf_type(value, **kwarg)  # pylint: disable=not-callable\n      except (TypeError, ValueError):\n        raise ValueError(\"Unable to initialize %s from type %s\" %\n                         (self.__class__.__name__, type(value)))\n\n    self.content.Append(DataBlob().SetValue(value))", "response": "Append a new element to the array."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef FromMany(cls, samples):\n    if not samples:\n      raise ValueError(\"Empty `samples` argument\")\n\n    # It only makes sense to average the CPU percentage. For all other values\n    # we simply take the biggest of them.\n    cpu_percent = sum(sample.cpu_percent for sample in samples) / len(samples)\n\n    return CpuSample(\n        timestamp=max(sample.timestamp for sample in samples),\n        cpu_percent=cpu_percent,\n        user_cpu_time=max(sample.user_cpu_time for sample in samples),\n        system_cpu_time=max(sample.system_cpu_time for sample in samples))", "response": "Constructs a single sample that best represents a list of samples."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef FromMany(cls, samples):\n    if not samples:\n      raise ValueError(\"Empty `samples` argument\")\n\n    return IOSample(\n        timestamp=max(sample.timestamp for sample in samples),\n        read_bytes=max(sample.read_bytes for sample in samples),\n        write_bytes=max(sample.write_bytes for sample in samples))", "response": "Constructs a single sample that best represents a list of samples."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nconstructs a copy of given stats but downsampled to given interval.", "response": "def Downsampled(cls, stats, interval=None):\n    \"\"\"Constructs a copy of given stats but downsampled to given interval.\n\n    Args:\n      stats: A `ClientStats` instance.\n      interval: A downsampling interval.\n\n    Returns:\n      A downsampled `ClientStats` instance.\n    \"\"\"\n    interval = interval or cls.DEFAULT_SAMPLING_INTERVAL\n\n    result = cls(stats)\n    result.cpu_samples = cls._Downsample(\n        kind=CpuSample, samples=stats.cpu_samples, interval=interval)\n    result.io_samples = cls._Downsample(\n        kind=IOSample, samples=stats.io_samples, interval=interval)\n    return result"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef GetUserInfo(knowledge_base, user):\n  # TODO: This docstring cannot be a raw literal because there are\n  # issues with raw unicode literals on Python 2. Once support for Python 2 is\n  # dropped, it can be made raw again.\n  # pylint: disable=g-docstring-has-escape\n  \"\"\"Get a User protobuf for a specific user.\n\n  Args:\n    knowledge_base: An rdf_client.KnowledgeBase object.\n    user: Username as string. May contain domain like DOMAIN\\\\user.\n\n  Returns:\n    A User rdfvalue or None\n  \"\"\"\n  # pylint: enable=g-docstring-has-escape\n  if \"\\\\\" in user:\n    domain, user = user.split(\"\\\\\", 1)\n    users = [\n        u for u in knowledge_base.users\n        if u.username == user and u.userdomain == domain\n    ]\n  else:\n    users = [u for u in knowledge_base.users if u.username == user]\n\n  if not users:\n    return\n  else:\n    return users[0]", "response": "Get a User protobuf for a specific user."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef UpdateVFSFileAndWait(client_id,\n                         vfs_file_urn,\n                         token=None,\n                         timeout=DEFAULT_TIMEOUT):\n  \"\"\"Waits for a file to be updated on the client.\n\n  Calls the UpdateVFSFile flow on a urn and waits for both it and the\n  ListDirectory flow it calls to finish.\n\n  Note that this is needed because any flows UpdateVFSFile calls via\n  VFS Update methods will not become child flows of UpdateVFSFile,\n  and therefore waiting for UpdateVFSFile to complete is not enough.\n\n  Args:\n    client_id: Which client to run the flow on.\n    vfs_file_urn: Path to VFSFile to update.\n    token: The datastore access token.\n    timeout: How long to wait for a flow to finish, maximum.\n  \"\"\"\n  # Wait for the UpdateVFSFile flow.\n  update_flow_urn = StartFlowAndWait(\n      client_id,\n      token=token,\n      timeout=timeout,\n      flow_name=aff4_grr.UpdateVFSFile.__name__,\n      vfs_file_urn=vfs_file_urn)\n\n  update_flow_obj = aff4.FACTORY.Open(\n      update_flow_urn, token=token, aff4_type=flow.GRRFlow)\n\n  # Get the child flow so we can wait for it too.\n  sub_flow_urn = update_flow_obj.state.get_file_flow_urn\n\n  # If there was no subflow, no need to wait for it.\n  if not sub_flow_urn:\n    return\n\n  WaitForFlow(sub_flow_urn, token=token, timeout=timeout)", "response": "Calls the UpdateVFSFile flow and waits for it to finish."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef WaitForFlow(flow_urn,\n                token=None,\n                timeout=DEFAULT_TIMEOUT,\n                max_sleep_time=1,\n                min_sleep_time=0.2,\n                dampening_multiplier=0.9):\n  \"\"\"Waits for a flow to finish, polling while we wait.\n\n  Args:\n    flow_urn: The urn of the flow to wait for.\n    token: The datastore access token.\n    timeout: How long to wait before giving up, usually because the client has\n      gone away.\n    max_sleep_time: The initial and longest time to wait in between polls.\n    min_sleep_time: The final and shortest time to wait in between polls.\n    dampening_multiplier: The current sleep time is multiplied by this number on\n      each iteration. Controls how fast the polling reaches its minimum sleep\n      time. You probably want this to be less than 1, unless you want to wait an\n      increasing amount of time in between flows.\n\n  Raises:\n    IOError: If we time out while waiting for the client.\n  \"\"\"\n\n  start_time = time.time()\n  sleep_time = max_sleep_time\n  while True:\n    # Reopen the AFF4Object to check if its status has changed, and also make\n    # sure it's a flow.\n    with aff4.FACTORY.Open(\n        flow_urn, token=token, aff4_type=flow.GRRFlow) as flow_obj:\n\n      # Stop if the flow is done or has timed out.\n      if time.time() - start_time > timeout:\n        logging.warning(\"Timed out after waiting %ss for %s!\", timeout,\n                        flow_obj)\n        raise IOError(\"Timed out trying to access client! Is it connected?\")\n      if not flow_obj.GetRunner().IsRunning():\n        break\n    # Decrease the time we sleep each iteration.\n    sleep_time = max(sleep_time * dampening_multiplier, min_sleep_time)\n    time.sleep(sleep_time)\n    logging.debug(\"Waiting for %s, sleeping for %.3fs\", flow_obj, sleep_time)", "response": "Waits for a flow to finish and returns it."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nrun a flow and waits for it to finish.", "response": "def StartFlowAndWait(client_id,\n                     token=None,\n                     timeout=DEFAULT_TIMEOUT,\n                     **flow_args):\n  \"\"\"Runs a flow and waits for it to finish.\n\n  Args:\n    client_id: The client id of the client to run on.\n    token: The datastore access token.\n    timeout: How long to wait for a flow to complete, maximum.\n    **flow_args: Pass through to flow.\n\n  Returns:\n    The urn of the flow that was run.\n  \"\"\"\n  flow_urn = flow.StartAFF4Flow(\n      client_id=client_id, token=token, sync=True, **flow_args)\n\n  WaitForFlow(flow_urn, token=token, timeout=timeout)\n\n  return flow_urn"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef InterpolatePath(path, knowledge_base, users=None, path_args=None, depth=0):\n\n  sys_formatters = {\n      # TODO(user): Collect this during discovery from the registry.\n      # HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\\n      # Value: SystemRoot\n      \"systemroot\": \"c:\\\\Windows\"\n  }\n\n  # Override any system formatters with path_args.\n  if path_args:\n    sys_formatters.update(path_args)\n\n  if users:\n    results = []\n    for user in users:\n      # Extract and interpolate user specific formatters.\n      user = GetUserInfo(knowledge_base, user)\n      if user:\n        formatters = dict((x.name, y) for x, y in user.ListSetFields())\n        formatters.update(sys_formatters)\n        try:\n          results.append(path.format(**formatters))\n        except KeyError:\n          pass  # We may be missing values for some users.\n    return results\n  else:\n    try:\n      path = path.format(**sys_formatters)\n    except KeyError:\n      logging.warning(\"Failed path interpolation on %s\", path)\n      return \"\"\n    if \"{\" in path and depth < 10:\n      path = InterpolatePath(\n          path,\n          knowledge_base=knowledge_base,\n          users=users,\n          path_args=path_args,\n          depth=depth + 1)\n    return path", "response": "Interpolate a path on a client and return a single string."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nbuilding a list of filter callables.", "response": "def BuildChecks(self, request):\n    \"\"\"Parses request and returns a list of filter callables.\n\n    Each callable will be called with the StatEntry and returns True if the\n    entry should be suppressed.\n\n    Args:\n      request: A FindSpec that describes the search.\n\n    Returns:\n      a list of callables which return True if the file is to be suppressed.\n    \"\"\"\n    result = []\n    if request.HasField(\"start_time\") or request.HasField(\"end_time\"):\n\n      def FilterTimestamp(file_stat, request=request):\n        return file_stat.HasField(\"st_mtime\") and (\n            file_stat.st_mtime < request.start_time or\n            file_stat.st_mtime > request.end_time)\n\n      result.append(FilterTimestamp)\n\n    if request.HasField(\"min_file_size\") or request.HasField(\"max_file_size\"):\n\n      def FilterSize(file_stat, request=request):\n        return file_stat.HasField(\"st_size\") and (\n            file_stat.st_size < request.min_file_size or\n            file_stat.st_size > request.max_file_size)\n\n      result.append(FilterSize)\n\n    if request.HasField(\"perm_mode\"):\n\n      def FilterPerms(file_stat, request=request):\n        return (file_stat.st_mode & request.perm_mask) != request.perm_mode\n\n      result.append(FilterPerms)\n\n    if request.HasField(\"uid\"):\n\n      def FilterUID(file_stat, request=request):\n        return file_stat.st_uid != request.uid\n\n      result.append(FilterUID)\n\n    if request.HasField(\"gid\"):\n\n      def FilterGID(file_stat, request=request):\n        return file_stat.st_gid != request.gid\n\n      result.append(FilterGID)\n\n    if request.HasField(\"path_regex\"):\n      regex = request.path_regex\n\n      def FilterPath(file_stat, regex=regex):\n        \"\"\"Suppress any filename not matching the regular expression.\"\"\"\n        return not regex.Search(file_stat.pathspec.Basename())\n\n      result.append(FilterPath)\n\n    if request.HasField(\"data_regex\"):\n\n      def FilterData(file_stat, **_):\n        \"\"\"Suppress files that do not match the content.\"\"\"\n        return not self.TestFileContent(file_stat)\n\n      result.append(FilterData)\n\n    return result"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nrun the Find action.", "response": "def Run(self, request):\n    \"\"\"Runs the Find action.\"\"\"\n\n    self.request = request\n    filters = self.BuildChecks(request)\n\n    files_checked = 0\n    for f in self.ListDirectory(request.pathspec):\n      self.Progress()\n\n      # Ignore this file if any of the checks fail.\n      if not any((check(f) for check in filters)):\n        self.SendReply(f)\n\n      files_checked += 1\n      if files_checked >= self.MAX_FILES_TO_CHECK:\n        return"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nsearches the data for a hit in the given regex.", "response": "def FindRegex(self, regex, data):\n    \"\"\"Search the data for a hit.\"\"\"\n    for match in re.finditer(regex, data, flags=re.I | re.S | re.M):\n      yield (match.start(), match.end())"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nsearches the data for a hit.", "response": "def FindLiteral(self, pattern, data):\n    \"\"\"Search the data for a hit.\"\"\"\n    pattern = utils.Xor(pattern, self.xor_in_key)\n\n    offset = 0\n    while 1:\n      # We assume here that data.find does not make a copy of pattern.\n      offset = data.find(pattern, offset)\n\n      if offset < 0:\n        break\n\n      yield (offset, offset + len(pattern))\n\n      offset += 1"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef Run(self, args):\n    fd = vfs.VFSOpen(args.target, progress_callback=self.Progress)\n    fd.Seek(args.start_offset)\n    base_offset = args.start_offset\n\n    self.xor_in_key = args.xor_in_key\n    self.xor_out_key = args.xor_out_key\n\n    if args.regex:\n      find_func = functools.partial(self.FindRegex, args.regex.AsBytes())\n    elif args.literal:\n      find_func = functools.partial(self.FindLiteral, args.literal.AsBytes())\n    else:\n      raise RuntimeError(\"Grep needs a regex or a literal.\")\n\n    preamble_size = 0\n    postscript_size = 0\n    hits = 0\n    data = b\"\"\n    while fd.Tell() < args.start_offset + args.length:\n\n      # Base size to read is at most the buffer size.\n      to_read = min(args.length, self.BUFF_SIZE,\n                    args.start_offset + args.length - fd.Tell())\n      # Read some more data for the snippet.\n      to_read += self.ENVELOPE_SIZE - postscript_size\n      read_data = fd.Read(to_read)\n\n      data = data[-postscript_size - self.ENVELOPE_SIZE:] + read_data\n\n      postscript_size = max(0, self.ENVELOPE_SIZE - (to_read - len(read_data)))\n      data_size = len(data) - preamble_size - postscript_size\n\n      if data_size == 0 and postscript_size == 0:\n        break\n\n      for (start, end) in find_func(data):\n        # Ignore hits in the preamble.\n        if end <= preamble_size:\n          continue\n\n        # Ignore hits in the postscript.\n        if end > preamble_size + data_size:\n          continue\n\n        # Offset of file in the end after length.\n        if end + base_offset - preamble_size > args.start_offset + args.length:\n          break\n\n        data_start = max(0, start - args.bytes_before)\n        data_end = min(len(data), end + args.bytes_after)\n        out_data = utils.Xor(data[data_start:data_end], self.xor_out_key)\n\n        hits += 1\n        self.SendReply(\n            rdf_client.BufferReference(\n                offset=base_offset + start - preamble_size,\n                data=out_data,\n                length=len(out_data),\n                pathspec=fd.pathspec))\n\n        if args.mode == rdf_client_fs.GrepSpec.Mode.FIRST_HIT:\n          return\n\n        if hits >= self.HIT_LIMIT:\n          msg = utils.Xor(\n              b\"This Grep has reached the maximum number of hits\"\n              b\" (%d).\" % self.HIT_LIMIT, self.xor_out_key)\n          self.SendReply(\n              rdf_client.BufferReference(offset=0, data=msg, length=len(msg)))\n          return\n\n      self.Progress()\n\n      base_offset += data_size\n\n      # Allow for overlap with previous matches.\n      preamble_size = min(len(data), self.ENVELOPE_SIZE)", "response": "This function searches the file for a pattern and returns the data."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nstops the hunt if number of crashes exceeds the limit.", "response": "def StopHuntIfCrashLimitExceeded(hunt_id):\n  \"\"\"Stops the hunt if number of crashes exceeds the limit.\"\"\"\n  hunt_obj = data_store.REL_DB.ReadHuntObject(hunt_id)\n\n  # Do nothing if the hunt is already stopped.\n  if hunt_obj.hunt_state == rdf_hunt_objects.Hunt.HuntState.STOPPED:\n    return hunt_obj\n\n  if hunt_obj.crash_limit:\n    hunt_counters = data_store.REL_DB.ReadHuntCounters(hunt_id)\n    if hunt_counters.num_crashed_clients >= hunt_obj.crash_limit:\n      # Remove our rules from the forman and cancel all the started flows.\n      # Hunt will be hard-stopped and it will be impossible to restart it.\n      reason = (\"Hunt %s reached the crashes limit of %d \"\n                \"and was stopped.\") % (hunt_obj.hunt_id, hunt_obj.crash_limit)\n      StopHunt(hunt_obj.hunt_id, reason=reason)\n\n  return hunt_obj"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nstopping the hunt if CPU or network limits are exceeded.", "response": "def StopHuntIfCPUOrNetworkLimitsExceeded(hunt_id):\n  \"\"\"Stops the hunt if average limites are exceeded.\"\"\"\n  hunt_obj = data_store.REL_DB.ReadHuntObject(hunt_id)\n\n  # Do nothing if the hunt is already stopped.\n  if hunt_obj.hunt_state == rdf_hunt_objects.Hunt.HuntState.STOPPED:\n    return hunt_obj\n\n  hunt_counters = data_store.REL_DB.ReadHuntCounters(hunt_id)\n\n  # Check global hunt network bytes limit first.\n  if (hunt_obj.total_network_bytes_limit and\n      hunt_counters.total_network_bytes_sent >\n      hunt_obj.total_network_bytes_limit):\n    reason = (\"Hunt %s reached the total network bytes sent limit of %d and \"\n              \"was stopped.\") % (hunt_obj.hunt_id,\n                                 hunt_obj.total_network_bytes_limit)\n    return StopHunt(hunt_obj.hunt_id, reason=reason)\n\n  # Check that we have enough clients to apply average limits.\n  if hunt_counters.num_clients < MIN_CLIENTS_FOR_AVERAGE_THRESHOLDS:\n    return hunt_obj\n\n  # Check average per-client results count limit.\n  if hunt_obj.avg_results_per_client_limit:\n    avg_results_per_client = (\n        hunt_counters.num_results / hunt_counters.num_clients)\n    if avg_results_per_client > hunt_obj.avg_results_per_client_limit:\n      # Stop the hunt since we get too many results per client.\n      reason = (\"Hunt %s reached the average results per client \"\n                \"limit of %d and was stopped.\") % (\n                    hunt_obj.hunt_id, hunt_obj.avg_results_per_client_limit)\n      return StopHunt(hunt_obj.hunt_id, reason=reason)\n\n  # Check average per-client CPU seconds limit.\n  if hunt_obj.avg_cpu_seconds_per_client_limit:\n    avg_cpu_seconds_per_client = (\n        hunt_counters.total_cpu_seconds / hunt_counters.num_clients)\n    if avg_cpu_seconds_per_client > hunt_obj.avg_cpu_seconds_per_client_limit:\n      # Stop the hunt since we use too many CPUs per client.\n      reason = (\"Hunt %s reached the average CPU seconds per client \"\n                \"limit of %d and was stopped.\") % (\n                    hunt_obj.hunt_id, hunt_obj.avg_cpu_seconds_per_client_limit)\n      return StopHunt(hunt_obj.hunt_id, reason=reason)\n\n  # Check average per-client network bytes limit.\n  if hunt_obj.avg_network_bytes_per_client_limit:\n    avg_network_bytes_per_client = (\n        hunt_counters.total_network_bytes_sent / hunt_counters.num_clients)\n    if (avg_network_bytes_per_client >\n        hunt_obj.avg_network_bytes_per_client_limit):\n      # Stop the hunt since we use too many network bytes sent\n      # per client.\n      reason = (\"Hunt %s reached the average network bytes per client \"\n                \"limit of %d and was stopped.\") % (\n                    hunt_obj.hunt_id,\n                    hunt_obj.avg_network_bytes_per_client_limit)\n      return StopHunt(hunt_obj.hunt_id, reason=reason)\n\n  return hunt_obj"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef CompleteHuntIfExpirationTimeReached(hunt_obj):\n  # TODO(hanuszczak): This should not set the hunt state to `COMPLETED` but we\n  # should have a sparate `EXPIRED` state instead and set that.\n  if (hunt_obj.hunt_state not in [\n      rdf_hunt_objects.Hunt.HuntState.STOPPED,\n      rdf_hunt_objects.Hunt.HuntState.COMPLETED\n  ] and hunt_obj.expired):\n    StopHunt(hunt_obj.hunt_id, reason=\"Hunt completed.\")\n\n    data_store.REL_DB.UpdateHuntObject(\n        hunt_obj.hunt_id, hunt_state=hunt_obj.HuntState.COMPLETED)\n    return data_store.REL_DB.ReadHuntObject(hunt_obj.hunt_id)\n\n  return hunt_obj", "response": "Marks the hunt as complete if it s past its expiry time."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ncreates a hunt using a given hunt object.", "response": "def CreateHunt(hunt_obj):\n  \"\"\"Creates a hunt using a given hunt object.\"\"\"\n  data_store.REL_DB.WriteHuntObject(hunt_obj)\n\n  if hunt_obj.HasField(\"output_plugins\"):\n    output_plugins_states = flow.GetOutputPluginStates(\n        hunt_obj.output_plugins,\n        source=\"hunts/%s\" % hunt_obj.hunt_id,\n        token=access_control.ACLToken(username=hunt_obj.creator))\n    data_store.REL_DB.WriteHuntOutputPluginsStates(hunt_obj.hunt_id,\n                                                   output_plugins_states)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef CreateAndStartHunt(flow_name, flow_args, creator, **kwargs):\n\n  # This interface takes a time when the hunt expires. However, the legacy hunt\n  # starting interface took an rdfvalue.Duration object which was then added to\n  # the current time to get the expiry. This check exists to make sure we don't\n  # confuse the two.\n  if \"duration\" in kwargs:\n    precondition.AssertType(kwargs[\"duration\"], rdfvalue.Duration)\n\n  hunt_args = rdf_hunt_objects.HuntArguments(\n      hunt_type=rdf_hunt_objects.HuntArguments.HuntType.STANDARD,\n      standard=rdf_hunt_objects.HuntArgumentsStandard(\n          flow_name=flow_name, flow_args=flow_args))\n\n  hunt_obj = rdf_hunt_objects.Hunt(\n      creator=creator,\n      args=hunt_args,\n      create_time=rdfvalue.RDFDatetime.Now(),\n      **kwargs)\n\n  CreateHunt(hunt_obj)\n  StartHunt(hunt_obj.hunt_id)\n\n  return hunt_obj.hunt_id", "response": "Creates and starts a new hunt."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nschedule foreman rules for a generic hunt.", "response": "def _ScheduleGenericHunt(hunt_obj):\n  \"\"\"Adds foreman rules for a generic hunt.\"\"\"\n  # TODO: Migrate foreman conditions to use relation expiration\n  # durations instead of absolute timestamps.\n  foreman_condition = foreman_rules.ForemanCondition(\n      creation_time=rdfvalue.RDFDatetime.Now(),\n      expiration_time=hunt_obj.init_start_time + hunt_obj.duration,\n      description=\"Hunt %s %s\" % (hunt_obj.hunt_id, hunt_obj.args.hunt_type),\n      client_rule_set=hunt_obj.client_rule_set,\n      hunt_id=hunt_obj.hunt_id)\n\n  # Make sure the rule makes sense.\n  foreman_condition.Validate()\n\n  data_store.REL_DB.WriteForemanRule(foreman_condition)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _ScheduleVariableHunt(hunt_obj):\n  if hunt_obj.client_rate != 0:\n    raise VariableHuntCanNotHaveClientRateError(hunt_obj.hunt_id,\n                                                hunt_obj.client_rate)\n\n  seen_clients = set()\n  for flow_group in hunt_obj.args.variable.flow_groups:\n    for client_id in flow_group.client_ids:\n      if client_id in seen_clients:\n        raise CanStartAtMostOneFlowPerClientError(hunt_obj.hunt_id, client_id)\n      seen_clients.add(client_id)\n\n  now = rdfvalue.RDFDatetime.Now()\n  for flow_group in hunt_obj.args.variable.flow_groups:\n    flow_cls = registry.FlowRegistry.FlowClassByName(flow_group.flow_name)\n    flow_args = flow_group.flow_args if flow_group.HasField(\n        \"flow_args\") else None\n\n    for client_id in flow_group.client_ids:\n      flow.StartFlow(\n          client_id=client_id,\n          creator=hunt_obj.creator,\n          cpu_limit=hunt_obj.per_client_cpu_limit,\n          network_bytes_limit=hunt_obj.per_client_network_bytes_limit,\n          flow_cls=flow_cls,\n          flow_args=flow_args,\n          # Setting start_at explicitly ensures that flow.StartFlow won't\n          # process flow's Start state right away. Only the flow request\n          # will be scheduled.\n          start_at=now,\n          parent_hunt_id=hunt_obj.hunt_id)", "response": "Schedules flows for a variable hunt."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef StartHunt(hunt_id):\n\n  hunt_obj = data_store.REL_DB.ReadHuntObject(hunt_id)\n  num_hunt_clients = data_store.REL_DB.CountHuntFlows(hunt_id)\n\n  if hunt_obj.hunt_state != hunt_obj.HuntState.PAUSED:\n    raise OnlyPausedHuntCanBeStartedError(hunt_obj)\n\n  data_store.REL_DB.UpdateHuntObject(\n      hunt_id,\n      hunt_state=hunt_obj.HuntState.STARTED,\n      start_time=rdfvalue.RDFDatetime.Now(),\n      num_clients_at_start_time=num_hunt_clients,\n  )\n  hunt_obj = data_store.REL_DB.ReadHuntObject(hunt_id)\n\n  if hunt_obj.args.hunt_type == hunt_obj.args.HuntType.STANDARD:\n    _ScheduleGenericHunt(hunt_obj)\n  elif hunt_obj.args.hunt_type == hunt_obj.args.HuntType.VARIABLE:\n    _ScheduleVariableHunt(hunt_obj)\n  else:\n    raise UnknownHuntTypeError(\"Invalid hunt type for hunt %s: %r\" %\n                               (hunt_id, hunt_obj.args.hunt_type))\n\n  return hunt_obj", "response": "Starts a hunt with a given id."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\npauses a hunt with a given id.", "response": "def PauseHunt(hunt_id, reason=None):\n  \"\"\"Pauses a hunt with a given id.\"\"\"\n\n  hunt_obj = data_store.REL_DB.ReadHuntObject(hunt_id)\n  if hunt_obj.hunt_state != hunt_obj.HuntState.STARTED:\n    raise OnlyStartedHuntCanBePausedError(hunt_obj)\n\n  data_store.REL_DB.UpdateHuntObject(\n      hunt_id, hunt_state=hunt_obj.HuntState.PAUSED, hunt_state_comment=reason)\n  data_store.REL_DB.RemoveForemanRule(hunt_id=hunt_obj.hunt_id)\n\n  return data_store.REL_DB.ReadHuntObject(hunt_id)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef StopHunt(hunt_id, reason=None):\n\n  hunt_obj = data_store.REL_DB.ReadHuntObject(hunt_id)\n  if hunt_obj.hunt_state not in [\n      hunt_obj.HuntState.STARTED, hunt_obj.HuntState.PAUSED\n  ]:\n    raise OnlyStartedOrPausedHuntCanBeStoppedError(hunt_obj)\n\n  data_store.REL_DB.UpdateHuntObject(\n      hunt_id, hunt_state=hunt_obj.HuntState.STOPPED, hunt_state_comment=reason)\n  data_store.REL_DB.RemoveForemanRule(hunt_id=hunt_obj.hunt_id)\n\n  if (reason is not None and\n      hunt_obj.creator not in aff4_users.GRRUser.SYSTEM_USERS):\n    notification.Notify(\n        hunt_obj.creator, rdf_objects.UserNotification.Type.TYPE_HUNT_STOPPED,\n        reason,\n        rdf_objects.ObjectReference(\n            reference_type=rdf_objects.ObjectReference.Type.HUNT,\n            hunt=rdf_objects.HuntReference(hunt_id=hunt_obj.hunt_id)))\n\n  return data_store.REL_DB.ReadHuntObject(hunt_id)", "response": "Stops a hunt with a given id."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef UpdateHunt(hunt_id, client_limit=None, client_rate=None, duration=None):\n\n  hunt_obj = data_store.REL_DB.ReadHuntObject(hunt_id)\n  if hunt_obj.hunt_state != hunt_obj.HuntState.PAUSED:\n    raise OnlyPausedHuntCanBeModifiedError(hunt_obj)\n\n  data_store.REL_DB.UpdateHuntObject(\n      hunt_id,\n      client_limit=client_limit,\n      client_rate=client_rate,\n      duration=duration)\n  return data_store.REL_DB.ReadHuntObject(hunt_id)", "response": "Updates a hunt (it must be paused to be updated)."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef StartHuntFlowOnClient(client_id, hunt_id):\n\n  hunt_obj = data_store.REL_DB.ReadHuntObject(hunt_id)\n  hunt_obj = CompleteHuntIfExpirationTimeReached(hunt_obj)\n  # There may be a little race between foreman rules being removed and\n  # foreman scheduling a client on an (already) paused hunt. Making sure\n  # we don't lose clients in such a race by accepting clients for paused\n  # hunts.\n  if not rdf_hunt_objects.IsHuntSuitableForFlowProcessing(hunt_obj.hunt_state):\n    return\n\n  if hunt_obj.args.hunt_type == hunt_obj.args.HuntType.STANDARD:\n    hunt_args = hunt_obj.args.standard\n\n    if hunt_obj.client_rate > 0:\n      # Given that we use caching in _GetNumClients and hunt_obj may be updated\n      # in another process, we have to account for cases where num_clients_diff\n      # may go below 0.\n      num_clients_diff = max(\n          0,\n          _GetNumClients(hunt_obj.hunt_id) - hunt_obj.num_clients_at_start_time)\n      next_client_due_msecs = int(\n          num_clients_diff / hunt_obj.client_rate * 60e6)\n\n      start_at = rdfvalue.RDFDatetime.FromMicrosecondsSinceEpoch(\n          hunt_obj.last_start_time.AsMicrosecondsSinceEpoch() +\n          next_client_due_msecs)\n    else:\n      start_at = None\n\n    # TODO(user): remove client_rate support when AFF4 is gone.\n    # In REL_DB always work as if client rate is 0.\n\n    flow_cls = registry.FlowRegistry.FlowClassByName(hunt_args.flow_name)\n    flow_args = hunt_args.flow_args if hunt_args.HasField(\"flow_args\") else None\n    flow.StartFlow(\n        client_id=client_id,\n        creator=hunt_obj.creator,\n        cpu_limit=hunt_obj.per_client_cpu_limit,\n        network_bytes_limit=hunt_obj.per_client_network_bytes_limit,\n        flow_cls=flow_cls,\n        flow_args=flow_args,\n        start_at=start_at,\n        parent_hunt_id=hunt_id)\n\n    if hunt_obj.client_limit:\n      if _GetNumClients(hunt_obj.hunt_id) >= hunt_obj.client_limit:\n        PauseHunt(hunt_id)\n\n  elif hunt_obj.args.hunt_type == hunt_obj.args.HuntType.VARIABLE:\n    raise NotImplementedError()\n  else:\n    raise UnknownHuntTypeError(\"Can't determine hunt type when starting \"\n                               \"hunt %s on client %s.\" % (client_id, hunt_id))", "response": "Starts a flow corresponding to a given hunt on a given client."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ncheck if the hunt has expired.", "response": "def expired(self):\n    \"\"\"Checks if the hunt has expired.\"\"\"\n    expiry_time = self.expiry_time\n    if expiry_time is not None:\n      return expiry_time < rdfvalue.RDFDatetime.Now()\n    else:\n      return False"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef GetApprovalForObject(object_urn, token=None, username=\"\"):\n    if token is None:\n      raise access_control.UnauthorizedAccess(\n          \"No token given, cannot authenticate.\")\n\n    if not username:\n      username = token.username\n\n    approvals_root_urn = aff4.ROOT_URN.Add(\"ACL\").Add(\n        object_urn.Path()).Add(username)\n\n    children_urns = list(aff4.FACTORY.ListChildren(approvals_root_urn))\n    if not children_urns:\n      raise access_control.UnauthorizedAccess(\n          \"No approval found for user %s\" % utils.SmartStr(username),\n          subject=object_urn)\n\n    last_error = None\n    approvals = aff4.FACTORY.MultiOpen(\n        children_urns,\n        mode=\"r\",\n        aff4_type=Approval,\n        age=aff4.ALL_TIMES,\n        token=token)\n    for approval in approvals:\n      try:\n        test_token = access_control.ACLToken(\n            username=username, reason=approval.Get(approval.Schema.REASON))\n        approval.CheckAccess(test_token)\n\n        return test_token\n      except access_control.UnauthorizedAccess as e:\n        last_error = e\n\n    if last_error:\n      # We tried all possible approvals, but got no usable results.\n      raise access_control.UnauthorizedAccess(last_error, subject=object_urn)\n    else:\n      # If last error is None, means that none of the URNs in children_urns\n      # could be opened. This shouldn't really happen ever, but we have\n      # to make sure to provide a meaningful error message.\n      raise access_control.UnauthorizedAccess(\n          \"Couldn't open any of %d approvals \"\n          \"for user %s\" % (len(children_urns), utils.SmartStr(username)),\n          subject=object_urn)", "response": "Returns a list of tokens that can be used to access an object."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nenforces a dual approver policy for access.", "response": "def CheckAccess(self, token):\n    \"\"\"Enforce a dual approver policy for access.\"\"\"\n    namespace, _ = self.urn.Split(2)\n\n    if namespace != \"ACL\":\n      raise access_control.UnauthorizedAccess(\n          \"Approval object has invalid urn %s.\" % self.urn,\n          subject=self.urn,\n          requested_access=token.requested_access)\n\n    user, subject_urn = self.InferUserAndSubjectFromUrn()\n    if user != token.username:\n      raise access_control.UnauthorizedAccess(\n          \"Approval object is not for user %s.\" % token.username,\n          subject=self.urn,\n          requested_access=token.requested_access)\n\n    now = rdfvalue.RDFDatetime.Now()\n\n    # Is this an emergency access?\n    break_glass = self.Get(self.Schema.BREAK_GLASS)\n    if break_glass and now < break_glass:\n      # This tags the token as an emergency token.\n      token.is_emergency = True\n      return True\n\n    # Check that there are enough approvers.\n    approvers = self.GetNonExpiredApprovers()\n    approvers_required = config.CONFIG[\"ACL.approvers_required\"]\n    if len(approvers) < approvers_required:\n      missing = approvers_required - len(approvers)\n      msg = (\"Need at least %d additional approver%s for access.\" %\n             (missing, \"s\" if missing > 1 else \"\"))\n\n      raise access_control.UnauthorizedAccess(\n          msg, subject=subject_urn, requested_access=token.requested_access)\n\n    # Check User labels\n    if self.checked_approvers_label:\n      approvers_with_label = []\n\n      # We need to check labels with high privilege since normal users can\n      # inspect other user's labels.\n      for approver in approvers:\n        try:\n          user = aff4.FACTORY.Open(\n              \"aff4:/users/%s\" % approver,\n              aff4_type=aff4_users.GRRUser,\n              token=token.SetUID())\n          if self.checked_approvers_label in user.GetLabelsNames():\n            approvers_with_label.append(approver)\n        except IOError:\n          pass\n\n      if len(approvers_with_label) < self.min_approvers_with_label:\n        missing = self.min_approvers_with_label - len(approvers_with_label)\n        raise access_control.UnauthorizedAccess(\n            \"Need at least 1 admin approver for access.\",\n            subject=subject_urn,\n            requested_access=token.requested_access)\n\n    return True"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a list of usernames of approvers who approved this approval.", "response": "def GetNonExpiredApprovers(self):\n    \"\"\"Returns a list of usernames of approvers who approved this approval.\"\"\"\n\n    lifetime = rdfvalue.Duration(\n        self.Get(self.Schema.LIFETIME) or config.CONFIG[\"ACL.token_expiry\"])\n\n    # Check that there are enough approvers.\n    #\n    # TODO(user): approvals have to be opened with\n    # age=aff4.ALL_TIMES because versioning is used to store lists\n    # of approvers. This doesn't seem right and has to be fixed.\n    approvers = set()\n    now = rdfvalue.RDFDatetime.Now()\n    for approver in self.GetValuesForAttribute(self.Schema.APPROVER):\n      if approver.age + lifetime > now:\n        approvers.add(utils.SmartStr(approver))\n\n    return list(approvers)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ninfer user name and subject urn from self. urn.", "response": "def InferUserAndSubjectFromUrn(self):\n    \"\"\"Infers user name and subject urn from self.urn.\"\"\"\n    _, client_id, user, _ = self.urn.Split(4)\n    return (user, rdf_client.ClientURN(client_id))"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ninfers user name and subject urn from self. urn.", "response": "def InferUserAndSubjectFromUrn(self):\n    \"\"\"Infers user name and subject urn from self.urn.\"\"\"\n    _, hunts_str, hunt_id, user, _ = self.urn.Split(5)\n\n    if hunts_str != \"hunts\":\n      raise access_control.UnauthorizedAccess(\n          \"Approval object has invalid urn %s.\" % self.urn,\n          requested_access=self.token.requested_access)\n\n    return (user, aff4.ROOT_URN.Add(\"hunts\").Add(hunt_id))"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ninfer user name and subject urn from self. urn.", "response": "def InferUserAndSubjectFromUrn(self):\n    \"\"\"Infers user name and subject urn from self.urn.\"\"\"\n    _, cron_str, cron_job_name, user, _ = self.urn.Split(5)\n\n    if cron_str != \"cron\":\n      raise access_control.UnauthorizedAccess(\n          \"Approval object has invalid urn %s.\" % self.urn,\n          requested_access=self.token.requested_access)\n\n    return (user, aff4.ROOT_URN.Add(\"cron\").Add(cron_job_name))"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef ApprovalUrnBuilder(subject, user, approval_id):\n    return aff4.ROOT_URN.Add(\"ACL\").Add(subject).Add(user).Add(approval_id)", "response": "Encode an approval URN."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef ApprovalSymlinkUrnBuilder(approval_type, subject_id, user, approval_id):\n    return aff4.ROOT_URN.Add(\"users\").Add(user).Add(\"approvals\").Add(\n        approval_type).Add(subject_id).Add(approval_id)", "response": "Builds an approval symlink URN."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef Request(self):\n\n    approval_id = \"approval:%X\" % random.UInt32()\n    approval_urn = self.BuildApprovalUrn(approval_id)\n\n    email_msg_id = email.utils.make_msgid()\n\n    with aff4.FACTORY.Create(\n        approval_urn, self.approval_type, mode=\"w\",\n        token=self.token) as approval_request:\n      approval_request.Set(approval_request.Schema.SUBJECT(self.subject_urn))\n      approval_request.Set(\n          approval_request.Schema.REQUESTOR(self.token.username))\n      approval_request.Set(approval_request.Schema.REASON(self.reason))\n      approval_request.Set(approval_request.Schema.EMAIL_MSG_ID(email_msg_id))\n\n      cc_addresses = (self.email_cc_address,\n                      config.CONFIG.Get(\"Email.approval_cc_address\"))\n      email_cc = \",\".join(filter(None, cc_addresses))\n\n      # When we reply with the approval we want to cc all the people to whom the\n      # original approval was sent, to avoid people approving stuff that was\n      # already approved.\n      if email_cc:\n        reply_cc = \",\".join((self.approver, email_cc))\n      else:\n        reply_cc = self.approver\n\n      approval_request.Set(approval_request.Schema.EMAIL_CC(reply_cc))\n\n      approval_request.Set(\n          approval_request.Schema.NOTIFIED_USERS(self.approver))\n\n      # We add ourselves as an approver as well (The requirement is that we have\n      # 2 approvers, so the requester is automatically an approver).\n      approval_request.AddAttribute(\n          approval_request.Schema.APPROVER(self.token.username))\n\n    approval_link_urns = self.BuildApprovalSymlinksUrns(approval_id)\n    for link_urn in approval_link_urns:\n      with aff4.FACTORY.Create(\n          link_urn, aff4.AFF4Symlink, mode=\"w\", token=self.token) as link:\n        link.Set(link.Schema.SYMLINK_TARGET(approval_urn))\n\n    return approval_urn", "response": "Create the Approval object and notify the Approval Granter."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ncreate the Approval object and notify the Approval Granter.", "response": "def Grant(self):\n    \"\"\"Create the Approval object and notify the Approval Granter.\"\"\"\n\n    approvals_root_urn = aff4.ROOT_URN.Add(\"ACL\").Add(\n        self.subject_urn.Path()).Add(self.delegate)\n    children_urns = list(aff4.FACTORY.ListChildren(approvals_root_urn))\n    if not children_urns:\n      raise access_control.UnauthorizedAccess(\n          \"No approval found for user %s\" % utils.SmartStr(self.token.username),\n          subject=self.subject_urn)\n\n    approvals = aff4.FACTORY.MultiOpen(\n        children_urns, mode=\"r\", aff4_type=Approval, token=self.token)\n    found_approval_urn = None\n    for approval in approvals:\n      approval_reason = approval.Get(approval.Schema.REASON)\n      if (utils.SmartUnicode(approval_reason) == utils.SmartUnicode(self.reason)\n          and (not found_approval_urn or\n               approval_reason.age > found_approval_urn.age)):\n        found_approval_urn = approval.urn\n        found_approval_urn.age = approval_reason.age\n\n    if not found_approval_urn:\n      raise access_control.UnauthorizedAccess(\n          \"No approval with reason '%s' found for user %s\" % (utils.SmartStr(\n              self.reason), utils.SmartStr(self.token.username)),\n          subject=self.subject_urn)\n\n    # This object must already exist.\n    try:\n      approval_request = aff4.FACTORY.Open(\n          found_approval_urn,\n          mode=\"rw\",\n          aff4_type=self.approval_type,\n          token=self.token)\n    except IOError:\n      raise access_control.UnauthorizedAccess(\n          \"Approval object does not exist.\", requested_access=\"rw\")\n\n    with approval_request:\n      # We are now an approver for this request.\n      approval_request.AddAttribute(\n          approval_request.Schema.APPROVER(self.token.username))\n\n    return found_approval_urn"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nbuilds approval object urn.", "response": "def BuildApprovalUrn(self, approval_id):\n    \"\"\"Builds approval object urn.\"\"\"\n    event = rdf_events.AuditEvent(\n        user=self.token.username,\n        action=\"CLIENT_APPROVAL_REQUEST\",\n        client=self.subject_urn,\n        description=self.reason)\n    events.Events.PublishEvent(\"Audit\", event, token=self.token)\n\n    return self.ApprovalUrnBuilder(self.subject_urn.Path(), self.token.username,\n                                   approval_id)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef BuildApprovalSymlinksUrns(self, approval_id):\n    return [\n        self.ApprovalSymlinkUrnBuilder(\"cron\", self.subject_urn.Basename(),\n                                       self.token.username, approval_id)\n    ]", "response": "Builds the list of symlinks URNs for the approval object."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef CheckBlobsExist(self, blob_ids):\n\n    result = {}\n    for blob_id in blob_ids:\n      result[blob_id] = blob_id in self.blobs\n\n    return result", "response": "Checks if given blobs exist."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef FileFinderOSFromClient(args):\n  stat_cache = filesystem.StatCache()\n\n  opts = args.action.stat\n\n  for path in GetExpandedPaths(args):\n    try:\n      content_conditions = conditions.ContentCondition.Parse(args.conditions)\n      for content_condition in content_conditions:\n        with io.open(path, \"rb\") as fd:\n          result = list(content_condition.Search(fd))\n        if not result:\n          raise _SkipFileException()\n      # TODO: `opts.resolve_links` has type `RDFBool`, not `bool`.\n      stat = stat_cache.Get(path, follow_symlink=bool(opts.resolve_links))\n      stat_entry = client_utils.StatEntryFromStatPathSpec(\n          stat, ext_attrs=opts.collect_ext_attrs)\n      yield stat_entry\n    except _SkipFileException:\n      pass", "response": "This function expands paths from the args and returns related stat entries."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning an iterator that yields absolute paths derived from input patterns.", "response": "def GetExpandedPaths(\n    args):\n  \"\"\"Expands given path patterns.\n\n  Args:\n    args: A `FileFinderArgs` instance that dictates the behaviour of the path\n      expansion.\n\n  Yields:\n    Absolute paths (as string objects) derived from input patterns.\n\n  Raises:\n    ValueError: For unsupported path types.\n  \"\"\"\n  if args.pathtype == rdf_paths.PathSpec.PathType.OS:\n    pathtype = rdf_paths.PathSpec.PathType.OS\n  else:\n    raise ValueError(\"Unsupported path type: \", args.pathtype)\n\n  opts = globbing.PathOpts(\n      follow_links=args.follow_links,\n      recursion_blacklist=_GetMountpointBlacklist(args.xdev),\n      pathtype=pathtype)\n\n  for path in args.paths:\n    for expanded_path in globbing.ExpandPath(str(path), opts):\n      yield expanded_path"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _GetMountpoints(only_physical=True):\n  partitions = psutil.disk_partitions(all=not only_physical)\n  return set(partition.mountpoint for partition in partitions)", "response": "Fetches a list of mountpoints."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _GetMountpointBlacklist(xdev):\n  if xdev == rdf_file_finder.FileFinderArgs.XDev.NEVER:\n    # Never cross device boundaries, stop at all mount points.\n    return _GetMountpoints(only_physical=False)\n\n  if xdev == rdf_file_finder.FileFinderArgs.XDev.LOCAL:\n    # Descend into file systems on physical devices only.\n    physical = _GetMountpoints(only_physical=True)\n    return _GetMountpoints(only_physical=False) - physical\n\n  if xdev == rdf_file_finder.FileFinderArgs.XDev.ALWAYS:\n    # Never stop at any device boundary.\n    return set()\n\n  raise ValueError(\"Incorrect `xdev` value: %s\" % xdev)", "response": "Returns a list of mountpoints to ignore during recursive searches."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef WriteHuntObject(self, hunt_obj):\n    if hunt_obj.hunt_id in self.hunts:\n      raise db.DuplicatedHuntError(hunt_id=hunt_obj.hunt_id)\n\n    clone = self._DeepCopy(hunt_obj)\n    clone.create_time = rdfvalue.RDFDatetime.Now()\n    clone.last_update_time = rdfvalue.RDFDatetime.Now()\n    self.hunts[(hunt_obj.hunt_id)] = clone", "response": "Writes a hunt object to the database."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef UpdateHuntObject(self, hunt_id, start_time=None, **kwargs):\n    hunt_obj = self.ReadHuntObject(hunt_id)\n\n    delta_suffix = \"_delta\"\n\n    for k, v in kwargs.items():\n      if v is None:\n        continue\n\n      if k.endswith(delta_suffix):\n        key = k[:-len(delta_suffix)]\n        current_value = getattr(hunt_obj, key)\n        setattr(hunt_obj, key, current_value + v)\n      else:\n        setattr(hunt_obj, k, v)\n\n    if start_time is not None:\n      hunt_obj.init_start_time = hunt_obj.init_start_time or start_time\n      hunt_obj.last_start_time = start_time\n\n    hunt_obj.last_update_time = rdfvalue.RDFDatetime.Now()\n    self.hunts[hunt_obj.hunt_id] = hunt_obj", "response": "Updates the hunt object by applying the update function."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nupdates hunt output plugin state for a given hunt.", "response": "def UpdateHuntOutputPluginState(self, hunt_id, state_index, update_fn):\n    \"\"\"Updates hunt output plugin state for a given output plugin.\"\"\"\n\n    if hunt_id not in self.hunts:\n      raise db.UnknownHuntError(hunt_id)\n\n    try:\n      state = rdf_flow_runner.OutputPluginState.FromSerializedString(\n          self.hunt_output_plugins_states[hunt_id][state_index])\n    except KeyError:\n      raise db.UnknownHuntOutputPluginError(hunt_id, state_index)\n\n    state.plugin_state = update_fn(state.plugin_state)\n\n    self.hunt_output_plugins_states[hunt_id][\n        state_index] = state.SerializeToString()\n\n    return state.plugin_state"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nread a hunt object from the database.", "response": "def ReadHuntObject(self, hunt_id):\n    \"\"\"Reads a hunt object from the database.\"\"\"\n    try:\n      return self._DeepCopy(self.hunts[hunt_id])\n    except KeyError:\n      raise db.UnknownHuntError(hunt_id)"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nread all hunt objects from the database.", "response": "def ReadHuntObjects(self,\n                      offset,\n                      count,\n                      with_creator=None,\n                      created_after=None,\n                      with_description_match=None):\n    \"\"\"Reads all hunt objects from the database.\"\"\"\n    filter_fns = []\n    if with_creator is not None:\n      filter_fns.append(lambda h: h.creator == with_creator)\n    if created_after is not None:\n      filter_fns.append(lambda h: h.create_time > created_after)\n    if with_description_match is not None:\n      filter_fns.append(lambda h: with_description_match in h.description)\n    filter_fn = lambda h: all(f(h) for f in filter_fns)\n\n    result = [self._DeepCopy(h) for h in self.hunts.values() if filter_fn(h)]\n    return sorted(\n        result, key=lambda h: h.create_time,\n        reverse=True)[offset:offset + (count or db.MAX_COUNT)]"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreading hunt log entries of a given hunt.", "response": "def ReadHuntLogEntries(self, hunt_id, offset, count, with_substring=None):\n    \"\"\"Reads hunt log entries of a given hunt using given query options.\"\"\"\n    all_entries = []\n    for flow_obj in self._GetHuntFlows(hunt_id):\n      for entry in self.ReadFlowLogEntries(\n          flow_obj.client_id,\n          flow_obj.flow_id,\n          0,\n          sys.maxsize,\n          with_substring=with_substring):\n\n        all_entries.append(\n            rdf_flow_objects.FlowLogEntry(\n                hunt_id=hunt_id,\n                client_id=flow_obj.client_id,\n                flow_id=flow_obj.flow_id,\n                timestamp=entry.timestamp,\n                message=entry.message))\n\n    return sorted(all_entries, key=lambda x: x.timestamp)[offset:offset + count]"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nread hunt results of a given hunt.", "response": "def ReadHuntResults(self,\n                      hunt_id,\n                      offset,\n                      count,\n                      with_tag=None,\n                      with_type=None,\n                      with_substring=None,\n                      with_timestamp=None):\n    \"\"\"Reads hunt results of a given hunt using given query options.\"\"\"\n    all_results = []\n    for flow_obj in self._GetHuntFlows(hunt_id):\n      for entry in self.ReadFlowResults(\n          flow_obj.client_id,\n          flow_obj.flow_id,\n          0,\n          sys.maxsize,\n          with_tag=with_tag,\n          with_type=with_type,\n          with_substring=with_substring):\n        all_results.append(\n            rdf_flow_objects.FlowResult(\n                hunt_id=hunt_id,\n                client_id=flow_obj.client_id,\n                flow_id=flow_obj.flow_id,\n                timestamp=entry.timestamp,\n                tag=entry.tag,\n                payload=entry.payload))\n\n    if with_timestamp:\n      all_results = [r for r in all_results if r.timestamp == with_timestamp]\n\n    return sorted(all_results, key=lambda x: x.timestamp)[offset:offset + count]"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncount hunt results of a given hunt using given query options.", "response": "def CountHuntResults(self, hunt_id, with_tag=None, with_type=None):\n    \"\"\"Counts hunt results of a given hunt using given query options.\"\"\"\n    return len(\n        self.ReadHuntResults(\n            hunt_id, 0, sys.maxsize, with_tag=with_tag, with_type=with_type))"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nread hunt flows matching given conditins.", "response": "def ReadHuntFlows(self,\n                    hunt_id,\n                    offset,\n                    count,\n                    filter_condition=db.HuntFlowsCondition.UNSET):\n    \"\"\"Reads hunt flows matching given conditins.\"\"\"\n    if filter_condition == db.HuntFlowsCondition.UNSET:\n      filter_fn = lambda _: True\n    elif filter_condition == db.HuntFlowsCondition.FAILED_FLOWS_ONLY:\n      filter_fn = lambda f: f.flow_state == f.FlowState.ERROR\n    elif filter_condition == db.HuntFlowsCondition.SUCCEEDED_FLOWS_ONLY:\n      filter_fn = lambda f: f.flow_state == f.FlowState.FINISHED\n    elif filter_condition == db.HuntFlowsCondition.COMPLETED_FLOWS_ONLY:\n      filter_fn = (\n          lambda f: f.flow_state in [f.FlowState.ERROR, f.FlowState.FINISHED])\n    elif filter_condition == db.HuntFlowsCondition.FLOWS_IN_PROGRESS_ONLY:\n      filter_fn = lambda f: f.flow_state == f.FlowState.RUNNING\n    elif filter_condition == db.HuntFlowsCondition.CRASHED_FLOWS_ONLY:\n      filter_fn = lambda f: f.flow_state == f.FlowState.CRASHED\n    else:\n      raise ValueError(\"Invalid filter condition: %d\" % filter_condition)\n\n    results = [\n        flow_obj for flow_obj in self._GetHuntFlows(hunt_id)\n        if filter_fn(flow_obj)\n    ]\n    results.sort(key=lambda f: f.last_update_time)\n    return results[offset:offset + count]"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ncount the number of hunt flows matching given conditions.", "response": "def CountHuntFlows(self,\n                     hunt_id,\n                     filter_condition=db.HuntFlowsCondition.UNSET):\n    \"\"\"Counts hunt flows matching given conditions.\"\"\"\n\n    return len(\n        self.ReadHuntFlows(\n            hunt_id, 0, sys.maxsize, filter_condition=filter_condition))"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef ReadHuntClientResourcesStats(self, hunt_id):\n\n    result = rdf_stats.ClientResourcesStats()\n    for f in self._GetHuntFlows(hunt_id):\n      cr = rdf_client_stats.ClientResources(\n          session_id=\"%s/%s\" % (f.client_id, f.flow_id),\n          client_id=f.client_id,\n          cpu_usage=f.cpu_time_used,\n          network_bytes_sent=f.network_bytes_sent)\n      result.RegisterResources(cr)\n\n    # TODO(user): remove this hack when compatibility with AFF4 is not\n    # important.\n    return rdf_stats.ClientResourcesStats.FromSerializedString(\n        result.SerializeToString())", "response": "Reads and calculates hunt client resources stats."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nread hunt flows states and timestamps.", "response": "def ReadHuntFlowsStatesAndTimestamps(self, hunt_id):\n    \"\"\"Reads hunt flows states and timestamps.\"\"\"\n\n    result = []\n    for f in self._GetHuntFlows(hunt_id):\n      result.append(\n          db.FlowStateAndTimestamps(\n              flow_state=f.flow_state,\n              create_time=f.create_time,\n              last_update_time=f.last_update_time))\n\n    return result"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreads hunt output plugin log entries.", "response": "def ReadHuntOutputPluginLogEntries(self,\n                                     hunt_id,\n                                     output_plugin_id,\n                                     offset,\n                                     count,\n                                     with_type=None):\n    \"\"\"Reads hunt output plugin log entries.\"\"\"\n\n    all_entries = []\n    for flow_obj in self._GetHuntFlows(hunt_id):\n      for entry in self.ReadFlowOutputPluginLogEntries(\n          flow_obj.client_id,\n          flow_obj.flow_id,\n          output_plugin_id,\n          0,\n          sys.maxsize,\n          with_type=with_type):\n        all_entries.append(\n            rdf_flow_objects.FlowOutputPluginLogEntry(\n                hunt_id=hunt_id,\n                client_id=flow_obj.client_id,\n                flow_id=flow_obj.flow_id,\n                output_plugin_id=output_plugin_id,\n                log_entry_type=entry.log_entry_type,\n                timestamp=entry.timestamp,\n                message=entry.message))\n\n    return sorted(all_entries, key=lambda x: x.timestamp)[offset:offset + count]"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncounts hunt output plugin log entries.", "response": "def CountHuntOutputPluginLogEntries(self,\n                                      hunt_id,\n                                      output_plugin_id,\n                                      with_type=None):\n    \"\"\"Counts hunt output plugin log entries.\"\"\"\n\n    return len(\n        self.ReadHuntOutputPluginLogEntries(\n            hunt_id, output_plugin_id, 0, sys.maxsize, with_type=with_type))"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nwrite a foreman rule to the database.", "response": "def WriteForemanRule(self, rule, cursor=None):\n    \"\"\"Writes a foreman rule to the database.\"\"\"\n    query = (\"INSERT INTO foreman_rules \"\n             \"  (hunt_id, expiration_time, rule) \"\n             \"VALUES (%s, FROM_UNIXTIME(%s), %s) \"\n             \"ON DUPLICATE KEY UPDATE \"\n             \"  expiration_time=FROM_UNIXTIME(%s), rule=%s\")\n\n    exp_str = mysql_utils.RDFDatetimeToTimestamp(rule.expiration_time),\n    rule_str = rule.SerializeToString()\n    cursor.execute(query, [rule.hunt_id, exp_str, rule_str, exp_str, rule_str])"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nrun the main test harness.", "response": "def main(_):\n  \"\"\"Run the main test harness.\"\"\"\n\n  if flags.FLAGS.version:\n    print(\"GRR Admin UI {}\".format(config_server.VERSION[\"packageversion\"]))\n    return\n\n  config.CONFIG.AddContext(\n      contexts.ADMIN_UI_CONTEXT,\n      \"Context applied when running the admin user interface GUI.\")\n  server_startup.Init()\n\n  if not config.CONFIG[\"AdminUI.headless\"] and (not os.path.exists(\n      os.path.join(config.CONFIG[\"AdminUI.document_root\"],\n                   \"dist/grr-ui.bundle.js\")) or not os.path.exists(\n                       os.path.join(config.CONFIG[\"AdminUI.document_root\"],\n                                    \"dist/grr-ui.bundle.css\"))):\n    raise RuntimeError(\"Can't find compiled JS/CSS bundles. \"\n                       \"Please reinstall the PIP package using \"\n                       \"\\\"pip install -e .\\\" to rebuild the bundles.\")\n\n  # Start up a server in another thread\n  bind_address = config.CONFIG[\"AdminUI.bind\"]\n  ip = ipaddress.ip_address(bind_address)\n  if ip.version == 4:\n    # Address looks like an IPv4 address.\n    ThreadedServer.address_family = socket.AF_INET\n\n  max_port = config.CONFIG.Get(\"AdminUI.port_max\",\n                               config.CONFIG[\"AdminUI.port\"])\n\n  for port in range(config.CONFIG[\"AdminUI.port\"], max_port + 1):\n    # Make a simple reference implementation WSGI server\n    try:\n      server = simple_server.make_server(\n          bind_address,\n          port,\n          wsgiapp.AdminUIApp().WSGIHandler(),\n          server_class=ThreadedServer)\n      break\n    except socket.error as e:\n      if e.errno == socket.errno.EADDRINUSE and port < max_port:\n        logging.info(\"Port %s in use, trying %s\", port, port + 1)\n      else:\n        raise\n\n  proto = \"HTTP\"\n\n  if config.CONFIG[\"AdminUI.enable_ssl\"]:\n    cert_file = config.CONFIG[\"AdminUI.ssl_cert_file\"]\n    if not cert_file:\n      raise ValueError(\"Need a valid cert file to enable SSL.\")\n\n    key_file = config.CONFIG[\"AdminUI.ssl_key_file\"]\n    server.socket = ssl.wrap_socket(\n        server.socket, certfile=cert_file, keyfile=key_file, server_side=True)\n    proto = \"HTTPS\"\n\n    # SSL errors are swallowed by the WSGIServer so if your configuration does\n    # not work, uncomment the line below, point your browser at the gui and look\n    # at the log file to see why SSL complains:\n    # server.socket.accept()\n\n  sa = server.socket.getsockname()\n  logging.info(\"Serving %s on %s port %d ...\", proto, sa[0], sa[1])\n  server_startup.DropPrivileges()\n\n  server.serve_forever()"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef WriteAllCrashDetails(client_id,\n                         crash_details,\n                         flow_session_id=None,\n                         hunt_session_id=None,\n                         token=None):\n  \"\"\"Updates the last crash attribute of the client.\"\"\"\n  # AFF4.\n  if data_store.AFF4Enabled():\n    with aff4.FACTORY.Create(\n        client_id, aff4_grr.VFSGRRClient, token=token) as client_obj:\n      client_obj.Set(client_obj.Schema.LAST_CRASH(crash_details))\n\n    # Duplicate the crash information in a number of places so we can find it\n    # easily.\n    client_urn = rdf_client.ClientURN(client_id)\n    client_crashes = aff4_grr.VFSGRRClient.CrashCollectionURNForCID(client_urn)\n    with data_store.DB.GetMutationPool() as pool:\n      grr_collections.CrashCollection.StaticAdd(\n          client_crashes, crash_details, mutation_pool=pool)\n\n  # Relational db.\n  if data_store.RelationalDBEnabled():\n    try:\n      data_store.REL_DB.WriteClientCrashInfo(client_id, crash_details)\n    except db.UnknownClientError:\n      pass\n\n  if not flow_session_id:\n    return\n\n  if data_store.RelationalDBEnabled():\n    flow_id = flow_session_id.Basename()\n    data_store.REL_DB.UpdateFlow(\n        client_id, flow_id, client_crash_info=crash_details)\n\n    flow_obj = data_store.REL_DB.ReadFlowObject(client_id, flow_id)\n    if flow_obj.parent_hunt_id:\n      db_compat.ProcessHuntClientCrash(\n          flow_obj, client_crash_info=crash_details)\n\n  # TODO(amoser): Registering crashes in hunts is currently not implemented for\n  # the relational db.\n  if not data_store.RelationalDBEnabled():\n    with aff4.FACTORY.Open(\n        flow_session_id,\n        flow.GRRFlow,\n        mode=\"rw\",\n        age=aff4.NEWEST_TIME,\n        token=token) as aff4_flow:\n      aff4_flow.Set(aff4_flow.Schema.CLIENT_CRASH(crash_details))\n\n    hunt_session_id = ExtractHuntId(flow_session_id)\n    if hunt_session_id and hunt_session_id != flow_session_id:\n      hunt_obj = aff4.FACTORY.Open(\n          hunt_session_id,\n          aff4_type=implementation.GRRHunt,\n          mode=\"rw\",\n          token=token)\n      hunt_obj.RegisterCrash(crash_details)", "response": "Writes the last crash information for a given client."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef ProcessMessages(self, msgs=None, token=None):\n    nanny_msg = \"\"\n\n    for crash_details in msgs:\n      client_urn = crash_details.client_id\n      client_id = client_urn.Basename()\n\n      # The session id of the flow that crashed.\n      session_id = crash_details.session_id\n\n      # Log.\n      logging.info(\"Client crash reported, client %s.\", client_urn)\n\n      # Export.\n      stats_collector_instance.Get().IncrementCounter(\"grr_client_crashes\")\n\n      # Write crash data.\n      if data_store.RelationalDBEnabled():\n        client = data_store.REL_DB.ReadClientSnapshot(client_id)\n        if client:\n          crash_details.client_info = client.startup_info.client_info\n          hostname = client.knowledge_base.fqdn\n        else:\n          hostname = \"\"\n\n      if data_store.AFF4Enabled():\n        client = aff4.FACTORY.Open(client_urn, token=token)\n        client_info = client.Get(client.Schema.CLIENT_INFO)\n        hostname = client.Get(client.Schema.FQDN)\n        if client_info:\n          crash_details.client_info = client_info\n\n      crash_details.crash_type = \"Client Crash\"\n\n      if nanny_msg:\n        termination_msg = \"Client crashed, \" + nanny_msg\n      else:\n        termination_msg = \"Client crashed.\"\n\n      # Terminate the flow.\n      if data_store.RelationalDBEnabled():\n        flow_id = session_id.Basename()\n        flow_base.TerminateFlow(\n            client_id,\n            flow_id,\n            reason=termination_msg,\n            flow_state=rdf_flow_objects.Flow.FlowState.CRASHED)\n      else:\n        flow.GRRFlow.TerminateAFF4Flow(\n            session_id, reason=termination_msg, token=token)\n\n      WriteAllCrashDetails(\n          client_id, crash_details, flow_session_id=session_id, token=token)\n\n      # Also send email.\n      to_send = []\n\n      try:\n        hunt_session_id = ExtractHuntId(session_id)\n        if hunt_session_id and hunt_session_id != session_id:\n\n          # TODO(amoser): Enable this for the relational db once we have hunt\n          # metadata.\n          if data_store.AFF4Enabled():\n            hunt_obj = aff4.FACTORY.Open(\n                hunt_session_id, aff4_type=implementation.GRRHunt, token=token)\n            email = hunt_obj.runner_args.crash_alert_email\n\n          if email:\n            to_send.append(email)\n      except aff4.InstantiationError:\n        logging.error(\"Failed to open hunt %s.\", hunt_session_id)\n\n      email = config.CONFIG[\"Monitoring.alert_email\"]\n      if email:\n        to_send.append(email)\n\n      for email_address in to_send:\n        if crash_details.nanny_status:\n          nanny_msg = \"Nanny status: %s\" % crash_details.nanny_status\n\n        body = self.__class__.mail_template.render(\n            client_id=client_id,\n            admin_ui=config.CONFIG[\"AdminUI.url\"],\n            hostname=utils.SmartUnicode(hostname),\n            url=\"/clients/%s\" % client_id,\n            nanny_msg=utils.SmartUnicode(nanny_msg),\n            signature=config.CONFIG[\"Email.signature\"])\n\n        email_alerts.EMAIL_ALERTER.SendEmail(\n            email_address,\n            \"GRR server\",\n            \"Client %s reported a crash.\" % client_id,\n            utils.SmartStr(body),\n            is_html=True)", "response": "Processes the given messages."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nprocess the response of a client.", "response": "def ProcessResponse(self, client_id, response):\n    \"\"\"Actually processes the contents of the response.\"\"\"\n    precondition.AssertType(client_id, Text)\n    downsampled = rdf_client_stats.ClientStats.Downsampled(response)\n\n    if data_store.AFF4Enabled():\n      urn = rdf_client.ClientURN(client_id).Add(\"stats\")\n\n      with aff4.FACTORY.Create(\n          urn, aff4_stats.ClientStats, token=self.token, mode=\"w\") as stats_fd:\n        # Only keep the average of all values that fall within one minute.\n        stats_fd.AddAttribute(stats_fd.Schema.STATS, downsampled)\n\n    if data_store.RelationalDBEnabled():\n      data_store.REL_DB.WriteClientStats(client_id, downsampled)\n\n    return downsampled"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nprocesses a stats response from the client.", "response": "def ProcessMessage(self, message):\n    \"\"\"Processes a stats response from the client.\"\"\"\n    self.ProcessResponse(message.source.Basename(), message.payload)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef ProcessMessage(self, message):\n    # Only accept authenticated messages\n    if (message.auth_state !=\n        rdf_flows.GrrMessage.AuthorizationState.AUTHENTICATED):\n      return\n\n    now = time.time()\n\n    # Maintain a cache of the foreman\n    with self.lock:\n      if (self.foreman_cache is None or\n          now > self.foreman_cache.age + self.cache_refresh_time):\n        self.foreman_cache = aff4.FACTORY.Open(\n            \"aff4:/foreman\", mode=\"rw\", token=self.token)\n        self.foreman_cache.age = now\n\n    if message.source:\n      self.foreman_cache.AssignTasksToClient(message.source.Basename())", "response": "Process a message from the client."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nsending an email to the GRR server.", "response": "def SendEmail(self, client_id, message):\n    \"\"\"Processes this event.\"\"\"\n    logging.info(self.logline, client_id, message)\n\n    client_info = None\n    hostname = None\n\n    # Write crash data.\n    if data_store.RelationalDBEnabled():\n      client = data_store.REL_DB.ReadClientSnapshot(client_id)\n      if client is not None:\n        client_info = client.startup_info.client_info\n        hostname = client.knowledge_base.fqdn\n    else:\n      client = aff4.FACTORY.Open(client_id, token=self.token)\n      client_info = client.Get(client.Schema.CLIENT_INFO)\n      hostname = client.Get(client.Schema.FQDN)\n\n    crash_details = rdf_client.ClientCrash(\n        client_id=client_id,\n        client_info=client_info,\n        crash_message=message,\n        timestamp=int(time.time() * 1e6),\n        crash_type=\"Nanny Message\")\n\n    WriteAllCrashDetails(client_id, crash_details, token=self.token)\n\n    # Also send email.\n    if config.CONFIG[\"Monitoring.alert_email\"]:\n      url = \"/clients/%s\" % client_id\n      body = self.__class__.mail_template.render(\n          client_id=client_id,\n          admin_ui=config.CONFIG[\"AdminUI.url\"],\n          hostname=utils.SmartUnicode(hostname),\n          signature=config.CONFIG[\"Email.signature\"],\n          url=url,\n          message=utils.SmartUnicode(message))\n      email_alerts.EMAIL_ALERTER.SendEmail(\n          config.CONFIG[\"Monitoring.alert_email\"],\n          \"GRR server\",\n          self.subject % client_id,\n          utils.SmartStr(body),\n          is_html=True)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef WriteClientStartupInfo(self, client_id, new_si):\n    drift = rdfvalue.Duration(\"5m\")\n\n    if data_store.RelationalDBEnabled():\n      current_si = data_store.REL_DB.ReadClientStartupInfo(client_id)\n\n      # We write the updated record if the client_info has any changes\n      # or the boot time is more than 5 minutes different.\n      if (not current_si or current_si.client_info != new_si.client_info or\n          not current_si.boot_time or\n          abs(current_si.boot_time - new_si.boot_time) > drift):\n        try:\n          data_store.REL_DB.WriteClientStartupInfo(client_id, new_si)\n        except db.UnknownClientError:\n          # On first contact with a new client, this write will fail.\n          logging.info(\"Can't write StartupInfo for unknown client %s\",\n                       client_id)\n    else:\n      changes = False\n      with aff4.FACTORY.Create(\n          client_id, aff4_grr.VFSGRRClient, mode=\"rw\",\n          token=self.token) as client:\n        old_info = client.Get(client.Schema.CLIENT_INFO)\n        old_boot = client.Get(client.Schema.LAST_BOOT_TIME, 0)\n\n        info = new_si.client_info\n\n        # Only write to the datastore if we have new information.\n        if info != old_info:\n          client.Set(client.Schema.CLIENT_INFO(info))\n          changes = True\n\n        client.AddLabels(info.labels, owner=\"GRR\")\n\n        # Allow for some drift in the boot times (5 minutes).\n        if not old_boot or abs(old_boot - new_si.boot_time) > drift:\n          client.Set(client.Schema.LAST_BOOT_TIME(new_si.boot_time))\n          changes = True\n\n      if data_store.RelationalDBEnabled() and changes:\n        try:\n          data_store.REL_DB.WriteClientStartupInfo(client_id, new_si)\n        except db.UnknownClientError:\n          pass", "response": "Write a startup info record for a given client."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef WriteMessageHandlerRequests(self, requests):\n    now = rdfvalue.RDFDatetime.Now()\n    for r in requests:\n      flow_dict = self.message_handler_requests.setdefault(r.handler_name, {})\n      cloned_request = r.Copy()\n      cloned_request.timestamp = now\n      flow_dict[cloned_request.request_id] = cloned_request", "response": "Writes a list of message handler requests to the database."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreading all message handler requests from the database.", "response": "def ReadMessageHandlerRequests(self):\n    \"\"\"Reads all message handler requests from the database.\"\"\"\n    res = []\n    leases = self.message_handler_leases\n    for requests in itervalues(self.message_handler_requests):\n      for r in itervalues(requests):\n        res.append(r.Copy())\n        existing_lease = leases.get(r.handler_name, {}).get(r.request_id, None)\n        res[-1].leased_until = existing_lease\n\n    return sorted(res, key=lambda r: -1 * r.timestamp)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef DeleteMessageHandlerRequests(self, requests):\n\n    for r in requests:\n      flow_dict = self.message_handler_requests.get(r.handler_name, {})\n      if r.request_id in flow_dict:\n        del flow_dict[r.request_id]\n      flow_dict = self.message_handler_leases.get(r.handler_name, {})\n      if r.request_id in flow_dict:\n        del flow_dict[r.request_id]", "response": "Deletes a list of message handler requests from the database."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nregistering a message handler.", "response": "def RegisterMessageHandler(self, handler, lease_time, limit=1000):\n    \"\"\"Leases a number of message handler requests up to the indicated limit.\"\"\"\n    self.UnregisterMessageHandler()\n\n    self.handler_stop = False\n    self.handler_thread = threading.Thread(\n        name=\"message_handler\",\n        target=self._MessageHandlerLoop,\n        args=(handler, lease_time, limit))\n    self.handler_thread.daemon = True\n    self.handler_thread.start()"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nread and lease some outstanding message handler requests.", "response": "def _LeaseMessageHandlerRequests(self, lease_time, limit):\n    \"\"\"Read and lease some outstanding message handler requests.\"\"\"\n    leased_requests = []\n\n    now = rdfvalue.RDFDatetime.Now()\n    zero = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(0)\n    expiration_time = now + lease_time\n\n    leases = self.message_handler_leases\n    for requests in itervalues(self.message_handler_requests):\n      for r in itervalues(requests):\n        existing_lease = leases.get(r.handler_name, {}).get(r.request_id, zero)\n        if existing_lease < now:\n          leases.setdefault(r.handler_name, {})[r.request_id] = expiration_time\n          r.leased_until = expiration_time\n          r.leased_by = utils.ProcessIdString()\n          leased_requests.append(r)\n          if len(leased_requests) >= limit:\n            break\n\n    return leased_requests"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef ReadAllClientActionRequests(self, client_id):\n    res = []\n    for key, orig_request in iteritems(self.client_action_requests):\n      request_client_id, _, _ = key\n      if request_client_id != client_id:\n        continue\n\n      request = orig_request.Copy()\n      current_lease = self.client_action_request_leases.get(key)\n      request.ttl = db.Database.CLIENT_MESSAGES_TTL\n      if current_lease is not None:\n        request.leased_until, request.leased_by, leased_count = current_lease\n        request.ttl -= leased_count\n      else:\n        request.leased_until = None\n        request.leased_by = None\n      res.append(request)\n\n    return res", "response": "Reads all client action requests available for a given client_id."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef LeaseClientActionRequests(self,\n                                client_id,\n                                lease_time=None,\n                                limit=sys.maxsize):\n    \"\"\"Leases available client action requests for a client.\"\"\"\n\n    leased_requests = []\n\n    now = rdfvalue.RDFDatetime.Now()\n    expiration_time = now + lease_time\n    process_id_str = utils.ProcessIdString()\n\n    leases = self.client_action_request_leases\n    # Can't use an iterator here since the dict might change when requests get\n    # deleted.\n    for key, request in sorted(self.client_action_requests.items()):\n      if key[0] != client_id:\n        continue\n\n      existing_lease = leases.get(key)\n      if not existing_lease or existing_lease[0] < now:\n        if existing_lease:\n          lease_count = existing_lease[-1] + 1\n          if lease_count > db.Database.CLIENT_MESSAGES_TTL:\n            self._DeleteClientActionRequest(*key)\n            continue\n        else:\n          lease_count = 1\n\n        leases[key] = (expiration_time, process_id_str, lease_count)\n        request.leased_until = expiration_time\n        request.leased_by = process_id_str\n        request.ttl = db.Database.CLIENT_MESSAGES_TTL - lease_count\n        leased_requests.append(request)\n        if len(leased_requests) >= limit:\n          break\n\n    return leased_requests", "response": "Leases available client action requests for a client."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nwrites messages that should go to the client to the db.", "response": "def WriteClientActionRequests(self, requests):\n    \"\"\"Writes messages that should go to the client to the db.\"\"\"\n    for r in requests:\n      req_dict = self.flow_requests.get((r.client_id, r.flow_id), {})\n      if r.request_id not in req_dict:\n        request_keys = [(r.client_id, r.flow_id, r.request_id) for r in requests\n                       ]\n        raise db.AtLeastOneUnknownRequestError(request_keys)\n\n    for r in requests:\n      request_key = (r.client_id, r.flow_id, r.request_id)\n      self.client_action_requests[request_key] = r"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef WriteFlowObject(self, flow_obj):\n    if flow_obj.client_id not in self.metadatas:\n      raise db.UnknownClientError(flow_obj.client_id)\n\n    clone = flow_obj.Copy()\n    clone.last_update_time = rdfvalue.RDFDatetime.Now()\n    self.flows[(flow_obj.client_id, flow_obj.flow_id)] = clone", "response": "Writes a flow object to the database."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nread a flow object from the database.", "response": "def ReadFlowObject(self, client_id, flow_id):\n    \"\"\"Reads a flow object from the database.\"\"\"\n    try:\n      return self.flows[(client_id, flow_id)].Copy()\n    except KeyError:\n      raise db.UnknownFlowError(client_id, flow_id)"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreads all flow objects.", "response": "def ReadAllFlowObjects(\n      self,\n      client_id = None,\n      min_create_time = None,\n      max_create_time = None,\n      include_child_flows = True,\n  ):\n    \"\"\"Returns all flow objects.\"\"\"\n    res = []\n    for flow in itervalues(self.flows):\n      if ((client_id is None or flow.client_id == client_id) and\n          (min_create_time is None or flow.create_time >= min_create_time) and\n          (max_create_time is None or flow.create_time <= max_create_time) and\n          (include_child_flows or not flow.parent_flow_id)):\n        res.append(flow.Copy())\n    return res"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef ReadChildFlowObjects(self, client_id, flow_id):\n    res = []\n    for flow in itervalues(self.flows):\n      if flow.client_id == client_id and flow.parent_flow_id == flow_id:\n        res.append(flow)\n    return res", "response": "Reads child flows that were started by a given flow."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef LeaseFlowForProcessing(self, client_id, flow_id, processing_time):\n    rdf_flow = self.ReadFlowObject(client_id, flow_id)\n    # TODO(user): remove the check for a legacy hunt prefix as soon as\n    # AFF4 is gone.\n    if rdf_flow.parent_hunt_id and not rdf_flow.parent_hunt_id.startswith(\"H:\"):\n      rdf_hunt = self.ReadHuntObject(rdf_flow.parent_hunt_id)\n      if not rdf_hunt_objects.IsHuntSuitableForFlowProcessing(\n          rdf_hunt.hunt_state):\n        raise db.ParentHuntIsNotRunningError(client_id, flow_id,\n                                             rdf_hunt.hunt_id,\n                                             rdf_hunt.hunt_state)\n\n    now = rdfvalue.RDFDatetime.Now()\n    if rdf_flow.processing_on and rdf_flow.processing_deadline > now:\n      raise ValueError(\"Flow %s on client %s is already being processed.\" %\n                       (client_id, flow_id))\n    processing_deadline = now + processing_time\n    process_id_string = utils.ProcessIdString()\n    self.UpdateFlow(\n        client_id,\n        flow_id,\n        processing_on=process_id_string,\n        processing_since=now,\n        processing_deadline=processing_deadline)\n    rdf_flow.processing_on = process_id_string\n    rdf_flow.processing_since = now\n    rdf_flow.processing_deadline = processing_deadline\n    return rdf_flow", "response": "Leases a flow for processing."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef UpdateFlow(self,\n                 client_id,\n                 flow_id,\n                 flow_obj=db.Database.unchanged,\n                 flow_state=db.Database.unchanged,\n                 client_crash_info=db.Database.unchanged,\n                 pending_termination=db.Database.unchanged,\n                 processing_on=db.Database.unchanged,\n                 processing_since=db.Database.unchanged,\n                 processing_deadline=db.Database.unchanged):\n    \"\"\"Updates flow objects in the database.\"\"\"\n\n    try:\n      flow = self.flows[(client_id, flow_id)]\n    except KeyError:\n      raise db.UnknownFlowError(client_id, flow_id)\n\n    if flow_obj != db.Database.unchanged:\n      self.flows[(client_id, flow_id)] = flow_obj\n      flow = flow_obj\n\n    if flow_state != db.Database.unchanged:\n      flow.flow_state = flow_state\n    if client_crash_info != db.Database.unchanged:\n      flow.client_crash_info = client_crash_info\n    if pending_termination != db.Database.unchanged:\n      flow.pending_termination = pending_termination\n    if processing_on != db.Database.unchanged:\n      flow.processing_on = processing_on\n    if processing_since != db.Database.unchanged:\n      flow.processing_since = processing_since\n    if processing_deadline != db.Database.unchanged:\n      flow.processing_deadline = processing_deadline\n    flow.last_update_time = rdfvalue.RDFDatetime.Now()", "response": "Updates the flow with the given client_id and flow_id."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef UpdateFlows(self,\n                  client_id_flow_id_pairs,\n                  pending_termination=db.Database.unchanged):\n    \"\"\"Updates flow objects in the database.\"\"\"\n    for client_id, flow_id in client_id_flow_id_pairs:\n      try:\n        self.UpdateFlow(\n            client_id, flow_id, pending_termination=pending_termination)\n      except db.UnknownFlowError:\n        pass", "response": "Updates the flow objects in the database."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nwriting a list of flow requests to the database.", "response": "def WriteFlowRequests(self, requests):\n    \"\"\"Writes a list of flow requests to the database.\"\"\"\n    flow_processing_requests = []\n\n    for request in requests:\n      if (request.client_id, request.flow_id) not in self.flows:\n        raise db.AtLeastOneUnknownFlowError([(request.client_id,\n                                              request.flow_id)])\n\n    for request in requests:\n      key = (request.client_id, request.flow_id)\n      request_dict = self.flow_requests.setdefault(key, {})\n      request_dict[request.request_id] = request.Copy()\n      request_dict[request.request_id].timestamp = rdfvalue.RDFDatetime.Now()\n\n      if request.needs_processing:\n        flow = self.flows[(request.client_id, request.flow_id)]\n        if flow.next_request_to_process == request.request_id:\n          flow_processing_requests.append(\n              rdf_flows.FlowProcessingRequest(\n                  client_id=request.client_id,\n                  flow_id=request.flow_id,\n                  delivery_time=request.start_time))\n\n    if flow_processing_requests:\n      self.WriteFlowProcessingRequests(flow_processing_requests)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef DeleteFlowRequests(self, requests):\n    for request in requests:\n      if (request.client_id, request.flow_id) not in self.flows:\n        raise db.UnknownFlowError(request.client_id, request.flow_id)\n\n    for request in requests:\n      key = (request.client_id, request.flow_id)\n      request_dict = self.flow_requests.get(key, {})\n      try:\n        del request_dict[request.request_id]\n      except KeyError:\n        raise db.UnknownFlowRequestError(request.client_id, request.flow_id,\n                                         request.request_id)\n\n      response_dict = self.flow_responses.get(key, {})\n      try:\n        del response_dict[request.request_id]\n      except KeyError:\n        pass", "response": "Deletes a list of flow requests from the database."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef WriteFlowResponses(self, responses):\n    status_available = set()\n    requests_updated = set()\n    task_ids_by_request = {}\n\n    for response in responses:\n      flow_key = (response.client_id, response.flow_id)\n      if flow_key not in self.flows:\n        logging.error(\"Received response for unknown flow %s, %s.\",\n                      response.client_id, response.flow_id)\n        continue\n\n      request_dict = self.flow_requests.get(flow_key, {})\n      if response.request_id not in request_dict:\n        logging.error(\"Received response for unknown request %s, %s, %d.\",\n                      response.client_id, response.flow_id, response.request_id)\n        continue\n\n      response_dict = self.flow_responses.setdefault(flow_key, {})\n      clone = response.Copy()\n      clone.timestamp = rdfvalue.RDFDatetime.Now()\n\n      response_dict.setdefault(response.request_id,\n                               {})[response.response_id] = clone\n\n      if isinstance(response, rdf_flow_objects.FlowStatus):\n        status_available.add(response)\n\n      request_key = (response.client_id, response.flow_id, response.request_id)\n      requests_updated.add(request_key)\n      try:\n        task_ids_by_request[request_key] = response.task_id\n      except AttributeError:\n        pass\n\n    # Every time we get a status we store how many responses are expected.\n    for status in status_available:\n      request_dict = self.flow_requests[(status.client_id, status.flow_id)]\n      request = request_dict[status.request_id]\n      request.nr_responses_expected = status.response_id\n\n    # And we check for all updated requests if we need to process them.\n    needs_processing = []\n    for client_id, flow_id, request_id in requests_updated:\n      flow_key = (client_id, flow_id)\n      request_dict = self.flow_requests[flow_key]\n      request = request_dict[request_id]\n      if request.nr_responses_expected and not request.needs_processing:\n        response_dict = self.flow_responses.setdefault(flow_key, {})\n        responses = response_dict.get(request_id, {})\n\n        if len(responses) == request.nr_responses_expected:\n          request.needs_processing = True\n          self._DeleteClientActionRequest(client_id, flow_id, request_id)\n\n          flow = self.flows[flow_key]\n          if flow.next_request_to_process == request_id:\n            needs_processing.append(\n                rdf_flows.FlowProcessingRequest(\n                    client_id=client_id, flow_id=flow_id))\n\n    if needs_processing:\n      self.WriteFlowProcessingRequests(needs_processing)", "response": "Writes FlowMessages and updates corresponding requests."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef ReadAllFlowRequestsAndResponses(self, client_id, flow_id):\n    flow_key = (client_id, flow_id)\n    try:\n      self.flows[flow_key]\n    except KeyError:\n      return []\n\n    request_dict = self.flow_requests.get(flow_key, {})\n    response_dict = self.flow_responses.get(flow_key, {})\n\n    res = []\n    for request_id in sorted(request_dict):\n      res.append((request_dict[request_id], response_dict.get(request_id, {})))\n\n    return res", "response": "Reads all requests and responses for a given flow from the database."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef DeleteAllFlowRequestsAndResponses(self, client_id, flow_id):\n    flow_key = (client_id, flow_id)\n    try:\n      self.flows[flow_key]\n    except KeyError:\n      raise db.UnknownFlowError(client_id, flow_id)\n\n    try:\n      del self.flow_requests[flow_key]\n    except KeyError:\n      pass\n\n    try:\n      del self.flow_responses[flow_key]\n    except KeyError:\n      pass", "response": "Deletes all requests and responses for a given flow from the database."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreading all requests for a flow that can be processed by the worker.", "response": "def ReadFlowRequestsReadyForProcessing(self,\n                                         client_id,\n                                         flow_id,\n                                         next_needed_request=None):\n    \"\"\"Reads all requests for a flow that can be processed by the worker.\"\"\"\n    request_dict = self.flow_requests.get((client_id, flow_id), {})\n    response_dict = self.flow_responses.get((client_id, flow_id), {})\n\n    res = {}\n    for request_id in sorted(request_dict):\n      # Ignore outdated requests.\n      if request_id < next_needed_request:\n        continue\n      # The request we are currently looking for is not in yet, we are done.\n      if request_id != next_needed_request:\n        break\n      request = request_dict[request_id]\n      if not request.needs_processing:\n        break\n\n      responses = sorted(\n          itervalues(response_dict.get(request_id, {})),\n          key=lambda response: response.response_id)\n      # Serialize/deserialize responses to better simulate the\n      # real DB behavior (where serialization/deserialization is almost\n      # guaranteed to be done).\n      # TODO(user): change mem-db implementation to do\n      # serialization/deserialization everywhere in a generic way.\n      responses = [\n          r.__class__.FromSerializedString(r.SerializeToString())\n          for r in responses\n      ]\n      res[request_id] = (request, responses)\n      next_needed_request += 1\n\n    return res"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nrelease a flow that the worker was processing.", "response": "def ReleaseProcessedFlow(self, flow_obj):\n    \"\"\"Releases a flow that the worker was processing to the database.\"\"\"\n    key = (flow_obj.client_id, flow_obj.flow_id)\n    next_id_to_process = flow_obj.next_request_to_process\n    request_dict = self.flow_requests.get(key, {})\n    if (next_id_to_process in request_dict and\n        request_dict[next_id_to_process].needs_processing):\n      return False\n\n    self.UpdateFlow(\n        flow_obj.client_id,\n        flow_obj.flow_id,\n        flow_obj=flow_obj,\n        processing_on=None,\n        processing_since=None,\n        processing_deadline=None)\n    return True"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nwrite a list of flow processing requests to the database.", "response": "def WriteFlowProcessingRequests(self, requests):\n    \"\"\"Writes a list of flow processing requests to the database.\"\"\"\n    # If we don't have a handler thread running, we might be able to process the\n    # requests inline. If we are not, we start the handler thread for real and\n    # queue the requests normally.\n    if not self.flow_handler_thread and self.flow_handler_target:\n      if self._InlineProcessingOK(requests):\n        for r in requests:\n          self.flow_handler_target(r)\n        return\n      else:\n        self._RegisterFlowProcessingHandler(self.flow_handler_target)\n        self.flow_handler_target = None\n\n    now = rdfvalue.RDFDatetime.Now()\n    for r in requests:\n      cloned_request = r.Copy()\n      cloned_request.timestamp = now\n      key = (r.client_id, r.flow_id)\n      self.flow_processing_requests[key] = cloned_request"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nacknowledging flow processing requests from the database.", "response": "def AckFlowProcessingRequests(self, requests):\n    \"\"\"Deletes a list of flow processing requests from the database.\"\"\"\n    for r in requests:\n      key = (r.client_id, r.flow_id)\n      if key in self.flow_processing_requests:\n        del self.flow_processing_requests[key]"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nregisters a handler to receive flow processing messages.", "response": "def _RegisterFlowProcessingHandler(self, handler):\n    \"\"\"Registers a handler to receive flow processing messages.\"\"\"\n    self.flow_handler_stop = False\n    self.flow_handler_thread = threading.Thread(\n        name=\"flow_processing_handler\",\n        target=self._HandleFlowProcessingRequestLoop,\n        args=(handler,))\n    self.flow_handler_thread.daemon = True\n    self.flow_handler_thread.start()"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef UnregisterFlowProcessingHandler(self, timeout=None):\n    self.flow_handler_target = None\n\n    if self.flow_handler_thread:\n      self.flow_handler_stop = True\n      self.flow_handler_thread.join(timeout)\n      if self.flow_handler_thread.isAlive():\n        raise RuntimeError(\"Flow processing handler did not join in time.\")\n      self.flow_handler_thread = None", "response": "Unregisters any registered flow processing handler."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef WaitUntilNoFlowsToProcess(self, timeout=None):\n    t = self.flow_handler_thread\n    if not t:\n      return\n\n    start_time = time.time()\n    while True:\n      with self.lock:\n        # If the thread is dead, or there are no requests\n        # to be processed/being processed, we stop waiting\n        # and return from the function.\n        if (not t.isAlive() or\n            (not self._GetFlowRequestsReadyForProcessing() and\n             not self.flow_handler_num_being_processed)):\n          return\n\n      time.sleep(0.2)\n\n      if timeout and time.time() - start_time > timeout:\n        raise TimeOutWhileWaitingForFlowsToBeProcessedError(\n            \"Flow processing didn't finish in time.\")", "response": "Waits until all flows in the sequence are done processing flows."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _HandleFlowProcessingRequestLoop(self, handler):\n    while not self.flow_handler_stop:\n      with self.lock:\n        todo = self._GetFlowRequestsReadyForProcessing()\n        for request in todo:\n          self.flow_handler_num_being_processed += 1\n          del self.flow_processing_requests[(request.client_id,\n                                             request.flow_id)]\n\n      for request in todo:\n        handler(request)\n        with self.lock:\n          self.flow_handler_num_being_processed -= 1\n\n      time.sleep(0.2)", "response": "Handler thread for the FlowProcessingRequest queue."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef WriteFlowResults(self, results):\n    for r in results:\n      dest = self.flow_results.setdefault((r.client_id, r.flow_id), [])\n      to_write = r.Copy()\n      to_write.timestamp = rdfvalue.RDFDatetime.Now()\n      dest.append(to_write)", "response": "Writes results for a given flow."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef ReadFlowResults(self,\n                      client_id,\n                      flow_id,\n                      offset,\n                      count,\n                      with_tag=None,\n                      with_type=None,\n                      with_substring=None):\n    \"\"\"Reads flow results of a given flow using given query options.\"\"\"\n    results = sorted(\n        [x.Copy() for x in self.flow_results.get((client_id, flow_id), [])],\n        key=lambda r: r.timestamp)\n\n    # This is done in order to pass the tests that try to deserialize\n    # value of an unrecognized type.\n    for r in results:\n      cls_name = compatibility.GetName(r.payload.__class__)\n      if cls_name not in rdfvalue.RDFValue.classes:\n        r.payload = rdf_objects.SerializedValueOfUnrecognizedType(\n            type_name=cls_name, value=r.payload.SerializeToString())\n\n    if with_tag is not None:\n      results = [i for i in results if i.tag == with_tag]\n\n    if with_type is not None:\n      results = [\n          i for i in results\n          if compatibility.GetName(i.payload.__class__) == with_type\n      ]\n\n    if with_substring is not None:\n      encoded_substring = with_substring.encode(\"utf8\")\n      results = [\n          i for i in results\n          if encoded_substring in i.payload.SerializeToString()\n      ]\n\n    return results[offset:offset + count]", "response": "Reads the flow results of a given flow using given query options."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncount the number of flow results of a given client and flow.", "response": "def CountFlowResults(self, client_id, flow_id, with_tag=None, with_type=None):\n    \"\"\"Counts flow results of a given flow using given query options.\"\"\"\n    return len(\n        self.ReadFlowResults(\n            client_id,\n            flow_id,\n            0,\n            sys.maxsize,\n            with_tag=with_tag,\n            with_type=with_type))"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns counts of flow results grouped by result type.", "response": "def CountFlowResultsByType(self, client_id, flow_id):\n    \"\"\"Returns counts of flow results grouped by result type.\"\"\"\n    result = collections.Counter()\n    for hr in self.ReadFlowResults(client_id, flow_id, 0, sys.maxsize):\n      key = compatibility.GetName(hr.payload.__class__)\n      result[key] += 1\n\n    return result"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nwriting log entries for a given flow.", "response": "def WriteFlowLogEntries(self, entries):\n    \"\"\"Writes flow output plugin log entries for a given flow.\"\"\"\n    flow_ids = [(e.client_id, e.flow_id) for e in entries]\n    for f in flow_ids:\n      if f not in self.flows:\n        raise db.AtLeastOneUnknownFlowError(flow_ids)\n\n    for e in entries:\n      dest = self.flow_log_entries.setdefault((e.client_id, e.flow_id), [])\n      to_write = e.Copy()\n      to_write.timestamp = rdfvalue.RDFDatetime.Now()\n      dest.append(to_write)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreading flow log entries of a given flow.", "response": "def ReadFlowLogEntries(self,\n                         client_id,\n                         flow_id,\n                         offset,\n                         count,\n                         with_substring=None):\n    \"\"\"Reads flow log entries of a given flow using given query options.\"\"\"\n    entries = sorted(\n        self.flow_log_entries.get((client_id, flow_id), []),\n        key=lambda e: e.timestamp)\n\n    if with_substring is not None:\n      entries = [i for i in entries if with_substring in i.message]\n\n    return entries[offset:offset + count]"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef CountFlowLogEntries(self, client_id, flow_id):\n    return len(self.ReadFlowLogEntries(client_id, flow_id, 0, sys.maxsize))", "response": "Returns the number of flow log entries of a given flow."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nread flow output plugin log entries.", "response": "def ReadFlowOutputPluginLogEntries(self,\n                                     client_id,\n                                     flow_id,\n                                     output_plugin_id,\n                                     offset,\n                                     count,\n                                     with_type=None):\n    \"\"\"Reads flow output plugin log entries.\"\"\"\n    entries = sorted(\n        self.flow_output_plugin_log_entries.get((client_id, flow_id), []),\n        key=lambda e: e.timestamp)\n\n    entries = [e for e in entries if e.output_plugin_id == output_plugin_id]\n\n    if with_type is not None:\n      entries = [e for e in entries if e.log_entry_type == with_type]\n\n    return entries[offset:offset + count]"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef CountFlowOutputPluginLogEntries(self,\n                                      client_id,\n                                      flow_id,\n                                      output_plugin_id,\n                                      with_type=None):\n    \"\"\"Returns number of flow output plugin log entries of a given flow.\"\"\"\n\n    return len(\n        self.ReadFlowOutputPluginLogEntries(\n            client_id,\n            flow_id,\n            output_plugin_id,\n            0,\n            sys.maxsize,\n            with_type=with_type))", "response": "Returns the number of flow output plugin log entries of a given flow."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nwrite signed binary references.", "response": "def WriteSignedBinaryReferences(self, binary_id,\n                                  references):\n    \"\"\"See db.Database.\"\"\"\n    self.signed_binary_references[_SignedBinaryKeyFromID(binary_id)] = (\n        references.Copy(), rdfvalue.RDFDatetime.Now())"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nread the signed binary references.", "response": "def ReadSignedBinaryReferences(\n      self, binary_id\n  ):\n    \"\"\"See db.Database.\"\"\"\n    binary_key = _SignedBinaryKeyFromID(binary_id)\n    try:\n      references, timestamp = self.signed_binary_references[binary_key]\n    except KeyError:\n      raise db.UnknownSignedBinaryError(binary_id)\n    return references.Copy(), timestamp.Copy()"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _GetSqliteSchema(self, proto_struct_class, prefix=\"\"):\n    schema = collections.OrderedDict()\n    for type_info in proto_struct_class.type_infos:\n      if type_info.__class__ is rdf_structs.ProtoEmbedded:\n        schema.update(\n            self._GetSqliteSchema(\n                type_info.type, prefix=\"%s%s.\" % (prefix, type_info.name)))\n      else:\n        field_name = utils.SmartStr(prefix + type_info.name)\n        schema[field_name] = Rdf2SqliteAdapter.GetConverter(type_info)\n    return schema", "response": "Returns a mapping of SQLite column names to Converter objects."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _ConvertToCanonicalSqlDict(self, schema, raw_dict, prefix=\"\"):\n    flattened_dict = {}\n    for k, v in iteritems(raw_dict):\n      if isinstance(v, dict):\n        flattened_dict.update(\n            self._ConvertToCanonicalSqlDict(\n                schema, v, prefix=\"%s%s.\" % (prefix, k)))\n      else:\n        field_name = prefix + k\n        flattened_dict[field_name] = schema[field_name].convert_fn(v)\n    return flattened_dict", "response": "Converts a dict of RDF values into a SQL - ready form."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _FlushAllRows(self, db_connection, table_name):\n    for sql in db_connection.iterdump():\n      if (sql.startswith(\"CREATE TABLE\") or\n          sql.startswith(\"BEGIN TRANSACTION\") or sql.startswith(\"COMMIT\")):\n        # These statements only need to be written once.\n        continue\n      # The archive generator expects strings (not Unicode objects returned by\n      # the pysqlite library).\n      yield self.archive_generator.WriteFileChunk((sql + \"\\n\").encode(\"utf-8\"))\n    with db_connection:\n      db_connection.cursor().execute(\"DELETE FROM \\\"%s\\\";\" % table_name)", "response": "Copies all rows from the given db into the output file then deletes them."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ndump the given aff4object into a yaml representation.", "response": "def YamlDumper(aff4object):\n  \"\"\"Dumps the given aff4object into a yaml representation.\"\"\"\n  aff4object.Flush()\n\n  result = {}\n  for attribute, values in iteritems(aff4object.synced_attributes):\n    result[attribute.predicate] = []\n    for value in values:\n      # This value is really a LazyDecoder() instance. We need to get at the\n      # real data here.\n      value = value.ToRDFValue()\n\n      result[attribute.predicate].append(\n          [value.__class__.__name__,\n           value.SerializeToString(),\n           str(value.age)])\n\n  return yaml.Dump({\n      \"aff4_class\": compatibility.GetName(aff4object),\n      \"_urn\": aff4object.urn.SerializeToString(),\n      \"attributes\": result,\n      \"age_policy\": aff4object.age_policy,\n  })"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef YamlLoader(string):\n  representation = yaml.Parse(string)\n  result_cls = aff4.FACTORY.AFF4Object(representation[\"aff4_class\"])\n  aff4_attributes = {}\n  for predicate, values in iteritems(representation[\"attributes\"]):\n    attribute = aff4.Attribute.PREDICATES[predicate]\n    tmp = aff4_attributes[attribute] = []\n\n    for rdfvalue_cls_name, value, age in values:\n      rdfvalue_cls = aff4.FACTORY.RDFValue(rdfvalue_cls_name)\n      value = rdfvalue_cls(value, age=rdfvalue.RDFDatetime(age))\n      tmp.append(value)\n\n  # Ensure the object is dirty so when we save it, it can be written to the data\n  # store.\n  result = result_cls(\n      urn=representation[\"_urn\"],\n      clone=aff4_attributes,\n      mode=\"rw\",\n      age=representation[\"age_policy\"])\n\n  result.new_attributes, result.synced_attributes = result.synced_attributes, {}\n\n  result._dirty = True  # pylint: disable=protected-access\n\n  return result", "response": "Load an AFF4 object from a serialized YAML representation."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef GetMetadata(client_id, client_full_info):\n\n  metadata = ExportedMetadata()\n\n  last_snapshot = None\n  if client_full_info.HasField(\"last_snapshot\"):\n    last_snapshot = client_full_info.last_snapshot\n\n  metadata.client_urn = client_id\n  metadata.client_age = client_full_info.metadata.first_seen\n\n  if last_snapshot is not None:\n    kb = client_full_info.last_snapshot.knowledge_base\n\n    metadata.hostname = kb.fqdn\n    metadata.os = kb.os\n    metadata.uname = last_snapshot.Uname()\n    metadata.os_release = last_snapshot.os_release\n    metadata.os_version = last_snapshot.os_version\n    metadata.usernames = \",\".join(user.username for user in kb.users)\n\n    addresses = last_snapshot.GetMacAddresses()\n    if addresses:\n      metadata.mac_address = last_snapshot.GetMacAddresses().pop()\n    metadata.hardware_info = last_snapshot.hardware_info\n    metadata.kernel_version = last_snapshot.kernel\n\n    ci = last_snapshot.cloud_instance\n    if ci is not None:\n      if ci.cloud_type == ci.InstanceType.AMAZON:\n        metadata.cloud_instance_type = metadata.CloudInstanceType.AMAZON\n        metadata.cloud_instance_id = ci.amazon.instance_id\n      elif ci.cloud_type == ci.InstanceType.GOOGLE:\n        metadata.cloud_instance_type = metadata.CloudInstanceType.GOOGLE\n        metadata.cloud_instance_id = ci.google.unique_id\n\n  system_labels = set()\n  user_labels = set()\n  for l in client_full_info.labels:\n    if l.owner == \"GRR\":\n      system_labels.add(l.name)\n    else:\n      user_labels.add(l.name)\n\n  metadata.labels = \",\".join(sorted(system_labels | user_labels))\n  metadata.system_labels = \",\".join(sorted(system_labels))\n  metadata.user_labels = \",\".join(sorted(user_labels))\n\n  return metadata", "response": "Builds ExportedMetadata object for a given client id and ClientFullInfo object."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef GetMetadataLegacy(client, token=None):\n  if isinstance(client, rdfvalue.RDFURN):\n    client_fd = aff4.FACTORY.Open(client, mode=\"r\", token=token)\n  else:\n    client_fd = client\n\n  metadata = ExportedMetadata()\n\n  metadata.client_urn = client_fd.urn\n  metadata.client_age = client_fd.urn.age\n\n  metadata.hostname = utils.SmartUnicode(\n      client_fd.Get(client_fd.Schema.HOSTNAME, \"\"))\n\n  metadata.os = utils.SmartUnicode(client_fd.Get(client_fd.Schema.SYSTEM, \"\"))\n\n  metadata.uname = utils.SmartUnicode(client_fd.Get(client_fd.Schema.UNAME, \"\"))\n\n  metadata.os_release = utils.SmartUnicode(\n      client_fd.Get(client_fd.Schema.OS_RELEASE, \"\"))\n\n  metadata.os_version = utils.SmartUnicode(\n      client_fd.Get(client_fd.Schema.OS_VERSION, \"\"))\n\n  kb = client_fd.Get(client_fd.Schema.KNOWLEDGE_BASE)\n  usernames = \"\"\n  if kb:\n    usernames = [user.username for user in kb.users] or \"\"\n  metadata.usernames = utils.SmartUnicode(usernames)\n\n  metadata.mac_address = utils.SmartUnicode(\n      client_fd.Get(client_fd.Schema.MAC_ADDRESS, \"\"))\n\n  system_labels = set()\n  user_labels = set()\n  for l in client_fd.GetLabels():\n    if l.owner == \"GRR\":\n      system_labels.add(l.name)\n    else:\n      user_labels.add(l.name)\n\n  metadata.labels = \",\".join(sorted(system_labels | user_labels))\n\n  metadata.system_labels = \",\".join(sorted(system_labels))\n\n  metadata.user_labels = \",\".join(sorted(user_labels))\n\n  metadata.hardware_info = client_fd.Get(client_fd.Schema.HARDWARE_INFO)\n\n  metadata.kernel_version = client_fd.Get(client_fd.Schema.KERNEL)\n\n  return metadata", "response": "Builds an ExportedMetadata object for a given client id."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nconvert a set of RDFValues into a set of export - friendly RDFValues.", "response": "def ConvertValuesWithMetadata(metadata_value_pairs, token=None, options=None):\n  \"\"\"Converts a set of RDFValues into a set of export-friendly RDFValues.\n\n  Args:\n    metadata_value_pairs: Tuples of (metadata, rdf_value), where metadata is an\n      instance of ExportedMetadata and rdf_value is an RDFValue subclass\n      instance to be exported.\n    token: Security token.\n    options: rdfvalue.ExportOptions instance that will be passed to\n      ExportConverters.\n\n  Yields:\n    Converted values. Converted values may be of different types.\n\n  Raises:\n    NoConverterFound: in case no suitable converters were found for a value in\n                      metadata_value_pairs. This error is only raised after\n                      all values in metadata_value_pairs are attempted to be\n                      converted. If there are multiple value types that could\n                      not be converted because of the lack of corresponding\n                      converters, only the last one will be specified in the\n                      exception message.\n  \"\"\"\n  no_converter_found_error = None\n  for metadata_values_group in itervalues(\n      collection.Group(\n          metadata_value_pairs, lambda pair: pair[1].__class__.__name__)):\n\n    _, first_value = metadata_values_group[0]\n    converters_classes = ExportConverter.GetConvertersByValue(first_value)\n    if not converters_classes:\n      no_converter_found_error = \"No converters found for value: %s\" % str(\n          first_value)\n      continue\n\n    converters = [cls(options) for cls in converters_classes]\n    for converter in converters:\n      for result in converter.BatchConvert(metadata_values_group, token=token):\n        yield result\n\n  if no_converter_found_error is not None:\n    raise NoConverterFound(no_converter_found_error)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef ConvertValues(default_metadata, values, token=None, options=None):\n  batch_data = [(default_metadata, obj) for obj in values]\n  return ConvertValuesWithMetadata(batch_data, token=token, options=options)", "response": "Converts a set of RDFValues into a set of export - friendly RDFValues."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef GetConvertersByClass(value_cls):\n    try:\n      return ExportConverter.converters_cache[value_cls]\n    except KeyError:\n      results = [\n          cls for cls in itervalues(ExportConverter.classes)\n          if cls.input_rdf_type == value_cls\n      ]\n      if not results:\n        results = [DataAgnosticExportConverter]\n\n      ExportConverter.converters_cache[value_cls] = results\n      return results", "response": "Returns all converters that take given value as an input value."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ngenerating a flattened RDFValue class definition for the given value.", "response": "def MakeFlatRDFClass(self, value):\n    \"\"\"Generates flattened RDFValue class definition for the given value.\"\"\"\n\n    def Flatten(self, metadata, value_to_flatten):\n      if metadata:\n        self.metadata = metadata\n\n      for desc in value_to_flatten.type_infos:\n        if desc.name == \"metadata\":\n          continue\n        if hasattr(self, desc.name) and value_to_flatten.HasField(desc.name):\n          setattr(self, desc.name, getattr(value_to_flatten, desc.name))\n\n    descriptors = []\n    enums = {}\n\n    # Metadata is always the first field of exported data.\n    descriptors.append(\n        rdf_structs.ProtoEmbedded(\n            name=\"metadata\", field_number=1, nested=ExportedMetadata))\n\n    for number, desc in sorted(iteritems(value.type_infos_by_field_number)):\n      # Name 'metadata' is reserved to store ExportedMetadata value.\n      if desc.name == \"metadata\":\n        logging.debug(\"Ignoring 'metadata' field in %s.\",\n                      value.__class__.__name__)\n        continue\n\n      # Copy descriptors for primivie values as-is, just make sure their\n      # field number is correct.\n      if isinstance(desc, (rdf_structs.ProtoBinary, rdf_structs.ProtoString,\n                           rdf_structs.ProtoUnsignedInteger,\n                           rdf_structs.ProtoRDFValue, rdf_structs.ProtoEnum)):\n        # Incrementing field number by 1, as 1 is always occuppied by metadata.\n        descriptors.append(desc.Copy(field_number=number + 1))\n\n      if (isinstance(desc, rdf_structs.ProtoEnum) and\n          not isinstance(desc, rdf_structs.ProtoBoolean)):\n        # Attach the enum container to the class for easy reference:\n        enums[desc.enum_name] = desc.enum_container\n\n    # Create the class as late as possible. This will modify a\n    # metaclass registry, we need to make sure there are no problems.\n    output_class = type(\n        self.ExportedClassNameForValue(value), (AutoExportedProtoStruct,),\n        dict(Flatten=Flatten))\n\n    for descriptor in descriptors:\n      output_class.AddDescriptor(descriptor)\n\n    for name, container in iteritems(enums):\n      setattr(output_class, name, container)\n\n    return output_class"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef ParseSignedData(signed_data, result):\n    try:\n      auth_data\n    except NameError:\n      # Verify_sigs is not available so we can't parse signatures. If you want\n      # this functionality, please install the verify-sigs package:\n      # https://github.com/anthrotype/verify-sigs\n      # TODO(amoser): Make verify-sigs a pip package and add a dependency.\n      return\n\n    try:\n      try:\n        auth = auth_data.AuthData(signed_data.certificate)\n      except Exception as e:  # pylint: disable=broad-except\n        # If we failed to parse the certificate, we want the user to know it.\n        result.cert_hasher_name = \"Error parsing certificate: %s\" % str(e)\n        raise\n\n      result.cert_hasher_name = auth.digest_algorithm().name\n      result.cert_program_name = str(auth.program_name)\n      result.cert_program_url = str(auth.program_url)\n      result.cert_signing_id = str(auth.signing_cert_id)\n\n      try:\n        # This fills in auth.cert_chain_head. We ignore Asn1Error because\n        # we want to extract as much data as possible, no matter if the\n        # certificate has expired or not.\n        auth.ValidateCertChains(time.gmtime())\n      except auth_data.Asn1Error:\n        pass\n      result.cert_chain_head_issuer = str(auth.cert_chain_head[2])\n\n      if auth.has_countersignature:\n        result.cert_countersignature_chain_head_issuer = str(\n            auth.counter_chain_head[2])\n\n      certs = []\n      for (issuer, serial), cert in iteritems(auth.certificates):\n        subject = cert[0][0][\"subject\"]\n        subject_dn = str(dn.DistinguishedName.TraverseRdn(subject[0]))\n        not_before = cert[0][0][\"validity\"][\"notBefore\"]\n        not_after = cert[0][0][\"validity\"][\"notAfter\"]\n        not_before_time = not_before.ToPythonEpochTime()\n        not_after_time = not_after.ToPythonEpochTime()\n        not_before_time_str = time.asctime(time.gmtime(not_before_time))\n        not_after_time_str = time.asctime(time.gmtime(not_after_time))\n\n        certs.append(\n            dict(\n                issuer=issuer,\n                serial=serial,\n                subject=subject_dn,\n                not_before_time=not_before_time_str,\n                not_after_time=not_after_time_str))\n      result.cert_certificates = str(certs)\n\n    # Verify_sigs library can basically throw all kinds of exceptions so\n    # we have to use broad except here.\n    except Exception as e:  # pylint: disable=broad-except\n      logging.error(e)", "response": "Parses the signed certificate data and updates the result rdfvalue."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef ParseFileHash(hash_obj, result):\n    if hash_obj.HasField(\"md5\"):\n      result.hash_md5 = str(hash_obj.md5)\n\n    if hash_obj.HasField(\"sha1\"):\n      result.hash_sha1 = str(hash_obj.sha1)\n\n    if hash_obj.HasField(\"sha256\"):\n      result.hash_sha256 = str(hash_obj.sha256)\n\n    if hash_obj.HasField(\"pecoff_md5\"):\n      result.pecoff_hash_md5 = str(hash_obj.pecoff_md5)\n\n    if hash_obj.HasField(\"pecoff_sha1\"):\n      result.pecoff_hash_sha1 = str(hash_obj.pecoff_sha1)\n\n    if hash_obj.HasField(\"signed_data\"):\n      StatEntryToExportedFileConverter.ParseSignedData(hash_obj.signed_data[0],\n                                                       result)", "response": "Parses a Hash rdfvalue into ExportedFile s fields."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef Convert(self, metadata, stat_entry, token=None):\n    return self.BatchConvert([(metadata, stat_entry)], token=token)", "response": "Converts a StatEntry to ExportedFile."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nfilter out registry keys to operate on files.", "response": "def _RemoveRegistryKeys(self, metadata_value_pairs):\n    \"\"\"Filter out registry keys to operate on files.\"\"\"\n    filtered_pairs = []\n    for metadata, stat_entry in metadata_value_pairs:\n      # Ignore registry keys.\n      if stat_entry.pathspec.pathtype != rdf_paths.PathSpec.PathType.REGISTRY:\n        filtered_pairs.append((metadata, stat_entry))\n\n    return filtered_pairs"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _OpenFilesForRead(self, metadata_value_pairs, token):\n    aff4_paths = [\n        result.AFF4Path(metadata.client_urn)\n        for metadata, result in metadata_value_pairs\n    ]\n    fds = aff4.FACTORY.MultiOpen(aff4_paths, mode=\"r\", token=token)\n    fds_dict = dict([(fd.urn, fd) for fd in fds])\n    return fds_dict", "response": "Open files for reading."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _ExportFileContent(self, aff4_object, result):\n    if self.options.export_files_contents:\n      try:\n        result.content = aff4_object.Read(self.MAX_CONTENT_SIZE)\n        result.content_sha256 = hashlib.sha256(result.content).hexdigest()\n      except (IOError, AttributeError) as e:\n        logging.warning(\"Can't read content of %s: %s\", aff4_object.urn, e)", "response": "Adds file content from aff4_object to result."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nconverts a batch of StatEntry values to ExportedFile values at once.", "response": "def BatchConvert(self, metadata_value_pairs, token=None):\n    \"\"\"Converts a batch of StatEntry value to ExportedFile values at once.\n\n    Args:\n      metadata_value_pairs: a list or a generator of tuples (metadata, value),\n        where metadata is ExportedMetadata to be used for conversion and value\n        is a StatEntry to be converted.\n      token: Security token:\n\n    Yields:\n      Resulting ExportedFile values. Empty list is a valid result and means that\n      conversion wasn't possible.\n    \"\"\"\n    if data_store.RelationalDBEnabled():\n      result_generator = self._BatchConvertRelational(metadata_value_pairs)\n    else:\n      result_generator = self._BatchConvertLegacy(\n          metadata_value_pairs, token=token)\n\n    for r in result_generator:\n      yield r"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nconvert a StatEntry to ExportedRegistryKey.", "response": "def Convert(self, metadata, stat_entry, token=None):\n    \"\"\"Converts StatEntry to ExportedRegistryKey.\n\n    Does nothing if StatEntry corresponds to a file and not a registry entry.\n\n    Args:\n      metadata: ExportedMetadata to be used for conversion.\n      stat_entry: StatEntry to be converted.\n      token: Security token.\n\n    Returns:\n      List or generator with resulting RDFValues. Empty list if StatEntry\n      corresponds to a file and not to a registry entry.\n    \"\"\"\n    if stat_entry.pathspec.pathtype != rdf_paths.PathSpec.PathType.REGISTRY:\n      return []\n\n    result = ExportedRegistryKey(\n        metadata=metadata,\n        urn=stat_entry.AFF4Path(metadata.client_urn),\n        last_modified=stat_entry.st_mtime)\n\n    if (stat_entry.HasField(\"registry_type\") and\n        stat_entry.HasField(\"registry_data\")):\n\n      result.type = stat_entry.registry_type\n\n      # `data` can be value of arbitrary type and we need to return `bytes`. So,\n      # if it is `bytes` we just pass it through. If it is not, we stringify it\n      # to some human-readable form and turn it to `bytes` by UTF-8 encoding.\n      data = stat_entry.registry_data.GetValue()\n      if isinstance(data, bytes):\n        result.data = data\n      else:\n        result.data = str(data).encode(\"utf-8\")\n\n    return [result]"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef Convert(self, metadata, conn, token=None):\n\n    result = ExportedNetworkConnection(\n        metadata=metadata,\n        family=conn.family,\n        type=conn.type,\n        local_address=conn.local_address,\n        remote_address=conn.remote_address,\n        state=conn.state,\n        pid=conn.pid,\n        ctime=conn.ctime)\n    return [result]", "response": "Converts a NetworkConnection to ExportedNetworkConnection."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nconvert Process to ExportedProcess.", "response": "def Convert(self, metadata, process, token=None):\n    \"\"\"Converts Process to ExportedProcess.\"\"\"\n\n    result = ExportedProcess(\n        metadata=metadata,\n        pid=process.pid,\n        ppid=process.ppid,\n        name=process.name,\n        exe=process.exe,\n        cmdline=\" \".join(process.cmdline),\n        ctime=process.ctime,\n        real_uid=process.real_uid,\n        effective_uid=process.effective_uid,\n        saved_uid=process.saved_uid,\n        real_gid=process.real_gid,\n        effective_gid=process.effective_gid,\n        saved_gid=process.saved_gid,\n        username=process.username,\n        terminal=process.terminal,\n        status=process.status,\n        nice=process.nice,\n        cwd=process.cwd,\n        num_threads=process.num_threads,\n        user_cpu_time=process.user_cpu_time,\n        system_cpu_time=process.system_cpu_time,\n        cpu_percent=process.cpu_percent,\n        rss_size=process.RSS_size,\n        vms_size=process.VMS_size,\n        memory_percent=process.memory_percent)\n    return [result]"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef Convert(self, metadata, process, token=None):\n\n    conn_converter = NetworkConnectionToExportedNetworkConnectionConverter(\n        options=self.options)\n    return conn_converter.BatchConvert(\n        [(metadata, conn) for conn in process.connections], token=token)", "response": "Converts Process to ExportedNetworkConnection."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nconvert Process to ExportedOpenFile.", "response": "def Convert(self, metadata, process, token=None):\n    \"\"\"Converts Process to ExportedOpenFile.\"\"\"\n\n    for f in process.open_files:\n      yield ExportedOpenFile(metadata=metadata, pid=process.pid, path=f)"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef Convert(self, metadata, interface, token=None):\n    ip4_addresses = []\n    ip6_addresses = []\n    for addr in interface.addresses:\n      if addr.address_type == addr.Family.INET:\n        ip4_addresses.append(addr.human_readable_address)\n      elif addr.address_type == addr.Family.INET6:\n        ip6_addresses.append(addr.human_readable_address)\n      else:\n        raise ValueError(\"Invalid address type: %s\" % addr.address_type)\n\n    result = ExportedNetworkInterface(\n        metadata=metadata,\n        ifname=interface.ifname,\n        ip4_addresses=\" \".join(ip4_addresses),\n        ip6_addresses=\" \".join(ip6_addresses))\n\n    if interface.mac_address:\n      result.mac_address = interface.mac_address.human_readable_address\n\n    yield result", "response": "Converts an Interface to ExportedNetworkInterfaces."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef Convert(self, metadata, config, token=None):\n    result = ExportedDNSClientConfiguration(\n        metadata=metadata,\n        dns_servers=\" \".join(config.dns_server),\n        dns_suffixes=\" \".join(config.dns_suffix))\n    yield result", "response": "Converts DNSClientConfiguration to ExportedDNSClientConfiguration."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef Convert(self, metadata, client_summary, token=None):\n    for interface in client_summary.interfaces:\n      yield super(ClientSummaryToExportedNetworkInterfaceConverter,\n                  self).Convert(\n                      metadata, interface, token=token).next()", "response": "Converts ClientSummary to ExportedNetworkInterfaces."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _SeparateTypes(self, metadata_value_pairs):\n    registry_pairs = []\n    file_pairs = []\n    match_pairs = []\n    for metadata, result in metadata_value_pairs:\n      if (result.stat_entry.pathspec.pathtype ==\n          rdf_paths.PathSpec.PathType.REGISTRY):\n        registry_pairs.append((metadata, result.stat_entry))\n      else:\n        file_pairs.append((metadata, result))\n\n      match_pairs.extend([(metadata, match) for match in result.matches])\n\n    return registry_pairs, file_pairs, match_pairs", "response": "Separate files registry keys grep matches."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nconverting GrrMessage into a set of RDFValues.", "response": "def Convert(self, metadata, grr_message, token=None):\n    \"\"\"Converts GrrMessage into a set of RDFValues.\n\n    Args:\n      metadata: ExportedMetadata to be used for conversion.\n      grr_message: GrrMessage to be converted.\n      token: Security token.\n\n    Returns:\n      List or generator with resulting RDFValues.\n    \"\"\"\n    return self.BatchConvert([(metadata, grr_message)], token=token)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef BatchConvert(self, metadata_value_pairs, token=None):\n\n    # Group messages by source (i.e. by client urn).\n    msg_dict = {}\n    for metadata, msg in metadata_value_pairs:\n      msg_dict.setdefault(msg.source, []).append((metadata, msg))\n\n    metadata_objects = []\n    metadata_to_fetch = []\n\n    # Open the clients we don't have metadata for and fetch metadata.\n    for client_urn in msg_dict:\n      try:\n        metadata_objects.append(self.cached_metadata[client_urn])\n      except KeyError:\n        metadata_to_fetch.append(client_urn)\n\n    if metadata_to_fetch:\n      if data_store.RelationalDBEnabled():\n        client_ids = set(urn.Basename() for urn in metadata_to_fetch)\n        infos = data_store.REL_DB.MultiReadClientFullInfo(client_ids)\n\n        fetched_metadata = [\n            GetMetadata(client_id, info) for client_id, info in infos.items()\n        ]\n      else:\n        client_fds = aff4.FACTORY.MultiOpen(\n            metadata_to_fetch, mode=\"r\", token=token)\n\n        fetched_metadata = [\n            GetMetadataLegacy(client_fd, token=token)\n            for client_fd in client_fds\n        ]\n\n      for metadata in fetched_metadata:\n        self.cached_metadata[metadata.client_urn] = metadata\n      metadata_objects.extend(fetched_metadata)\n\n    data_by_type = {}\n    for metadata in metadata_objects:\n      try:\n        for original_metadata, message in msg_dict[metadata.client_urn]:\n          # Get source_urn and annotations from the original metadata\n          # provided and original_timestamp from the payload age.\n          new_metadata = ExportedMetadata(metadata)\n          new_metadata.source_urn = original_metadata.source_urn\n          new_metadata.annotations = original_metadata.annotations\n          new_metadata.original_timestamp = message.payload.age\n          cls_name = message.payload.__class__.__name__\n\n          # Create a dict of values for conversion keyed by type, so we can\n          # apply the right converters to the right object types\n          if cls_name not in data_by_type:\n            converters_classes = ExportConverter.GetConvertersByValue(\n                message.payload)\n            data_by_type[cls_name] = {\n                \"converters\": [cls(self.options) for cls in converters_classes],\n                \"batch_data\": [(new_metadata, message.payload)]\n            }\n          else:\n            data_by_type[cls_name][\"batch_data\"].append(\n                (new_metadata, message.payload))\n\n      except KeyError:\n        pass\n\n    # Run all converters against all objects of the relevant type\n    converted_batch = []\n    for dataset in itervalues(data_by_type):\n      for converter in dataset[\"converters\"]:\n        converted_batch.extend(\n            converter.BatchConvert(dataset[\"batch_data\"], token=token))\n\n    return converted_batch", "response": "Converts a batch of GrrMessages into a set of RDFValues at once."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef BatchConvert(self, metadata_value_pairs, token=None):\n\n    urns = [urn for metadata, urn in metadata_value_pairs]\n    urns_dict = dict((urn, metadata) for metadata, urn in metadata_value_pairs)\n\n    results = []\n    for hash_urn, client_files in filestore.HashFileStore.GetClientsForHashes(\n        urns, token=token):\n      for hit in client_files:\n        metadata = ExportedMetadata(urns_dict[hash_urn])\n        metadata.client_urn = rdfvalue.RDFURN(hit).Split(2)[0]\n\n        result = ExportedFileStoreHash(\n            metadata=metadata,\n            hash=hash_urn.hash_value,\n            fingerprint_type=hash_urn.fingerprint_type,\n            hash_type=hash_urn.hash_type,\n            target_urn=hit)\n        results.append(result)\n\n    return results", "response": "Convert batch of FileStoreHashs."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef Convert(self, metadata, checkresult, token=None):\n    if checkresult.HasField(\"anomaly\"):\n      for anomaly in checkresult.anomaly:\n        exported_anomaly = ExportedAnomaly(\n            type=anomaly.type,\n            severity=anomaly.severity,\n            confidence=anomaly.confidence)\n        if anomaly.symptom:\n          exported_anomaly.symptom = anomaly.symptom\n        if anomaly.explanation:\n          exported_anomaly.explanation = anomaly.explanation\n        if anomaly.generated_by:\n          exported_anomaly.generated_by = anomaly.generated_by\n        if anomaly.anomaly_reference_id:\n          exported_anomaly.anomaly_reference_id = \"\\n\".join(\n              anomaly.anomaly_reference_id)\n        if anomaly.finding:\n          exported_anomaly.finding = \"\\n\".join(anomaly.finding)\n        yield ExportedCheckResult(\n            metadata=metadata,\n            check_id=checkresult.check_id,\n            anomaly=exported_anomaly)\n    else:\n      yield ExportedCheckResult(\n          metadata=metadata, check_id=checkresult.check_id)", "response": "Converts a single CheckResult into a list of ExportedCheckResult objects."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nconverting original result via given converter.", "response": "def GetExportedResult(self,\n                        original_result,\n                        converter,\n                        metadata=None,\n                        token=None):\n    \"\"\"Converts original result via given converter..\"\"\"\n\n    exported_results = list(\n        converter.Convert(\n            metadata or ExportedMetadata(), original_result, token=token))\n\n    if not exported_results:\n      raise ExportError(\"Got 0 exported result when a single one \"\n                        \"was expected.\")\n\n    if len(exported_results) > 1:\n      raise ExportError(\"Got > 1 exported results when a single \"\n                        \"one was expected, seems like a logical bug.\")\n\n    return exported_results[0]"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nchecks if given RDFValue is a registry StatEntry.", "response": "def IsRegistryStatEntry(self, original_result):\n    \"\"\"Checks if given RDFValue is a registry StatEntry.\"\"\"\n    return (original_result.pathspec.pathtype ==\n            rdf_paths.PathSpec.PathType.REGISTRY)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncheck if given RDFValue is a file StatEntry.", "response": "def IsFileStatEntry(self, original_result):\n    \"\"\"Checks if given RDFValue is a file StatEntry.\"\"\"\n    return (original_result.pathspec.pathtype in [\n        rdf_paths.PathSpec.PathType.OS, rdf_paths.PathSpec.PathType.TSK\n    ])"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef Convert(self, metadata, value, token=None):\n\n    for r in self.BatchConvert([(metadata, value)], token=token):\n      yield r", "response": "Converts a single ArtifactFilesDownloaderResult."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nconvert a single YaraProcessScanMatch.", "response": "def Convert(self, metadata, yara_match, token=None):\n    \"\"\"Convert a single YaraProcessScanMatch.\"\"\"\n\n    conv = ProcessToExportedProcessConverter(options=self.options)\n    process = list(\n        conv.Convert(ExportedMetadata(), yara_match.process, token=token))[0]\n\n    seen_rules = set()\n    for m in yara_match.match:\n      if m.rule_name in seen_rules:\n        continue\n\n      seen_rules.add(m.rule_name)\n      yield ExportedYaraProcessScanMatch(\n          metadata=metadata,\n          process=process,\n          rule_name=m.rule_name,\n          scan_time_us=yara_match.scan_time_us)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncreating a dynamic RDF proto struct class for given osquery table.", "response": "def _RDFClass(cls, table):\n    \"\"\"Creates a dynamic RDF proto struct class for given osquery table.\n\n    The fields of the proto will correspond to the columns of the table.\n\n    Args:\n      table: An osquery table for which the class is about to be generated.\n\n    Returns:\n      A class object corresponding to the given table.\n    \"\"\"\n    rdf_cls_name = \"OsqueryTable{}\".format(hash(table.query))\n    try:\n      return cls._rdf_cls_cache[rdf_cls_name]\n    except KeyError:\n      pass\n\n    rdf_cls = compatibility.MakeType(rdf_cls_name,\n                                     (rdf_structs.RDFProtoStruct,), {})\n\n    rdf_cls.AddDescriptor(\n        rdf_structs.ProtoEmbedded(\n            name=\"metadata\", field_number=1, nested=ExportedMetadata))\n\n    rdf_cls.AddDescriptor(\n        rdf_structs.ProtoString(name=\"__query__\", field_number=2))\n\n    for idx, column in enumerate(table.header.columns):\n      # It is possible that RDF column is named \"metadata\". To avoid name clash\n      # we must rename it to `__metadata__`.\n      if column.name == \"metadata\":\n        name = \"__metadata__\"\n      else:\n        name = column.name\n\n      descriptor = rdf_structs.ProtoString(name=name, field_number=idx + 3)\n      rdf_cls.AddDescriptor(descriptor)\n\n    cls._rdf_cls_cache[rdf_cls_name] = rdf_cls\n    return rdf_cls"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nextracts the path from a client action response.", "response": "def _ExtractPath(response, pathspec_attribute=None):\n  \"\"\"Returns the path from a client action response as a string.\n\n  Args:\n    response: A client action response.\n    pathspec_attribute: Specifies the field which stores the pathspec.\n\n  Returns:\n    The path as a string or None if no path is found.\n\n  \"\"\"\n  path_specification = response\n\n  if pathspec_attribute is not None:\n    if response.HasField(pathspec_attribute):\n      path_specification = response.Get(pathspec_attribute)\n\n  if path_specification.HasField(\"pathspec\"):\n    path_specification = path_specification.pathspec\n\n  if path_specification.HasField(\"path\"):\n    path_specification = path_specification.path\n\n  if isinstance(path_specification, Text):\n    return path_specification\n  return None"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning an rdf_artifacts. CollectedArtifact rdf object for the requested artifact.", "response": "def _CollectArtifact(self, artifact, apply_parsers):\n    \"\"\"Returns an `CollectedArtifact` rdf object for the requested artifact.\"\"\"\n    artifact_result = rdf_artifacts.CollectedArtifact(name=artifact.name)\n\n    if apply_parsers:\n      parser_factory = parsers.ArtifactParserFactory(str(artifact.name))\n    else:\n      parser_factory = None\n\n    for source_result_list in self._ProcessSources(artifact.sources,\n                                                   parser_factory):\n      for response in source_result_list:\n        action_result = rdf_artifacts.ClientActionResult()\n        action_result.type = response.__class__.__name__\n        action_result.value = response\n        artifact_result.action_results.append(action_result)\n        self.UpdateKnowledgeBase(response, artifact.provides)\n\n    return artifact_result"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef UpdateKnowledgeBase(self, response, provides):\n\n    if isinstance(response, rdf_anomaly.Anomaly):\n      return\n\n    if isinstance(response, rdf_client.User):\n      self.knowledge_base.MergeOrAddUser(response)\n      return\n\n    if isinstance(response, rdf_protodict.Dict):\n      response_dict = response.ToDict()\n      for attribute, value in iteritems(response_dict):\n        if attribute in provides:\n          self.SetKnowledgeBaseValue(attribute, value)\n      return\n\n    # If its not a dict we only support a single value.\n    if len(provides) == 1:\n      self.SetKnowledgeBaseValue(provides[0], response)", "response": "Updates the values in the knowledge base based on responses."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _ProcessSources(self, sources, parser_factory):\n    for source in sources:\n      for action, request in self._ParseSourceType(source):\n        yield self._RunClientAction(action, request, parser_factory,\n                                    source.path_type)", "response": "Iterates through sources yielding action responses."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nruns the client action with the request and parses the result.", "response": "def _RunClientAction(self, action, request, parser_factory, path_type):\n    \"\"\"Runs the client action  with the request and parses the result.\"\"\"\n    responses = list(action(request))\n\n    if parser_factory is None:\n      return responses\n\n    # parse the responses\n    parsed_responses = []\n\n    for response in responses:\n      for parser in parser_factory.SingleResponseParsers():\n        parsed_responses.extend(\n            parser.ParseResponse(self.knowledge_base, response, path_type))\n\n      for parser in parser_factory.SingleFileParsers():\n        precondition.AssertType(response, rdf_client_fs.StatEntry)\n        pathspec = response.pathspec\n        with vfs.VFSOpen(pathspec) as filedesc:\n          parsed_responses.extend(\n              parser.ParseFile(self.knowledge_base, pathspec, filedesc))\n\n    for parser in parser_factory.MultiResponseParsers():\n      parsed_responses.extend(\n          parser.ParseResponses(self.knowledge_base, responses))\n\n    for parser in parser_factory.MultiFileParsers():\n      precondition.AssertIterableType(responses, rdf_client_fs.StatEntry)\n      pathspecs = [response.pathspec for response in responses]\n      with vfs.VFSMultiOpen(pathspecs) as filedescs:\n        parsed_responses.extend(\n            parser.ParseFiles(self.knowledge_base, pathspecs, filedescs))\n\n    return parsed_responses"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _ParseSourceType(self, source):\n    type_name = rdf_artifacts.ArtifactSource.SourceType\n    switch = {\n        type_name.COMMAND: self._ProcessCommandSource,\n        type_name.DIRECTORY: self._ProcessFileSource,\n        type_name.FILE: self._ProcessFileSource,\n        type_name.GREP: self._ProcessGrepSource,\n        type_name.REGISTRY_KEY: self._ProcessRegistryKeySource,\n        type_name.REGISTRY_VALUE: self._ProcessRegistryValueSource,\n        type_name.WMI: self._ProcessWmiSource,\n        type_name.ARTIFACT_FILES: self._ProcessArtifactFilesSource,\n        type_name.GRR_CLIENT_ACTION: self._ProcessClientActionSource\n    }\n    source_type = source.base_source.type\n\n    try:\n      source_type_action = switch[source_type]\n    except KeyError:\n      raise ValueError(\"Incorrect source type: %s\" % source_type)\n\n    for res in source_type_action(source):\n      yield res", "response": "Parses the source type and yields the results."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _ProcessRegistryKeySource(self, source):\n    keys = source.base_source.attributes.get(\"keys\", [])\n    if not keys:\n      return\n\n    interpolated_paths = artifact_utils.InterpolateListKbAttributes(\n        input_list=keys,\n        knowledge_base=self.knowledge_base,\n        ignore_errors=self.ignore_interpolation_errors)\n\n    glob_expressions = map(rdf_paths.GlobExpression, interpolated_paths)\n\n    patterns = []\n    for pattern in glob_expressions:\n      patterns.extend(pattern.Interpolate(knowledge_base=self.knowledge_base))\n    patterns.sort(key=len, reverse=True)\n\n    file_finder_action = rdf_file_finder.FileFinderAction.Stat()\n    request = rdf_file_finder.FileFinderArgs(\n        paths=patterns,\n        action=file_finder_action,\n        follow_links=True,\n        pathtype=rdf_paths.PathSpec.PathType.REGISTRY)\n    action = vfs_file_finder.RegistryKeyFromClient\n\n    yield action, request", "response": "Process a registry key source."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _ProcessGrepSource(self, source):\n    attributes = source.base_source.attributes\n    paths = artifact_utils.InterpolateListKbAttributes(\n        attributes[\"paths\"], self.knowledge_base,\n        self.ignore_interpolation_errors)\n    regex = utils.RegexListDisjunction(attributes[\"content_regex_list\"])\n    condition = rdf_file_finder.FileFinderCondition.ContentsRegexMatch(\n        regex=regex, mode=\"ALL_HITS\")\n    file_finder_action = rdf_file_finder.FileFinderAction.Stat()\n    request = rdf_file_finder.FileFinderArgs(\n        paths=paths,\n        action=file_finder_action,\n        conditions=[condition],\n        follow_links=True)\n    action = file_finder.FileFinderOSFromClient\n\n    yield action, request", "response": "Yields a generator that yields a file finder action and request."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ngetting artifact responses extract paths and send corresponding files.", "response": "def _ProcessArtifactFilesSource(self, source):\n    \"\"\"Get artifact responses, extract paths and send corresponding files.\"\"\"\n\n    if source.path_type != rdf_paths.PathSpec.PathType.OS:\n      raise ValueError(\"Only supported path type is OS.\")\n\n    # TODO(user): Check paths for GlobExpressions.\n    # If it contains a * then FileFinder will interpret it as GlobExpression and\n    # expand it. FileFinderArgs needs an option to treat paths literally.\n\n    paths = []\n    pathspec_attribute = source.base_source.attributes.get(\"pathspec_attribute\")\n\n    for source_result_list in self._ProcessSources(\n        source.artifact_sources, parser_factory=None):\n      for response in source_result_list:\n        path = _ExtractPath(response, pathspec_attribute)\n        if path is not None:\n          paths.append(path)\n\n    file_finder_action = rdf_file_finder.FileFinderAction.Download()\n    request = rdf_file_finder.FileFinderArgs(\n        paths=paths, pathtype=source.path_type, action=file_finder_action)\n    action = file_finder.FileFinderOSFromClient\n\n    yield action, request"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nprocess a file source.", "response": "def _ProcessFileSource(self, source):\n    \"\"\"Glob paths and return StatEntry objects.\"\"\"\n\n    if source.path_type != rdf_paths.PathSpec.PathType.OS:\n      raise ValueError(\"Only supported path type is OS.\")\n\n    paths = artifact_utils.InterpolateListKbAttributes(\n        source.base_source.attributes[\"paths\"], self.knowledge_base,\n        self.ignore_interpolation_errors)\n\n    file_finder_action = rdf_file_finder.FileFinderAction.Stat()\n    request = rdf_file_finder.FileFinderArgs(\n        paths=paths, pathtype=source.path_type, action=file_finder_action)\n    action = file_finder.FileFinderOSFromClient\n\n    yield action, request"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nprocesses a command source.", "response": "def _ProcessCommandSource(self, source):\n    \"\"\"Prepare a request for calling the execute command action.\"\"\"\n    action = standard.ExecuteCommandFromClient\n    request = rdf_client_action.ExecuteRequest(\n        cmd=source.base_source.attributes[\"cmd\"],\n        args=source.base_source.attributes[\"args\"],\n    )\n    yield action, request"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nwrite metadata about a client.", "response": "def WriteClientMetadata(self,\n                          client_id,\n                          certificate=None,\n                          fleetspeak_enabled=None,\n                          first_seen=None,\n                          last_ping=None,\n                          last_clock=None,\n                          last_ip=None,\n                          last_foreman=None,\n                          cursor=None):\n    \"\"\"Write metadata about the client.\"\"\"\n    placeholders = []\n    values = collections.OrderedDict()\n\n    placeholders.append(\"%(client_id)s\")\n    values[\"client_id\"] = db_utils.ClientIDToInt(client_id)\n\n    if certificate:\n      placeholders.append(\"%(certificate)s\")\n      values[\"certificate\"] = certificate.SerializeToString()\n    if fleetspeak_enabled is not None:\n      placeholders.append(\"%(fleetspeak_enabled)s\")\n      values[\"fleetspeak_enabled\"] = fleetspeak_enabled\n    if first_seen is not None:\n      placeholders.append(\"FROM_UNIXTIME(%(first_seen)s)\")\n      values[\"first_seen\"] = mysql_utils.RDFDatetimeToTimestamp(first_seen)\n    if last_ping is not None:\n      placeholders.append(\"FROM_UNIXTIME(%(last_ping)s)\")\n      values[\"last_ping\"] = mysql_utils.RDFDatetimeToTimestamp(last_ping)\n    if last_clock:\n      placeholders.append(\"FROM_UNIXTIME(%(last_clock)s)\")\n      values[\"last_clock\"] = mysql_utils.RDFDatetimeToTimestamp(last_clock)\n    if last_ip:\n      placeholders.append(\"%(last_ip)s\")\n      values[\"last_ip\"] = last_ip.SerializeToString()\n    if last_foreman:\n      placeholders.append(\"FROM_UNIXTIME(%(last_foreman)s)\")\n      values[\"last_foreman\"] = mysql_utils.RDFDatetimeToTimestamp(last_foreman)\n\n    updates = []\n    for column in iterkeys(values):\n      updates.append(\"{column} = VALUES({column})\".format(column=column))\n\n    query = \"\"\"\n    INSERT INTO clients ({columns})\n    VALUES ({placeholders})\n    ON DUPLICATE KEY UPDATE {updates}\n    \"\"\".format(\n        columns=\", \".join(iterkeys(values)),\n        placeholders=\", \".join(placeholders),\n        updates=\", \".join(updates))\n\n    cursor.execute(query, values)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef MultiReadClientMetadata(self, client_ids, cursor=None):\n    ids = [db_utils.ClientIDToInt(client_id) for client_id in client_ids]\n    query = (\"SELECT client_id, fleetspeak_enabled, certificate, \"\n             \"UNIX_TIMESTAMP(last_ping), \"\n             \"UNIX_TIMESTAMP(last_clock), last_ip, \"\n             \"UNIX_TIMESTAMP(last_foreman), UNIX_TIMESTAMP(first_seen), \"\n             \"UNIX_TIMESTAMP(last_crash_timestamp), \"\n             \"UNIX_TIMESTAMP(last_startup_timestamp) FROM \"\n             \"clients WHERE client_id IN ({})\").format(\", \".join([\"%s\"] *\n                                                                 len(ids)))\n    ret = {}\n    cursor.execute(query, ids)\n    while True:\n      row = cursor.fetchone()\n      if not row:\n        break\n      cid, fs, crt, ping, clk, ip, foreman, first, lct, lst = row\n      ret[db_utils.IntToClientID(cid)] = rdf_objects.ClientMetadata(\n          certificate=crt,\n          fleetspeak_enabled=fs,\n          first_seen=mysql_utils.TimestampToRDFDatetime(first),\n          ping=mysql_utils.TimestampToRDFDatetime(ping),\n          clock=mysql_utils.TimestampToRDFDatetime(clk),\n          ip=mysql_utils.StringToRDFProto(rdf_client_network.NetworkAddress,\n                                          ip),\n          last_foreman_time=mysql_utils.TimestampToRDFDatetime(foreman),\n          startup_info_timestamp=mysql_utils.TimestampToRDFDatetime(lst),\n          last_crash_timestamp=mysql_utils.TimestampToRDFDatetime(lct))\n    return ret", "response": "Reads ClientMetadata records for a list of clients."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef WriteClientSnapshot(self, snapshot, cursor=None):\n    insert_history_query = (\n        \"INSERT INTO client_snapshot_history(client_id, timestamp, \"\n        \"client_snapshot) VALUES (%s, FROM_UNIXTIME(%s), %s)\")\n    insert_startup_query = (\n        \"INSERT INTO client_startup_history(client_id, timestamp, \"\n        \"startup_info) VALUES(%s, FROM_UNIXTIME(%s), %s)\")\n\n    now = rdfvalue.RDFDatetime.Now()\n\n    client_platform = snapshot.knowledge_base.os\n    current_timestamp = mysql_utils.RDFDatetimeToTimestamp(now)\n    client_info = {\n        \"last_snapshot_timestamp\": current_timestamp,\n        \"last_startup_timestamp\": current_timestamp,\n        \"last_version_string\": snapshot.GetGRRVersionString(),\n        \"last_platform_release\": snapshot.Uname(),\n    }\n    update_clauses = [\n        \"last_snapshot_timestamp = FROM_UNIXTIME(%(last_snapshot_timestamp)s)\",\n        \"last_startup_timestamp = FROM_UNIXTIME(%(last_startup_timestamp)s)\",\n        \"last_version_string = %(last_version_string)s\",\n        \"last_platform_release = %(last_platform_release)s\",\n    ]\n    if client_platform:\n      client_info[\"last_platform\"] = client_platform\n      update_clauses.append(\"last_platform = %(last_platform)s\")\n\n    update_query = (\n        \"UPDATE clients SET {} WHERE client_id = %(client_id)s\".format(\n            \", \".join(update_clauses)))\n\n    int_client_id = db_utils.ClientIDToInt(snapshot.client_id)\n    client_info[\"client_id\"] = int_client_id\n\n    startup_info = snapshot.startup_info\n    snapshot.startup_info = None\n    try:\n      cursor.execute(\n          insert_history_query,\n          (int_client_id, current_timestamp, snapshot.SerializeToString()))\n      cursor.execute(\n          insert_startup_query,\n          (int_client_id, current_timestamp, startup_info.SerializeToString()))\n      cursor.execute(update_query, client_info)\n    except MySQLdb.IntegrityError as e:\n      raise db.UnknownClientError(snapshot.client_id, cause=e)\n    finally:\n      snapshot.startup_info = startup_info", "response": "Writes a new client snapshot."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nread the latest client snapshots for a list of clients.", "response": "def MultiReadClientSnapshot(self, client_ids, cursor=None):\n    \"\"\"Reads the latest client snapshots for a list of clients.\"\"\"\n    int_ids = [db_utils.ClientIDToInt(cid) for cid in client_ids]\n    query = (\n        \"SELECT h.client_id, h.client_snapshot, UNIX_TIMESTAMP(h.timestamp),\"\n        \"       s.startup_info \"\n        \"FROM clients as c FORCE INDEX (PRIMARY), \"\n        \"client_snapshot_history as h FORCE INDEX (PRIMARY), \"\n        \"client_startup_history as s FORCE INDEX (PRIMARY) \"\n        \"WHERE h.client_id = c.client_id \"\n        \"AND s.client_id = c.client_id \"\n        \"AND h.timestamp = c.last_snapshot_timestamp \"\n        \"AND s.timestamp = c.last_startup_timestamp \"\n        \"AND c.client_id IN ({})\").format(\", \".join([\"%s\"] * len(client_ids)))\n    ret = {cid: None for cid in client_ids}\n    cursor.execute(query, int_ids)\n\n    while True:\n      row = cursor.fetchone()\n      if not row:\n        break\n      cid, snapshot, timestamp, startup_info = row\n      client_obj = mysql_utils.StringToRDFProto(rdf_objects.ClientSnapshot,\n                                                snapshot)\n      client_obj.startup_info = mysql_utils.StringToRDFProto(\n          rdf_client.StartupInfo, startup_info)\n      client_obj.timestamp = mysql_utils.TimestampToRDFDatetime(timestamp)\n      ret[db_utils.IntToClientID(cid)] = client_obj\n    return ret"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreading the full history for a particular client.", "response": "def ReadClientSnapshotHistory(self, client_id, timerange=None, cursor=None):\n    \"\"\"Reads the full history for a particular client.\"\"\"\n\n    client_id_int = db_utils.ClientIDToInt(client_id)\n\n    query = (\"SELECT sn.client_snapshot, st.startup_info, \"\n             \"       UNIX_TIMESTAMP(sn.timestamp) FROM \"\n             \"client_snapshot_history AS sn, \"\n             \"client_startup_history AS st WHERE \"\n             \"sn.client_id = st.client_id AND \"\n             \"sn.timestamp = st.timestamp AND \"\n             \"sn.client_id=%s \")\n\n    args = [client_id_int]\n    if timerange:\n      time_from, time_to = timerange  # pylint: disable=unpacking-non-sequence\n\n      if time_from is not None:\n        query += \"AND sn.timestamp >= FROM_UNIXTIME(%s) \"\n        args.append(mysql_utils.RDFDatetimeToTimestamp(time_from))\n\n      if time_to is not None:\n        query += \"AND sn.timestamp <= FROM_UNIXTIME(%s) \"\n        args.append(mysql_utils.RDFDatetimeToTimestamp(time_to))\n\n    query += \"ORDER BY sn.timestamp DESC\"\n\n    ret = []\n    cursor.execute(query, args)\n    for snapshot, startup_info, timestamp in cursor.fetchall():\n      client = rdf_objects.ClientSnapshot.FromSerializedString(snapshot)\n      client.startup_info = rdf_client.StartupInfo.FromSerializedString(\n          startup_info)\n      client.timestamp = mysql_utils.TimestampToRDFDatetime(timestamp)\n\n      ret.append(client)\n    return ret"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nwrites the full history for a particular client.", "response": "def WriteClientSnapshotHistory(self, clients, cursor=None):\n    \"\"\"Writes the full history for a particular client.\"\"\"\n    client_id = clients[0].client_id\n    latest_timestamp = max(client.timestamp for client in clients)\n\n    query = \"\"\n    params = {\n        \"client_id\": db_utils.ClientIDToInt(client_id),\n        \"latest_timestamp\": mysql_utils.RDFDatetimeToTimestamp(latest_timestamp)\n    }\n\n    for idx, client in enumerate(clients):\n      startup_info = client.startup_info\n      client.startup_info = None\n\n      query += \"\"\"\n      INSERT INTO client_snapshot_history (client_id, timestamp,\n                                           client_snapshot)\n      VALUES (%(client_id)s, FROM_UNIXTIME(%(timestamp_{idx})s),\n              %(client_snapshot_{idx})s);\n\n      INSERT INTO client_startup_history (client_id, timestamp,\n                                          startup_info)\n      VALUES (%(client_id)s, FROM_UNIXTIME(%(timestamp_{idx})s),\n              %(startup_info_{idx})s);\n      \"\"\".format(idx=idx)\n\n      params.update({\n          \"timestamp_{idx}\".format(idx=idx):\n              mysql_utils.RDFDatetimeToTimestamp(client.timestamp),\n          \"client_snapshot_{idx}\".format(idx=idx):\n              client.SerializeToString(),\n          \"startup_info_{idx}\".format(idx=idx):\n              startup_info.SerializeToString(),\n      })\n\n      client.startup_info = startup_info\n\n    query += \"\"\"\n    UPDATE clients\n       SET last_snapshot_timestamp = FROM_UNIXTIME(%(latest_timestamp)s)\n     WHERE client_id = %(client_id)s\n       AND (last_snapshot_timestamp IS NULL OR\n            last_snapshot_timestamp < FROM_UNIXTIME(%(latest_timestamp)s));\n\n    UPDATE clients\n       SET last_startup_timestamp = FROM_UNIXTIME(%(latest_timestamp)s)\n     WHERE client_id = %(client_id)s\n       AND (last_startup_timestamp IS NULL OR\n            last_startup_timestamp < FROM_UNIXTIME(%(latest_timestamp)s));\n    \"\"\"\n\n    try:\n      cursor.execute(query, params)\n    except MySQLdb.IntegrityError as error:\n      raise db.UnknownClientError(client_id, cause=error)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef WriteClientStartupInfo(self, client_id, startup_info, cursor=None):\n    query = \"\"\"\n    SET @now = NOW(6);\n\n    INSERT INTO client_startup_history (client_id, timestamp, startup_info)\n         VALUES (%(client_id)s, @now, %(startup_info)s);\n\n    UPDATE clients\n       SET last_startup_timestamp = @now\n     WHERE client_id = %(client_id)s;\n    \"\"\"\n\n    params = {\n        \"client_id\": db_utils.ClientIDToInt(client_id),\n        \"startup_info\": startup_info.SerializeToString(),\n    }\n\n    try:\n      cursor.execute(query, params)\n    except MySQLdb.IntegrityError as e:\n      raise db.UnknownClientError(client_id, cause=e)", "response": "Writes a new client startup record."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef ReadClientStartupInfo(self, client_id, cursor=None):\n    query = (\n        \"SELECT startup_info, UNIX_TIMESTAMP(timestamp) \"\n        \"FROM clients, client_startup_history \"\n        \"WHERE clients.last_startup_timestamp=client_startup_history.timestamp \"\n        \"AND clients.client_id=client_startup_history.client_id \"\n        \"AND clients.client_id=%s\")\n    cursor.execute(query, [db_utils.ClientIDToInt(client_id)])\n    row = cursor.fetchone()\n    if row is None:\n      return None\n\n    startup_info, timestamp = row\n    res = rdf_client.StartupInfo.FromSerializedString(startup_info)\n    res.timestamp = mysql_utils.TimestampToRDFDatetime(timestamp)\n    return res", "response": "Reads the latest client startup record for a single client."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreads the full startup history for a particular client.", "response": "def ReadClientStartupInfoHistory(self, client_id, timerange=None,\n                                   cursor=None):\n    \"\"\"Reads the full startup history for a particular client.\"\"\"\n\n    client_id_int = db_utils.ClientIDToInt(client_id)\n\n    query = (\"SELECT startup_info, UNIX_TIMESTAMP(timestamp) \"\n             \"FROM client_startup_history \"\n             \"WHERE client_id=%s \")\n    args = [client_id_int]\n\n    if timerange:\n      time_from, time_to = timerange  # pylint: disable=unpacking-non-sequence\n\n      if time_from is not None:\n        query += \"AND timestamp >= FROM_UNIXTIME(%s) \"\n        args.append(mysql_utils.RDFDatetimeToTimestamp(time_from))\n\n      if time_to is not None:\n        query += \"AND timestamp <= FROM_UNIXTIME(%s) \"\n        args.append(mysql_utils.RDFDatetimeToTimestamp(time_to))\n\n    query += \"ORDER BY timestamp DESC \"\n\n    ret = []\n    cursor.execute(query, args)\n\n    for startup_info, timestamp in cursor.fetchall():\n      si = rdf_client.StartupInfo.FromSerializedString(startup_info)\n      si.timestamp = mysql_utils.TimestampToRDFDatetime(timestamp)\n      ret.append(si)\n    return ret"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nconverts a database response to a list of ClientFullInfo objects.", "response": "def _ResponseToClientsFullInfo(self, response):\n    \"\"\"Creates a ClientFullInfo object from a database response.\"\"\"\n    c_full_info = None\n    prev_cid = None\n    for row in response:\n      (cid, fs, crt, ping, clk, ip, foreman, first, last_client_ts,\n       last_crash_ts, last_startup_ts, client_obj, client_startup_obj,\n       last_startup_obj, label_owner, label_name) = row\n\n      if cid != prev_cid:\n        if c_full_info:\n          yield db_utils.IntToClientID(prev_cid), c_full_info\n\n        metadata = rdf_objects.ClientMetadata(\n            certificate=crt,\n            fleetspeak_enabled=fs,\n            first_seen=mysql_utils.TimestampToRDFDatetime(first),\n            ping=mysql_utils.TimestampToRDFDatetime(ping),\n            clock=mysql_utils.TimestampToRDFDatetime(clk),\n            ip=mysql_utils.StringToRDFProto(rdf_client_network.NetworkAddress,\n                                            ip),\n            last_foreman_time=mysql_utils.TimestampToRDFDatetime(foreman),\n            startup_info_timestamp=mysql_utils.TimestampToRDFDatetime(\n                last_startup_ts),\n            last_crash_timestamp=mysql_utils.TimestampToRDFDatetime(\n                last_crash_ts))\n\n        if client_obj is not None:\n          l_snapshot = rdf_objects.ClientSnapshot.FromSerializedString(\n              client_obj)\n          l_snapshot.timestamp = mysql_utils.TimestampToRDFDatetime(\n              last_client_ts)\n          l_snapshot.startup_info = rdf_client.StartupInfo.FromSerializedString(\n              client_startup_obj)\n          l_snapshot.startup_info.timestamp = l_snapshot.timestamp\n        else:\n          l_snapshot = rdf_objects.ClientSnapshot(\n              client_id=db_utils.IntToClientID(cid))\n\n        if last_startup_obj is not None:\n          startup_info = rdf_client.StartupInfo.FromSerializedString(\n              last_startup_obj)\n          startup_info.timestamp = mysql_utils.TimestampToRDFDatetime(\n              last_startup_ts)\n        else:\n          startup_info = None\n\n        prev_cid = cid\n        c_full_info = rdf_objects.ClientFullInfo(\n            metadata=metadata,\n            labels=[],\n            last_snapshot=l_snapshot,\n            last_startup_info=startup_info)\n\n      if label_owner and label_name:\n        c_full_info.labels.append(\n            rdf_objects.ClientLabel(name=label_name, owner=label_owner))\n\n    if c_full_info:\n      yield db_utils.IntToClientID(prev_cid), c_full_info"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef MultiReadClientFullInfo(self, client_ids, min_last_ping=None,\n                              cursor=None):\n    \"\"\"Reads full client information for a list of clients.\"\"\"\n    if not client_ids:\n      return {}\n\n    query = (\n        \"SELECT \"\n        \"c.client_id, c.fleetspeak_enabled, c.certificate, \"\n        \"UNIX_TIMESTAMP(c.last_ping), UNIX_TIMESTAMP(c.last_clock), \"\n        \"c.last_ip, UNIX_TIMESTAMP(c.last_foreman), \"\n        \"UNIX_TIMESTAMP(c.first_seen), \"\n        \"UNIX_TIMESTAMP(c.last_snapshot_timestamp), \"\n        \"UNIX_TIMESTAMP(c.last_crash_timestamp), \"\n        \"UNIX_TIMESTAMP(c.last_startup_timestamp), \"\n        \"h.client_snapshot, s.startup_info, s_last.startup_info, \"\n        \"l.owner_username, l.label \"\n        \"FROM clients as c \"\n        \"FORCE INDEX (PRIMARY) \"\n        \"LEFT JOIN client_snapshot_history as h FORCE INDEX (PRIMARY) ON ( \"\n        \"c.client_id = h.client_id AND \"\n        \"h.timestamp = c.last_snapshot_timestamp) \"\n        \"LEFT JOIN client_startup_history as s FORCE INDEX (PRIMARY) ON ( \"\n        \"c.client_id = s.client_id AND \"\n        \"s.timestamp = c.last_snapshot_timestamp) \"\n        \"LEFT JOIN client_startup_history as s_last FORCE INDEX (PRIMARY) ON ( \"\n        \"c.client_id = s_last.client_id \"\n        \"AND s_last.timestamp = c.last_startup_timestamp) \"\n        \"LEFT JOIN client_labels AS l FORCE INDEX (PRIMARY) \"\n        \"ON (c.client_id = l.client_id) \")\n\n    query += \"WHERE c.client_id IN (%s) \" % \", \".join([\"%s\"] * len(client_ids))\n\n    values = [db_utils.ClientIDToInt(cid) for cid in client_ids]\n    if min_last_ping is not None:\n      query += \"AND c.last_ping >= FROM_UNIXTIME(%s)\"\n      values.append(mysql_utils.RDFDatetimeToTimestamp(min_last_ping))\n\n    cursor.execute(query, values)\n    return dict(self._ResponseToClientsFullInfo(cursor.fetchall()))", "response": "Reads full client information for a list of clients."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef ReadClientLastPings(self,\n                          min_last_ping=None,\n                          max_last_ping=None,\n                          fleetspeak_enabled=None,\n                          cursor=None):\n    \"\"\"Reads client ids for all clients in the database.\"\"\"\n    query = \"SELECT client_id, UNIX_TIMESTAMP(last_ping) FROM clients \"\n    query_values = []\n    where_filters = []\n    if min_last_ping is not None:\n      where_filters.append(\"last_ping >= FROM_UNIXTIME(%s) \")\n      query_values.append(mysql_utils.RDFDatetimeToTimestamp(min_last_ping))\n    if max_last_ping is not None:\n      where_filters.append(\n          \"(last_ping IS NULL OR last_ping <= FROM_UNIXTIME(%s))\")\n      query_values.append(mysql_utils.RDFDatetimeToTimestamp(max_last_ping))\n    if fleetspeak_enabled is not None:\n      if fleetspeak_enabled:\n        where_filters.append(\"fleetspeak_enabled IS TRUE\")\n      else:\n        where_filters.append(\n            \"(fleetspeak_enabled IS NULL OR fleetspeak_enabled IS FALSE)\")\n\n    if where_filters:\n      query += \"WHERE \" + \"AND \".join(where_filters)\n    cursor.execute(query, query_values)\n    last_pings = {}\n    for int_client_id, last_ping in cursor.fetchall():\n      client_id = db_utils.IntToClientID(int_client_id)\n      last_pings[client_id] = mysql_utils.TimestampToRDFDatetime(last_ping)\n    return last_pings", "response": "Reads the last ping timestamps for all clients in the database."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nassociate the provided keywords with the client.", "response": "def AddClientKeywords(self, client_id, keywords, cursor=None):\n    \"\"\"Associates the provided keywords with the client.\"\"\"\n    cid = db_utils.ClientIDToInt(client_id)\n    keywords = set(keywords)\n    args = [(cid, mysql_utils.Hash(kw), kw) for kw in keywords]\n    args = list(collection.Flatten(args))\n\n    query = \"\"\"\n        INSERT INTO client_keywords (client_id, keyword_hash, keyword)\n        VALUES {}\n        ON DUPLICATE KEY UPDATE timestamp = NOW(6)\n            \"\"\".format(\", \".join([\"(%s, %s, %s)\"] * len(keywords)))\n    try:\n      cursor.execute(query, args)\n    except MySQLdb.IntegrityError as e:\n      raise db.UnknownClientError(client_id, cause=e)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nremoving the association of a particular client to a keyword.", "response": "def RemoveClientKeyword(self, client_id, keyword, cursor=None):\n    \"\"\"Removes the association of a particular client to a keyword.\"\"\"\n    cursor.execute(\n        \"DELETE FROM client_keywords \"\n        \"WHERE client_id = %s AND keyword_hash = %s\",\n        [db_utils.ClientIDToInt(client_id),\n         mysql_utils.Hash(keyword)])"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef ListClientsForKeywords(self, keywords, start_time=None, cursor=None):\n    keywords = set(keywords)\n    hash_to_kw = {mysql_utils.Hash(kw): kw for kw in keywords}\n    result = {kw: [] for kw in keywords}\n\n    query = \"\"\"\n      SELECT keyword_hash, client_id\n      FROM client_keywords\n      FORCE INDEX (client_index_by_keyword_hash)\n      WHERE keyword_hash IN ({})\n    \"\"\".format(\", \".join([\"%s\"] * len(result)))\n    args = list(iterkeys(hash_to_kw))\n    if start_time:\n      query += \" AND timestamp >= FROM_UNIXTIME(%s)\"\n      args.append(mysql_utils.RDFDatetimeToTimestamp(start_time))\n    cursor.execute(query, args)\n\n    for kw_hash, cid in cursor.fetchall():\n      result[hash_to_kw[kw_hash]].append(db_utils.IntToClientID(cid))\n    return result", "response": "Lists the clients associated with keywords."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef AddClientLabels(self, client_id, owner, labels, cursor=None):\n    cid = db_utils.ClientIDToInt(client_id)\n    labels = set(labels)\n    args = [(cid, mysql_utils.Hash(owner), owner, label) for label in labels]\n    args = list(collection.Flatten(args))\n\n    query = \"\"\"\n          INSERT IGNORE INTO client_labels\n              (client_id, owner_username_hash, owner_username, label)\n          VALUES {}\n          \"\"\".format(\", \".join([\"(%s, %s, %s, %s)\"] * len(labels)))\n    try:\n      cursor.execute(query, args)\n    except MySQLdb.IntegrityError as e:\n      raise db.UnknownClientError(client_id, cause=e)", "response": "Adds a list of user labels to a client."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreads the user labels for a list of clients.", "response": "def MultiReadClientLabels(self, client_ids, cursor=None):\n    \"\"\"Reads the user labels for a list of clients.\"\"\"\n\n    int_ids = [db_utils.ClientIDToInt(cid) for cid in client_ids]\n    query = (\"SELECT client_id, owner_username, label \"\n             \"FROM client_labels \"\n             \"WHERE client_id IN ({})\").format(\", \".join([\"%s\"] *\n                                                         len(client_ids)))\n\n    ret = {client_id: [] for client_id in client_ids}\n    cursor.execute(query, int_ids)\n    for client_id, owner, label in cursor.fetchall():\n      ret[db_utils.IntToClientID(client_id)].append(\n          rdf_objects.ClientLabel(name=label, owner=owner))\n\n    for r in itervalues(ret):\n      r.sort(key=lambda label: (label.owner, label.name))\n    return ret"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nremove a list of user labels from a given client.", "response": "def RemoveClientLabels(self, client_id, owner, labels, cursor=None):\n    \"\"\"Removes a list of user labels from a given client.\"\"\"\n\n    query = (\"DELETE FROM client_labels \"\n             \"WHERE client_id = %s AND owner_username_hash = %s \"\n             \"AND label IN ({})\").format(\", \".join([\"%s\"] * len(labels)))\n    args = [db_utils.ClientIDToInt(client_id), mysql_utils.Hash(owner)] + labels\n    cursor.execute(query, args)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreading the user labels for a list of clients.", "response": "def ReadAllClientLabels(self, cursor=None):\n    \"\"\"Reads the user labels for a list of clients.\"\"\"\n\n    cursor.execute(\"SELECT DISTINCT owner_username, label FROM client_labels\")\n\n    result = []\n    for owner, label in cursor.fetchall():\n      result.append(rdf_objects.ClientLabel(name=label, owner=owner))\n\n    result.sort(key=lambda label: (label.owner, label.name))\n    return result"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nwrite a new client crash record.", "response": "def WriteClientCrashInfo(self, client_id, crash_info, cursor=None):\n    \"\"\"Writes a new client crash record.\"\"\"\n    query = \"\"\"\n    SET @now = NOW(6);\n\n    INSERT INTO client_crash_history (client_id, timestamp, crash_info)\n         VALUES (%(client_id)s, @now, %(crash_info)s);\n\n    UPDATE clients\n       SET last_crash_timestamp = @now\n     WHERE client_id = %(client_id)s\n    \"\"\"\n\n    params = {\n        \"client_id\": db_utils.ClientIDToInt(client_id),\n        \"crash_info\": crash_info.SerializeToString(),\n    }\n\n    try:\n      cursor.execute(query, params)\n    except MySQLdb.IntegrityError as e:\n      raise db.UnknownClientError(client_id, cause=e)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nread the latest client crash record for a single client.", "response": "def ReadClientCrashInfo(self, client_id, cursor=None):\n    \"\"\"Reads the latest client crash record for a single client.\"\"\"\n    cursor.execute(\n        \"SELECT UNIX_TIMESTAMP(timestamp), crash_info \"\n        \"FROM clients, client_crash_history WHERE \"\n        \"clients.client_id = client_crash_history.client_id AND \"\n        \"clients.last_crash_timestamp = client_crash_history.timestamp AND \"\n        \"clients.client_id = %s\", [db_utils.ClientIDToInt(client_id)])\n    row = cursor.fetchone()\n    if not row:\n      return None\n\n    timestamp, crash_info = row\n    res = rdf_client.ClientCrash.FromSerializedString(crash_info)\n    res.timestamp = mysql_utils.TimestampToRDFDatetime(timestamp)\n    return res"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreading the full crash history for a particular client.", "response": "def ReadClientCrashInfoHistory(self, client_id, cursor=None):\n    \"\"\"Reads the full crash history for a particular client.\"\"\"\n    cursor.execute(\n        \"SELECT UNIX_TIMESTAMP(timestamp), crash_info \"\n        \"FROM client_crash_history WHERE \"\n        \"client_crash_history.client_id = %s \"\n        \"ORDER BY timestamp DESC\", [db_utils.ClientIDToInt(client_id)])\n    ret = []\n    for timestamp, crash_info in cursor.fetchall():\n      ci = rdf_client.ClientCrash.FromSerializedString(crash_info)\n      ci.timestamp = mysql_utils.TimestampToRDFDatetime(timestamp)\n      ret.append(ci)\n    return ret"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nwrites a ClientStats instance to the database.", "response": "def WriteClientStats(self,\n                       client_id,\n                       stats,\n                       cursor=None):\n    \"\"\"Stores a ClientStats instance.\"\"\"\n\n    try:\n      cursor.execute(\n          \"\"\"\n          INSERT INTO client_stats (client_id, payload, timestamp)\n          VALUES (%s, %s, FROM_UNIXTIME(%s))\n          ON DUPLICATE KEY UPDATE payload=VALUES(payload)\n          \"\"\", [\n              db_utils.ClientIDToInt(client_id),\n              stats.SerializeToString(),\n              mysql_utils.RDFDatetimeToTimestamp(rdfvalue.RDFDatetime.Now())\n          ])\n    except MySQLdb.IntegrityError as e:\n      if e.args[0] == mysql_error_constants.NO_REFERENCED_ROW_2:\n        raise db.UnknownClientError(client_id, cause=e)\n      else:\n        raise"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef ReadClientStats(self,\n                      client_id,\n                      min_timestamp,\n                      max_timestamp,\n                      cursor=None):\n    \"\"\"Reads ClientStats for a given client and time range.\"\"\"\n\n    cursor.execute(\n        \"\"\"\n        SELECT payload FROM client_stats\n        WHERE client_id = %s\n          AND timestamp BETWEEN FROM_UNIXTIME(%s) AND FROM_UNIXTIME(%s)\n        ORDER BY timestamp ASC\n        \"\"\", [\n            db_utils.ClientIDToInt(client_id),\n            mysql_utils.RDFDatetimeToTimestamp(min_timestamp),\n            mysql_utils.RDFDatetimeToTimestamp(max_timestamp)\n        ])\n    return [\n        rdf_client_stats.ClientStats.FromSerializedString(stats_bytes)\n        for stats_bytes, in cursor.fetchall()\n    ]", "response": "Reads ClientStats for a given client and time range."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef DeleteOldClientStats(self, yield_after_count,\n                           retention_time\n                          ):\n    \"\"\"Deletes ClientStats older than a given timestamp.\"\"\"\n\n    yielded = False\n\n    while True:\n      deleted_count = self._DeleteClientStats(\n          limit=yield_after_count, retention_time=retention_time)\n\n      # Do not yield a trailing 0 after any non-zero count has been yielded.\n      # A trailing zero occurs, when an exact multiple of `yield_after_count`\n      # rows were in the table.\n      if not yielded or deleted_count > 0:\n        yield deleted_count\n        yielded = True\n\n      # Return, when no more rows can be deleted, indicated by a transaction\n      # that does not reach the deletion limit.\n      if deleted_count < yield_after_count:\n        return", "response": "Deletes ClientStats older than a given timestamp."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _DeleteClientStats(self,\n                         limit,\n                         retention_time,\n                         cursor=None):\n    \"\"\"Deletes up to `limit` ClientStats older than `retention_time`.\"\"\"\n    cursor.execute(\n        \"DELETE FROM client_stats WHERE timestamp < FROM_UNIXTIME(%s) LIMIT %s\",\n        [mysql_utils.RDFDatetimeToTimestamp(retention_time), limit])\n    return cursor.rowcount", "response": "Deletes up to limit ClientStats older than retention_time."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncounting the client - activity metrics for a given statistic.", "response": "def _CountClientStatisticByLabel(self, statistic, day_buckets, cursor):\n    \"\"\"Returns client-activity metrics for a given statistic.\n\n    Args:\n      statistic: The name of the statistic, which should also be a column in the\n        'clients' table.\n      day_buckets: A set of n-day-active buckets.\n      cursor: MySQL cursor for executing queries.\n    \"\"\"\n    day_buckets = sorted(day_buckets)\n    sum_clauses = []\n    ping_cast_clauses = []\n    timestamp_buckets = []\n    now = rdfvalue.RDFDatetime.Now()\n\n    for day_bucket in day_buckets:\n      column_name = \"days_active_{}\".format(day_bucket)\n      sum_clauses.append(\n          \"CAST(SUM({0}) AS UNSIGNED) AS {0}\".format(column_name))\n      ping_cast_clauses.append(\n          \"CAST(c.last_ping > FROM_UNIXTIME(%s) AS UNSIGNED) AS {}\".format(\n              column_name))\n      timestamp_bucket = now - rdfvalue.Duration.FromDays(day_bucket)\n      timestamp_buckets.append(\n          mysql_utils.RDFDatetimeToTimestamp(timestamp_bucket))\n\n    query = \"\"\"\n    SELECT j.{statistic}, j.label, {sum_clauses}\n    FROM (\n      SELECT c.{statistic} AS {statistic}, l.label AS label, {ping_cast_clauses}\n      FROM clients c\n      LEFT JOIN client_labels l USING(client_id)\n      WHERE c.last_ping IS NOT NULL AND l.owner_username = 'GRR'\n    ) AS j\n    GROUP BY j.{statistic}, j.label\n    \"\"\".format(\n        statistic=statistic,\n        sum_clauses=\", \".join(sum_clauses),\n        ping_cast_clauses=\", \".join(ping_cast_clauses))\n\n    cursor.execute(query, timestamp_buckets)\n\n    counts = {}\n    for response_row in cursor.fetchall():\n      statistic_value, client_label = response_row[:2]\n      for i, num_actives in enumerate(response_row[2:]):\n        if num_actives <= 0:\n          continue\n        stats_key = (statistic_value, client_label, day_buckets[i])\n        counts[stats_key] = num_actives\n    return counts"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nparsing a string path component into a PathComponent instance.", "response": "def ParsePathItem(item, opts=None):\n  \"\"\"Parses string path component to an `PathComponent` instance.\n\n  Args:\n    item: A path component string to be parsed.\n    opts: A `PathOpts` object.\n\n  Returns:\n    `PathComponent` instance corresponding to given path fragment.\n\n  Raises:\n    ValueError: If the path item contains a recursive component fragment but\n      cannot be parsed as such.\n  \"\"\"\n  if item == os.path.curdir:\n    return CurrentComponent()\n\n  if item == os.path.pardir:\n    return ParentComponent()\n\n  recursion = PATH_RECURSION_REGEX.search(item)\n  if recursion is None:\n    return GlobComponent(item, opts)\n\n  start, end = recursion.span()\n  if not (start == 0 and end == len(item)):\n    raise ValueError(\"malformed recursive component\")\n\n  if recursion.group(\"max_depth\"):\n    max_depth = int(recursion.group(\"max_depth\"))\n  else:\n    max_depth = None\n\n  return RecursiveComponent(max_depth=max_depth, opts=opts)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nparse a given path into a stream of PathComponent instances.", "response": "def ParsePath(path,\n              opts = None):\n  \"\"\"Parses given path into a stream of `PathComponent` instances.\n\n  Args:\n    path: A path to be parsed.\n    opts: An `PathOpts` object.\n\n  Yields:\n    `PathComponent` instances corresponding to the components of the given path.\n\n  Raises:\n    ValueError: If path contains more than one recursive component.\n  \"\"\"\n  precondition.AssertType(path, Text)\n\n  rcount = 0\n\n  # Split the path at all forward slashes and if running under Windows, also\n  # backward slashes. This allows ParsePath to handle native paths and also\n  # normalized VFS paths like /HKEY_LOCAL_MACHINE/SAM.\n  normalized_path = path.replace(os.path.sep, \"/\")\n\n  for item in normalized_path.split(\"/\"):\n    component = ParsePathItem(item, opts=opts)\n    if isinstance(component, RecursiveComponent):\n      rcount += 1\n      if rcount > 1:\n        raise ValueError(\"path cannot have more than one recursive component\")\n    yield component"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\napplying all expansion mechanisms to a given path.", "response": "def ExpandPath(path, opts=None):\n  \"\"\"Applies all expansion mechanisms to the given path.\n\n  Args:\n    path: A path to expand.\n    opts: A `PathOpts` object.\n\n  Yields:\n    All paths possible to obtain from a given path by performing expansions.\n  \"\"\"\n  precondition.AssertType(path, Text)\n\n  for grouped_path in ExpandGroups(path):\n    for globbed_path in ExpandGlobs(grouped_path, opts):\n      yield globbed_path"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef ExpandGroups(path):\n  precondition.AssertType(path, Text)\n\n  chunks = []\n  offset = 0\n\n  for match in PATH_GROUP_REGEX.finditer(path):\n    chunks.append([path[offset:match.start()]])\n    chunks.append(match.group(\"alts\").split(\",\"))\n    offset = match.end()\n\n  chunks.append([path[offset:]])\n\n  for prod in itertools.product(*chunks):\n    yield \"\".join(prod)", "response": "Performs group expansion on a given path."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nperform glob expansion on a given path.", "response": "def ExpandGlobs(path, opts = None):\n  \"\"\"Performs glob expansion on a given path.\n\n  Path can contain regular glob elements (such as `**`, `*`, `?`, `[a-z]`). For\n  example, having files `foo`, `bar`, `baz` glob expansion of `ba?` will yield\n  `bar` and `baz`.\n\n  Args:\n    path: A path to expand.\n    opts: A `PathOpts` object.\n\n  Returns:\n    Generator over all possible glob expansions of a given path.\n\n  Raises:\n    ValueError: If given path is empty or relative.\n  \"\"\"\n  precondition.AssertType(path, Text)\n  if not path:\n    raise ValueError(\"Path is empty\")\n\n  if not _IsAbsolutePath(path, opts):\n    raise ValueError(\"Path '%s' is not absolute\" % path)\n\n  if opts is not None and opts.pathtype == rdf_paths.PathSpec.PathType.REGISTRY:\n    # Handle HKLM\\Foo and /HKLM/Foo identically.\n    root_dir, tail = path.replace(\"\\\\\", \"/\").lstrip(\"/\").split(\"/\", 1)\n    components = list(ParsePath(tail, opts=opts))\n  else:\n    drive, tail = os.path.splitdrive(path)\n    root_dir = os.path.join(drive, os.path.sep).upper()\n    components = list(ParsePath(tail[1:], opts=opts))\n\n  return _ExpandComponents(root_dir, components)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _ListDir(dirpath, pathtype):\n  pathspec = rdf_paths.PathSpec(path=dirpath, pathtype=pathtype)\n  childpaths = []\n  try:\n    file_obj = vfs.VFSOpen(pathspec)\n    for path in file_obj.ListNames():\n      # For Windows registry, ignore the empty string which corresponds to the\n      # default value in the current key. Otherwise, globbing a key will yield\n      # the key itself, because joining the name of the default value u\"\" with\n      # a key name yields the key name again.\n      if pathtype != rdf_paths.PathSpec.PathType.REGISTRY or path:\n        childpaths.append(path)\n  except IOError:\n    pass\n\n  return childpaths", "response": "Returns a list of potential children of a given directory."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef ListNetworkConnectionsFromClient(args):\n  for proc in psutil.process_iter():\n    try:\n      connections = proc.connections()\n    except (psutil.NoSuchProcess, psutil.AccessDenied):\n      continue\n\n    for conn in connections:\n      if args.listening_only and conn.status != \"LISTEN\":\n        continue\n\n      res = rdf_client_network.NetworkConnection()\n      res.pid = proc.pid\n      res.process_name = proc.name()\n      res.family = conn.family\n      res.type = conn.type\n      try:\n        if conn.status:\n          res.state = conn.status\n      except ValueError:\n        logging.warning(\"Encountered unknown connection status (%s).\",\n                        conn.status)\n\n      res.local_address.ip, res.local_address.port = conn.laddr\n      if conn.raddr:\n        res.remote_address.ip, res.remote_address.port = conn.raddr\n\n      yield res", "response": "Gather open network connection stats."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef IterIfaddrs(ifaddrs):\n  precondition.AssertOptionalType(ifaddrs, ctypes.POINTER(Ifaddrs))\n\n  while ifaddrs:\n    yield ifaddrs.contents\n    ifaddrs = ifaddrs.contents.ifa_next", "response": "Iterates over contents of the intrusive linked list of ifaddrs."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nparses the contents of the intrusive linked list of ifaddrs.", "response": "def ParseIfaddrs(ifaddrs):\n  \"\"\"Parses contents of the intrusive linked list of `ifaddrs`.\n\n  Args:\n    ifaddrs: A pointer to the first node of `ifaddrs` linked list. Can be NULL.\n\n  Returns:\n    An iterator over instances of `rdf_client_network.Interface`.\n  \"\"\"\n  precondition.AssertOptionalType(ifaddrs, ctypes.POINTER(Ifaddrs))\n\n  ifaces = {}\n\n  for ifaddr in IterIfaddrs(ifaddrs):\n    ifname = ctypes.string_at(ifaddr.ifa_name).decode(\"utf-8\")\n    iface = ifaces.setdefault(ifname, rdf_client_network.Interface())\n    iface.ifname = ifname\n\n    if not ifaddr.ifa_addr:\n      continue\n\n    sockaddr = ctypes.cast(ifaddr.ifa_addr, ctypes.POINTER(Sockaddr))\n    iffamily = sockaddr.contents.sa_family\n    if iffamily == AF_INET:\n      sockaddrin = ctypes.cast(ifaddr.ifa_addr, ctypes.POINTER(Sockaddrin))\n\n      address = rdf_client_network.NetworkAddress()\n      address.address_type = rdf_client_network.NetworkAddress.Family.INET\n      address.packed_bytes = struct.pack(\"=L\", sockaddrin.contents.sin_addr)\n      iface.addresses.append(address)\n    elif iffamily == AF_INET6:\n      sockaddrin = ctypes.cast(ifaddr.ifa_addr, ctypes.POINTER(Sockaddrin6))\n\n      address = rdf_client_network.NetworkAddress()\n      address.address_type = rdf_client_network.NetworkAddress.Family.INET6\n      address.packed_bytes = bytes(list(sockaddrin.contents.sin6_addr))\n      iface.addresses.append(address)\n    elif iffamily == AF_LINK:\n      sockaddrdl = ctypes.cast(ifaddr.ifa_addr, ctypes.POINTER(Sockaddrdl))\n\n      nlen = sockaddrdl.contents.sdl_nlen\n      alen = sockaddrdl.contents.sdl_alen\n      iface.mac_address = bytes(sockaddrdl.contents.sdl_data[nlen:nlen + alen])\n    else:\n      raise ValueError(\"Unexpected socket address family: %s\" % iffamily)\n\n  return itervalues(ifaces)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nenumerate all MAC addresses.", "response": "def EnumerateInterfacesFromClient(args):\n  \"\"\"Enumerate all MAC addresses.\"\"\"\n  del args  # Unused\n\n  libc = ctypes.cdll.LoadLibrary(ctypes.util.find_library(\"c\"))\n  ifa = Ifaddrs()\n  p_ifa = ctypes.pointer(ifa)\n  libc.getifaddrs(ctypes.pointer(p_ifa))\n\n  for iface in ParseIfaddrs(p_ifa):\n    yield iface\n\n  libc.freeifaddrs(p_ifa)"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef EnumerateFilesystemsFromClient(args):\n  del args  # Unused.\n  for fs_struct in client_utils_osx.GetFileSystems():\n    yield rdf_client_fs.Filesystem(\n        device=fs_struct.f_mntfromname,\n        mount_point=fs_struct.f_mntonname,\n        type=fs_struct.f_fstypename)\n\n  drive_re = re.compile(\"r?disk[0-9].*\")\n  for drive in os.listdir(\"/dev\"):\n    if not drive_re.match(drive):\n      continue\n\n    path = os.path.join(\"/dev\", drive)\n    try:\n      img_inf = pytsk3.Img_Info(path)\n      # This is a volume or a partition - we send back a TSK device.\n      yield rdf_client_fs.Filesystem(device=path)\n\n      vol_inf = pytsk3.Volume_Info(img_inf)\n\n      for volume in vol_inf:\n        if volume.flags == pytsk3.TSK_VS_PART_FLAG_ALLOC:\n          offset = volume.start * vol_inf.info.block_size\n          yield rdf_client_fs.Filesystem(\n              device=\"{path}:{offset}\".format(path=path, offset=offset),\n              type=\"partition\")\n\n    except (IOError, RuntimeError):\n      continue", "response": "Enumerate all local filesystems mounted on this system."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncreate the Service protobuf.", "response": "def CreateServiceProto(job):\n  \"\"\"Create the Service protobuf.\n\n  Args:\n    job: Launchdjobdict from servicemanagement framework.\n\n  Returns:\n    sysinfo_pb2.OSXServiceInformation proto\n  \"\"\"\n  service = rdf_client.OSXServiceInformation(\n      label=job.get(\"Label\"),\n      program=job.get(\"Program\"),\n      sessiontype=job.get(\"LimitLoadToSessionType\"),\n      lastexitstatus=int(job[\"LastExitStatus\"]),\n      timeout=int(job[\"TimeOut\"]),\n      ondemand=bool(job[\"OnDemand\"]))\n\n  for arg in job.get(\"ProgramArguments\", \"\", stringify=False):\n    # Returns CFArray of CFStrings\n    service.args.Append(str(arg))\n\n  mach_dict = job.get(\"MachServices\", {}, stringify=False)\n  for key, value in iteritems(mach_dict):\n    service.machservice.Append(\"%s:%s\" % (key, value))\n\n  job_mach_dict = job.get(\"PerJobMachServices\", {}, stringify=False)\n  for key, value in iteritems(job_mach_dict):\n    service.perjobmachservice.Append(\"%s:%s\" % (key, value))\n\n  if \"PID\" in job:\n    service.pid = job[\"PID\"].value\n\n  return service"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nenumerating running launchd jobs.", "response": "def OSXEnumerateRunningServicesFromClient(args):\n  \"\"\"Get running launchd jobs.\n\n  Args:\n    args: Unused.\n\n  Yields:\n    `rdf_client.OSXServiceInformation` instances.\n\n  Raises:\n      UnsupportedOSVersionError: for OS X earlier than 10.6.\n  \"\"\"\n  del args  # Unused.\n  osx_version = client_utils_osx.OSXVersion()\n  version_array = osx_version.VersionAsMajorMinor()\n\n  if version_array[:2] < [10, 6]:\n    raise UnsupportedOSVersionError(\n        \"ServiceManagement API unsupported on < 10.6. This client is %s\" %\n        osx_version.VersionString())\n\n  launchd_list = GetRunningLaunchDaemons()\n\n  parser = osx_launchd.OSXLaunchdJobDict(launchd_list)\n  for job in parser.Parse():\n    response = CreateServiceProto(job)\n    yield response"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef Run(self, unused_arg):\n    logging.debug(\"Disabling service\")\n\n    msg = \"Service disabled.\"\n    if hasattr(sys, \"frozen\"):\n      grr_binary = os.path.abspath(sys.executable)\n    elif __file__:\n      grr_binary = os.path.abspath(__file__)\n\n    try:\n      os.remove(grr_binary)\n    except OSError:\n      msg = \"Could not remove binary.\"\n\n    try:\n      os.remove(config.CONFIG[\"Client.plist_path\"])\n    except OSError:\n      if \"Could not\" in msg:\n        msg += \" Could not remove plist file.\"\n      else:\n        msg = \"Could not remove plist file.\"\n\n    # Get the directory we are running in from pyinstaller. This is either the\n    # GRR directory which we should delete (onedir mode) or a generated temp\n    # directory which we can delete without problems in onefile mode.\n    directory = getattr(sys, \"_MEIPASS\", None)\n    if directory:\n      shutil.rmtree(directory, ignore_errors=True)\n\n    self.SendReply(rdf_protodict.DataBlob(string=msg))", "response": "This kills us with no cleanups."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nstarting remote debugging using pydevd.", "response": "def _start_remote_debugging(port):\n  \"\"\"Sets up remote debugging using pydevd, connecting to localhost:`port`.\"\"\"\n  try:\n    print(\"Connecting to remote debugger on localhost:{}.\".format(port))\n    import pydevd  # pylint: disable=g-import-not-at-top\n    pydevd.settrace(\n        \"localhost\",\n        port=port,\n        stdoutToServer=True,\n        stderrToServer=True,\n        suspend=flags.FLAGS.break_on_start)\n  except ImportError:\n    print(\n        \"pydevd is required for remote debugging. Please follow the PyCharm\"\n        \"manual or run `pip install pydevd-pycharm` to install.\",\n        file=sys.stderr)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef ExtractConfig(self):\n    logging.info(\"Extracting config file from .pkg.\")\n    pkg_path = os.environ.get(\"PACKAGE_PATH\", None)\n    if pkg_path is None:\n      logging.error(\"Could not locate package, giving up.\")\n      return\n\n    zf = zipfile.ZipFile(pkg_path, mode=\"r\")\n    fd = zf.open(\"config.yaml\")\n    install_dir = os.path.dirname(config.CONFIG.parser.filename)\n\n    # We write this config to disk so that Intialize can find the build.yaml\n    # referenced inside the config as a relative path. This config isn't used\n    # after install time.\n    installer_config = os.path.join(install_dir, \"installer_config.yaml\")\n    with open(installer_config, \"wb\") as f:\n      f.write(fd.read())\n\n    packaged_config = config.CONFIG.MakeNewConfig()\n    packaged_config.Initialize(\n        filename=installer_config, parser=config_lib.YamlParser)\n\n    new_config = config.CONFIG.MakeNewConfig()\n    new_config.SetWriteBack(config.CONFIG[\"Config.writeback\"])\n\n    for info in config.CONFIG.type_infos:\n      try:\n        new_value = packaged_config.GetRaw(info.name, None)\n      except type_info.TypeValueError:\n        continue\n\n      try:\n        old_value = config.CONFIG.GetRaw(info.name, None)\n\n        if not new_value or new_value == old_value:\n          continue\n      except type_info.TypeValueError:\n        pass\n\n      new_config.SetRaw(info.name, new_value)\n\n    new_config.Write()\n    logging.info(\"Config file extracted successfully.\")\n\n    logging.info(\"Extracting additional files.\")\n\n    for zinfo in zf.filelist:\n      basename = os.path.basename(zinfo.filename)\n      if basename != \"config.yaml\":\n        with open(os.path.join(install_dir, basename), \"wb\") as f:\n          f.write(zf.open(zinfo.filename).read())", "response": "This installer extracts a config file from the. pkg file."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _LoadFlows(self, client_id, min_create_time, token):\n    if data_store.RelationalDBEnabled():\n      if isinstance(client_id, rdfvalue.RDFURN):\n        client_id = client_id.Basename()\n\n      flow_list = data_store.REL_DB.ReadAllFlowObjects(\n          client_id=client_id,\n          min_create_time=min_create_time,\n          include_child_flows=False)\n      for flow_obj in flow_list:\n        yield flow_obj\n    else:\n      now = rdfvalue.RDFDatetime.Now()\n      client_id_urn = rdf_client.ClientURN(client_id)\n      flows_dir = aff4.FACTORY.Open(client_id_urn.Add(\"flows\"), token=token)\n      # Save DB roundtrips by checking both conditions at once.\n      flow_list = flows_dir.ListChildren(\n          age=(min_create_time.AsMicrosecondsSinceEpoch(),\n               now.AsMicrosecondsSinceEpoch()))\n      for flow_obj in aff4.FACTORY.MultiOpen(flow_list, token=token):\n        yield rdf_flow_objects.Flow(\n            args=flow_obj.args,\n            flow_class_name=flow_obj.runner_args.flow_name,\n            flow_id=flow_obj.urn.Basename(),\n            create_time=flow_obj.context.create_time,\n            creator=flow_obj.creator,\n        )", "response": "Load all flows for a given client_id and time range."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef EnforceLimits(self, client_id, user, flow_name, flow_args, token=None):\n    if not self.dup_interval and not self.daily_req_limit:\n      return\n\n    now = rdfvalue.RDFDatetime.Now()\n    yesterday = now - rdfvalue.Duration(\"1d\")\n    dup_boundary = now - self.dup_interval\n    min_create_time = min(yesterday, dup_boundary)\n\n    flow_count = 0\n    flows = self._LoadFlows(client_id, min_create_time, token=token)\n\n    if flow_args is None:\n      flow_args = flow.EmptyFlowArgs()\n\n    for flow_obj in flows:\n      if (flow_obj.create_time > dup_boundary and\n          flow_obj.flow_class_name == flow_name and flow_obj.args == flow_args):\n        raise DuplicateFlowError(\n            \"Identical %s already run on %s at %s\" %\n            (flow_name, client_id, flow_obj.create_time),\n            flow_id=flow_obj.flow_id)\n\n      # Filter for flows started by user within the 1 day window.\n      if flow_obj.creator == user and flow_obj.create_time > yesterday:\n        flow_count += 1\n\n    # If limit is set, enforce it.\n    if self.daily_req_limit and flow_count >= self.daily_req_limit:\n      raise DailyFlowRequestLimitExceededError(\n          \"%s flows run since %s, limit: %s\" %\n          (flow_count, yesterday, self.daily_req_limit))", "response": "Enforces DailyFlowRequestLimit and FlowDuplicateInterval."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nbuilds a StatEntry object from a given path.", "response": "def StatEntryFromPath(path, pathspec, ext_attrs=True):\n  \"\"\"Builds a stat entry object from a given path.\n\n  Args:\n    path: A path (string value) to stat.\n    pathspec: A `PathSpec` corresponding to the `path`.\n    ext_attrs: Whether to include extended file attributes in the result.\n\n  Returns:\n    `StatEntry` object.\n  \"\"\"\n  try:\n    stat = filesystem.Stat.FromPath(path)\n  except (IOError, OSError) as error:\n    logging.error(\"Failed to obtain stat for '%s': %s\", pathspec, error)\n    return rdf_client_fs.StatEntry(pathspec=pathspec)\n\n  return StatEntryFromStat(stat, pathspec, ext_attrs=ext_attrs)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef StatEntryFromStat(stat,\n                      pathspec,\n                      ext_attrs = True):\n  \"\"\"Build a stat entry object from a given stat object.\n\n  Args:\n    stat: A `Stat` object.\n    pathspec: A `PathSpec` from which `stat` was obtained.\n    ext_attrs: Whether to include extended file attributes in the result.\n\n  Returns:\n    `StatEntry` object.\n  \"\"\"\n  result = rdf_client_fs.StatEntry(pathspec=pathspec)\n\n  for attr in _STAT_ATTRS:\n    value = getattr(stat.GetRaw(), attr, None)\n    if value is None:\n      continue\n\n    # TODO(hanuszczak): Why are we doing this?\n    value = int(value)\n    if value < 0:\n      value &= 0xFFFFFFFF\n\n    setattr(result, attr, value)\n\n  result.st_flags_linux = stat.GetLinuxFlags()\n  result.st_flags_osx = stat.GetOsxFlags()\n  if ext_attrs:\n    # TODO(hanuszczak): Can we somehow incorporate extended attribute getter to\n    # the `Stat` class? That would make the code a lot prettier but would force\n    # `utils` to depend on `xattrs`.\n    result.ext_attrs = list(GetExtAttrs(stat.GetPath()))\n\n  return result", "response": "Builds a stat entry object from a given stat object."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning a os. stat_result object with most information from a StatEntry.", "response": "def StatResultFromStatEntry(\n    stat_entry):\n  \"\"\"Returns a `os.stat_result` with most information from `StatEntry`.\n\n  This is a lossy conversion, only the 10 first stat_result fields are\n  populated, because the os.stat_result constructor is inflexible.\n\n  Args:\n    stat_entry: An instance of rdf_client_fs.StatEntry.\n\n  Returns:\n    An instance of `os.stat_result` with basic fields populated.\n  \"\"\"\n  values = []\n  for attr in _STAT_ATTRS[:10]:\n    values.append(stat_entry.Get(attr))\n  return os.stat_result(values)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ninitializing Api ( Client Hunt or CronJob ) Approval from an AFF4 object.", "response": "def _InitApiApprovalFromAff4Object(api_approval, approval_obj):\n  \"\"\"Initializes Api(Client|Hunt|CronJob)Approval from an AFF4 object.\"\"\"\n\n  api_approval.id = approval_obj.urn.Basename()\n  api_approval.reason = approval_obj.Get(approval_obj.Schema.REASON)\n  api_approval.requestor = approval_obj.Get(approval_obj.Schema.REQUESTOR)\n\n  # We should check the approval validity from the standpoint of the user\n  # who had requested it.\n  test_token = access_control.ACLToken(\n      username=approval_obj.Get(approval_obj.Schema.REQUESTOR))\n  try:\n    approval_obj.CheckAccess(test_token)\n    api_approval.is_valid = True\n  except access_control.UnauthorizedAccess as e:\n    api_approval.is_valid = False\n    api_approval.is_valid_message = utils.SmartStr(e)\n\n  notified_users = approval_obj.Get(approval_obj.Schema.NOTIFIED_USERS)\n  if notified_users:\n    api_approval.notified_users = sorted(\n        u.strip() for u in notified_users.split(\",\"))\n\n  api_approval.email_message_id = approval_obj.Get(\n      approval_obj.Schema.EMAIL_MSG_ID)\n\n  email_cc = approval_obj.Get(approval_obj.Schema.EMAIL_CC)\n  email_cc_addresses = sorted(s.strip() for s in email_cc.split(\",\"))\n  api_approval.email_cc_addresses = (\n      set(email_cc_addresses) - set(api_approval.notified_users))\n\n  api_approval.approvers = sorted(approval_obj.GetNonExpiredApprovers())\n\n  return api_approval"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ninitialize Api ( Client Hunt | CronJob ) Approval from the database object.", "response": "def _InitApiApprovalFromDatabaseObject(api_approval, db_obj):\n  \"\"\"Initializes Api(Client|Hunt|CronJob)Approval from the database object.\"\"\"\n\n  api_approval.id = db_obj.approval_id\n  api_approval.requestor = db_obj.requestor_username\n  api_approval.reason = db_obj.reason\n\n  api_approval.notified_users = sorted(db_obj.notified_users)\n  api_approval.email_cc_addresses = sorted(db_obj.email_cc_addresses)\n  api_approval.email_message_id = db_obj.email_message_id\n\n  api_approval.approvers = sorted([g.grantor_username for g in db_obj.grants])\n\n  try:\n    approval_checks.CheckApprovalRequest(db_obj)\n    api_approval.is_valid = True\n  except access_control.UnauthorizedAccess as e:\n    api_approval.is_valid_message = utils.SmartStr(e)\n    api_approval.is_valid = False\n\n  return api_approval"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef InitFromNotification(self, notification, is_pending=False):\n    self.timestamp = notification.timestamp\n    self.message = notification.message\n    self.subject = str(notification.subject)\n    self.is_pending = is_pending\n\n    reference_type_enum = ApiNotificationReference.Type\n\n    # Please see the comments to aff4_objects.GRRUser.Notify implementation\n    # for the details of notification.type format. Short summary:\n    # notification.type may be one of legacy values (i.e. \"ViewObject\") or\n    # have a format of \"[legacy value]:[new-style notification type]\", i.e.\n    # \"ViewObject:TYPE_CLIENT_INTERROGATED\".\n    legacy_type = None\n    if \":\" in notification.type:\n      legacy_type, new_type = notification.type.split(\":\", 2)\n      self.notification_type = new_type\n    else:\n      legacy_type = notification.type\n\n    # TODO(user): refactor notifications, so that we send a meaningful\n    # notification from the start, so that we don't have to do the\n    # bridging/conversion/guessing here.\n    components = self._GetUrnComponents(notification)\n    if legacy_type == \"Discovery\":\n      self.reference.type = reference_type_enum.CLIENT\n      self.reference.client = ApiNotificationClientReference(\n          client_id=components[0])\n    elif legacy_type == \"ViewObject\":\n      if len(components) >= 2 and components[0] == \"hunts\":\n        self.reference.type = reference_type_enum.HUNT\n        self.reference.hunt.hunt_id = components[1]\n      elif len(components) >= 2 and components[0] == \"cron\":\n        self.reference.type = reference_type_enum.CRON\n        self.reference.cron.cron_job_id = components[1]\n      elif len(components) >= 3 and components[1] == \"flows\":\n        self.reference.type = reference_type_enum.FLOW\n        self.reference.flow.flow_id = components[2]\n        self.reference.flow.client_id = components[0]\n      elif len(components) == 1 and rdf_client.ClientURN.Validate(\n          components[0]):\n        self.reference.type = reference_type_enum.CLIENT\n        self.reference.client.client_id = components[0]\n      else:\n        if notification.subject:\n          path = notification.subject.Path()\n          for prefix in itervalues(rdf_paths.PathSpec.AFF4_PREFIXES):\n            part = \"/%s%s\" % (components[0], prefix)\n            if path.startswith(part):\n              self.reference.type = reference_type_enum.VFS\n              self.reference.vfs.client_id = components[0]\n              self.reference.vfs.vfs_path = (prefix +\n                                             path[len(part):]).lstrip(\"/\")\n              break\n\n        if self.reference.type != reference_type_enum.VFS:\n          self.reference.type = reference_type_enum.UNKNOWN\n          self.reference.unknown.subject_urn = notification.subject\n\n    elif legacy_type == \"FlowStatus\":\n      if not components or not rdf_client.ClientURN.Validate(components[0]):\n        self.reference.type = reference_type_enum.UNKNOWN\n        self.reference.unknown.subject_urn = notification.subject\n      else:\n        self.reference.type = reference_type_enum.FLOW\n        self.reference.flow.flow_id = notification.source.Basename()\n        self.reference.flow.client_id = components[0]\n\n    # TODO(user): refactor GrantAccess notification so that we don't have\n    # to infer approval type from the URN.\n    elif legacy_type == \"GrantAccess\":\n      if rdf_client.ClientURN.Validate(components[1]):\n        self.reference.type = reference_type_enum.CLIENT_APPROVAL\n        self.reference.client_approval.client_id = components[1]\n        self.reference.client_approval.approval_id = components[-1]\n        self.reference.client_approval.username = components[-2]\n      elif components[1] == \"hunts\":\n        self.reference.type = reference_type_enum.HUNT_APPROVAL\n        self.reference.hunt_approval.hunt_id = components[2]\n        self.reference.hunt_approval.approval_id = components[-1]\n        self.reference.hunt_approval.username = components[-2]\n      elif components[1] == \"cron\":\n        self.reference.type = reference_type_enum.CRON_JOB_APPROVAL\n        self.reference.cron_job_approval.cron_job_id = components[2]\n        self.reference.cron_job_approval.approval_id = components[-1]\n        self.reference.cron_job_approval.username = components[-2]\n\n    else:\n      self.reference.type = reference_type_enum.UNKNOWN\n      self.reference.unknown.subject_urn = notification.subject\n      self.reference.unknown.source_urn = notification.source\n\n    return self", "response": "Initializes this object from an existing notification."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning a list of approvals for a given user and approval type.", "response": "def _GetApprovals(self,\n                    approval_type,\n                    offset,\n                    count,\n                    filter_func=None,\n                    token=None):\n    \"\"\"Gets all approvals for a given user and approval type.\n\n    Args:\n      approval_type: The type of approvals to get.\n      offset: The starting index within the collection.\n      count: The number of items to return.\n      filter_func: A predicate function, returning True if a specific approval\n        should be included in the result and False otherwise.\n      token: The token identifying the user.\n\n    Returns:\n      A list of approvals of the given approval type.\n    \"\"\"\n    approvals_base_urn = aff4.ROOT_URN.Add(\"users\").Add(\n        token.username).Add(\"approvals\").Add(approval_type)\n\n    all_children = aff4.FACTORY.RecursiveMultiListChildren([approvals_base_urn])\n\n    approvals_urns = []\n    for subject, children in all_children:\n      # We only want to process leaf nodes.\n      if children:\n        continue\n      approvals_urns.append(subject)\n\n    approvals_urns.sort(key=lambda x: x.age, reverse=True)\n    approvals = list(\n        aff4.FACTORY.MultiOpen(\n            approvals_urns,\n            mode=\"r\",\n            aff4_type=aff4_security.Approval,\n            age=aff4.ALL_TIMES,\n            token=token))\n    approvals_by_urn = {}\n    for approval in approvals:\n      approvals_by_urn[approval.symlink_urn or approval.urn] = approval\n\n    cur_offset = 0\n    sorted_approvals = []\n    for approval_urn in approvals_urns:\n      try:\n        approval = approvals_by_urn[approval_urn]\n      except KeyError:\n        continue\n\n      if filter_func is not None and not filter_func(approval):\n        continue\n      cur_offset += 1\n      if cur_offset <= offset:\n        continue\n      if count and len(sorted_approvals) >= count:\n        break\n      sorted_approvals.append(approval)\n\n    subjects_urns = [a.Get(a.Schema.SUBJECT) for a in approvals]\n    subjects_by_urn = {}\n    for subject in aff4.FACTORY.MultiOpen(subjects_urns, mode=\"r\", token=token):\n      subjects_by_urn[subject.urn] = subject\n\n    return sorted_approvals, subjects_by_urn"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nfetches and renders current user s settings.", "response": "def Handle(self, unused_args, token=None):\n    \"\"\"Fetches and renders current user's settings.\"\"\"\n\n    result = ApiGrrUser(username=token.username)\n\n    if data_store.RelationalDBEnabled():\n      user_record = data_store.REL_DB.ReadGRRUser(token.username)\n      result.InitFromDatabaseObject(user_record)\n    else:\n      try:\n        user_record = aff4.FACTORY.Open(\n            aff4.ROOT_URN.Add(\"users\").Add(token.username),\n            aff4_users.GRRUser,\n            token=token)\n\n        result.InitFromAff4Object(user_record)\n      except IOError:\n        result.settings = aff4_users.GRRUser.SchemaCls.GUI_SETTINGS()\n\n    result.interface_traits = (\n        self.interface_traits or ApiGrrUserInterfaceTraits())\n\n    return result"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef Handle(self, args, token=None):\n    if data_store.RelationalDBEnabled():\n      self.HandleRelationalDB(args, token=token)\n    else:\n      self.HandleLegacy(args, token=token)", "response": "Deletes the notification from the pending notifications."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nschedules all system cron jobs.", "response": "def ScheduleSystemCronJobs(names=None):\n  \"\"\"Schedules all system cron jobs.\"\"\"\n\n  errors = []\n  disabled_classes = config.CONFIG[\"Cron.disabled_cron_jobs\"]\n  for name in disabled_classes:\n    try:\n      cls = registry.SystemCronJobRegistry.CronJobClassByName(name)\n    except ValueError:\n      errors.append(\"Cron job not found: %s.\" % name)\n      continue\n\n  if names is None:\n    names = iterkeys(registry.SystemCronJobRegistry.SYSTEM_CRON_REGISTRY)\n\n  for name in names:\n\n    cls = registry.SystemCronJobRegistry.CronJobClassByName(name)\n\n    enabled = cls.enabled and name not in disabled_classes\n    system = rdf_cronjobs.CronJobAction.ActionType.SYSTEM_CRON_ACTION\n    args = rdf_cronjobs.CronJobAction(\n        action_type=system,\n        system_cron_action=rdf_cronjobs.SystemCronAction(job_class_name=name))\n\n    job = rdf_cronjobs.CronJob(\n        cron_job_id=name,\n        args=args,\n        enabled=enabled,\n        frequency=cls.frequency,\n        lifetime=cls.lifetime,\n        allow_overruns=cls.allow_overruns)\n    data_store.REL_DB.WriteCronJob(job)\n\n  if errors:\n    raise ValueError(\n        \"Error(s) while parsing Cron.disabled_cron_jobs: %s\" % errors)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nstart a new run for the given cron job.", "response": "def StartRun(self, wait_for_start_event, signal_event, wait_for_write_event):\n    \"\"\"Starts a new run for the given cron job.\"\"\"\n    # Signal that the cron thread has started. This way the cron scheduler\n    # will know that the task is not sitting in a threadpool queue, but is\n    # actually executing.\n    wait_for_start_event.set()\n    # Wait until the cron scheduler acknowledges the run. If it doesn't\n    # acknowledge, just return (it means that the cron scheduler considers\n    # this task as \"not started\" and has returned the lease so that another\n    # worker can pick it up).\n    if not signal_event.wait(TASK_STARTUP_WAIT):\n      return\n\n    try:\n      logging.info(\"Processing cron job: %s\", self.job.cron_job_id)\n      self.run_state.started_at = rdfvalue.RDFDatetime.Now()\n      self.run_state.status = \"RUNNING\"\n\n      data_store.REL_DB.WriteCronJobRun(self.run_state)\n      data_store.REL_DB.UpdateCronJob(\n          self.job.cron_job_id,\n          last_run_time=rdfvalue.RDFDatetime.Now(),\n          current_run_id=self.run_state.run_id,\n          forced_run_requested=False)\n    finally:\n      # Notify the cron scheduler that all the DB updates are done. At this\n      # point the cron scheduler can safely return this job's lease.\n      wait_for_write_event.set()\n\n    try:\n      self.Run()\n      self.run_state.status = \"FINISHED\"\n    except LifetimeExceededError:\n      self.run_state.status = \"LIFETIME_EXCEEDED\"\n      stats_collector_instance.Get().IncrementCounter(\n          \"cron_job_failure\", fields=[self.job.cron_job_id])\n    except Exception as e:  # pylint: disable=broad-except\n      logging.exception(\"Cronjob %s failed with an error: %s\",\n                        self.job.cron_job_id, e)\n      stats_collector_instance.Get().IncrementCounter(\n          \"cron_job_failure\", fields=[self.job.cron_job_id])\n      self.run_state.status = \"ERROR\"\n      self.run_state.backtrace = str(e)\n\n    finally:\n      self.run_state.finished_at = rdfvalue.RDFDatetime.Now()\n      elapsed = self.run_state.finished_at - self.run_state.started_at\n      stats_collector_instance.Get().RecordEvent(\n          \"cron_job_latency\", elapsed.seconds, fields=[self.job.cron_job_id])\n      if self.job.lifetime:\n        expiration_time = self.run_state.started_at + self.job.lifetime\n        if self.run_state.finished_at > expiration_time:\n          self.run_state.status = \"LIFETIME_EXCEEDED\"\n          stats_collector_instance.Get().IncrementCounter(\n              \"cron_job_timeout\", fields=[self.job.cron_job_id])\n      data_store.REL_DB.WriteCronJobRun(self.run_state)\n\n    current_job = data_store.REL_DB.ReadCronJob(self.job.cron_job_id)\n    # If no other job was started while we were running, update last status\n    # information.\n    if current_job.current_run_id == self.run_state.run_id:\n      data_store.REL_DB.UpdateCronJob(\n          self.job.cron_job_id,\n          current_run_id=None,\n          last_run_status=self.run_state.status)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nterminate a cronjob - run if it has exceeded its maximum runtime.", "response": "def HeartBeat(self):\n    \"\"\"Terminates a cronjob-run if it has exceeded its maximum runtime.\n\n    This is a no-op for cronjobs that allow overruns.\n\n    Raises:\n      LifetimeExceededError: If the cronjob has exceeded its maximum runtime.\n    \"\"\"\n    # In prod, self.job.lifetime is guaranteed to always be set, and is\n    # always equal to self.__class__.lifetime. Some tests however, do not\n    # set the job lifetime, which isn't great.\n    if self.allow_overruns or not self.job.lifetime:\n      return\n\n    runtime = rdfvalue.RDFDatetime.Now() - self.run_state.started_at\n    if runtime > self.lifetime:\n      raise LifetimeExceededError(\n          \"Cronjob run has exceeded the maximum runtime of %s.\" % self.lifetime)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncreating a new cron job object that runs given flow with given frequency.", "response": "def CreateJob(self, cron_args=None, job_id=None, enabled=True, token=None):\n    \"\"\"Creates a cron job that runs given flow with a given frequency.\n\n    Args:\n      cron_args: A protobuf of type rdf_cronjobs.CreateCronJobArgs.\n      job_id: Use this job_id instead of an autogenerated unique name (used for\n        system cron jobs - we want them to have well-defined persistent name).\n      enabled: If False, the job object will be created, but will be disabled.\n      token: Security token used for data store access. Unused.\n\n    Returns:\n      URN of the cron job created.\n\n    Raises:\n      ValueError: This function expects an arg protobuf that starts a\n                  CreateAndRunGenericHuntFlow flow. If the args specify\n                  something else, ValueError is raised.\n    \"\"\"\n    # TODO(amoser): Remove the token from this method once the aff4\n    # cronjobs are gone.\n    del token\n    if not job_id:\n      uid = random.UInt16()\n      job_id = \"%s_%s\" % (cron_args.flow_name, uid)\n\n    args = rdf_cronjobs.CronJobAction(\n        action_type=rdf_cronjobs.CronJobAction.ActionType.HUNT_CRON_ACTION,\n        hunt_cron_action=rdf_cronjobs.HuntCronAction(\n            flow_name=cron_args.flow_name,\n            flow_args=cron_args.flow_args,\n            hunt_runner_args=cron_args.hunt_runner_args))\n\n    job = rdf_cronjobs.CronJob(\n        cron_job_id=job_id,\n        description=cron_args.description,\n        frequency=cron_args.frequency,\n        lifetime=cron_args.lifetime,\n        allow_overruns=cron_args.allow_overruns,\n        args=args,\n        enabled=enabled)\n    data_store.REL_DB.WriteCronJob(job)\n\n    return job_id"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef ListJobs(self, token=None):\n    del token\n    return [job.cron_job_id for job in data_store.REL_DB.ReadCronJobs()]", "response": "Returns a list of ids of all currently running cron jobs."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef RunOnce(self, names=None, token=None):\n    del token\n\n    leased_jobs = data_store.REL_DB.LeaseCronJobs(\n        cronjob_ids=names, lease_time=rdfvalue.Duration(\"10m\"))\n    logging.info(\"Leased %d cron jobs for processing.\", len(leased_jobs))\n    if not leased_jobs:\n      return\n\n    errors = {}\n    processed_count = 0\n    for job in sorted(leased_jobs, key=lambda j: j.cron_job_id):\n      if self.TerminateStuckRunIfNeeded(job):\n        continue\n\n      if not self.JobDueToRun(job):\n        continue\n\n      try:\n        if self.RunJob(job):\n          processed_count += 1\n        else:\n          logging.info(\n              \"Can't schedule cron job %s on a thread pool \"\n              \"(all threads are busy or CPU load is high)\", job.cron_job_id)\n          break\n      except Exception as e:  # pylint: disable=broad-except\n        logging.exception(\"Cron job %s has failed: %s\", job.cron_job_id, e)\n        errors[job.cron_job_id] = e\n\n    logging.info(\"Processed %d cron jobs.\", processed_count)\n    data_store.REL_DB.ReturnLeasedCronJobs(leased_jobs)\n\n    if errors:\n      raise OneOrMoreCronJobsFailedError(errors)", "response": "Attempts to run one or more cron jobs."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef TerminateStuckRunIfNeeded(self, job):\n    if job.current_run_id and job.last_run_time and job.lifetime:\n      now = rdfvalue.RDFDatetime.Now()\n      # We add additional 10 minutes to give the job run a chance to kill itself\n      # during one of the HeartBeat calls (HeartBeat checks if a cron job is\n      # run is running too long and raises if it is).\n      expiration_time = (\n          job.last_run_time + job.lifetime + rdfvalue.Duration(\"10m\"))\n      if now > expiration_time:\n        run = data_store.REL_DB.ReadCronJobRun(job.cron_job_id,\n                                               job.current_run_id)\n        run.status = \"LIFETIME_EXCEEDED\"\n        run.finished_at = now\n        data_store.REL_DB.WriteCronJobRun(run)\n        data_store.REL_DB.UpdateCronJob(\n            job.cron_job_id, current_run_id=None, last_run_status=run.status)\n        stats_collector_instance.Get().RecordEvent(\n            \"cron_job_latency\", (now - job.last_run_time).seconds,\n            fields=[job.cron_job_id])\n        stats_collector_instance.Get().IncrementCounter(\n            \"cron_job_timeout\", fields=[job.cron_job_id])\n\n        return True\n\n    return False", "response": "Cleans up job state if the last run is stuck."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef RunJob(self, job):\n    if not job.leased_until:\n      raise LockError(\"CronJob must be leased for Run() to be called.\")\n    if job.leased_until < rdfvalue.RDFDatetime.Now():\n      raise LockError(\"CronJob lease expired for %s.\" % job.cron_job_id)\n\n    logging.info(\"Starting cron job: %s\", job.cron_job_id)\n\n    if job.args.action_type == job.args.ActionType.SYSTEM_CRON_ACTION:\n      cls_name = job.args.system_cron_action.job_class_name\n      job_cls = registry.SystemCronJobRegistry.CronJobClassByName(cls_name)\n      name = \"%s runner\" % cls_name\n    elif job.args.action_type == job.args.ActionType.HUNT_CRON_ACTION:\n      job_cls = registry.CronJobRegistry.CronJobClassByName(\"RunHunt\")\n      name = \"Hunt runner\"\n    else:\n      raise ValueError(\n          \"CronJob %s doesn't have a valid args type set.\" % job.cron_job_id)\n\n    run_state = rdf_cronjobs.CronJobRun(\n        cron_job_id=job.cron_job_id, status=\"RUNNING\")\n    run_state.GenerateRunId()\n\n    run_obj = job_cls(run_state, job)\n    wait_for_start_event, signal_event, wait_for_write_event = (\n        threading.Event(), threading.Event(), threading.Event())\n    try:\n      self._GetThreadPool().AddTask(\n          target=run_obj.StartRun,\n          args=(wait_for_start_event, signal_event, wait_for_write_event),\n          name=name,\n          blocking=False,\n          inline=False)\n      if not wait_for_start_event.wait(TASK_STARTUP_WAIT):\n        logging.error(\"Cron job run task for %s is too slow to start.\",\n                      job.cron_job_id)\n        # Most likely the thread pool is full and the task is sitting on the\n        # queue. Make sure we don't put more things on the queue by returning\n        # False.\n        return False\n\n      # We know that the cron job task has started, unblock it by setting\n      # the signal event. If signal_event is not set (this happens if the\n      # task sits on a ThreadPool's queue doing nothing, see the\n      # if-statement above) the task will just be a no-op when ThreadPool\n      # finally gets to it. This way we can ensure that we can safely return\n      # the lease and let another worker schedule the same job.\n      signal_event.set()\n\n      wait_for_write_event.wait(TASK_STARTUP_WAIT)\n\n      return True\n    except threadpool.Full:\n      return False", "response": "Does the actual work of the Cron, if the job is due to run.\n\n    Args:\n      job: The cronjob rdfvalue that should be run. Must be leased.\n\n    Returns:\n      A boolean indicating if this cron job was started or not. False may\n      be returned when the threadpool is already full.\n\n    Raises:\n      LockError: if the object is not locked.\n      ValueError: If the job argument is invalid."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef JobDueToRun(self, job):\n    if not job.enabled:\n      return False\n\n    if job.forced_run_requested:\n      return True\n\n    now = rdfvalue.RDFDatetime.Now()\n\n    if (job.last_run_time is not None and\n        job.last_run_time + job.frequency > now):\n      return False\n\n    # No currently executing job - lets go.\n    if not job.current_run_id:\n      return True\n\n    # There is a job executing but we allow overruns.\n    if job.allow_overruns:\n      return True\n\n    return False", "response": "Determines if the given job is due to run."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef DeleteOldRuns(self, cutoff_timestamp=None):\n    if cutoff_timestamp is None:\n      raise ValueError(\"cutoff_timestamp can't be None\")\n\n    return data_store.REL_DB.DeleteOldCronJobRuns(\n        cutoff_timestamp=cutoff_timestamp)", "response": "Deletes runs that were started before the timestamp given."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef WriteMessageHandlerRequests(self, requests, cursor=None):\n    query = (\"INSERT IGNORE INTO message_handler_requests \"\n             \"(handlername, request_id, request) VALUES \")\n\n    value_templates = []\n    args = []\n    for r in requests:\n      args.extend([r.handler_name, r.request_id, r.SerializeToString()])\n      value_templates.append(\"(%s, %s, %s)\")\n\n    query += \",\".join(value_templates)\n    cursor.execute(query, args)", "response": "Writes a list of message handler requests to the database."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef ReadMessageHandlerRequests(self, cursor=None):\n\n    query = (\"SELECT UNIX_TIMESTAMP(timestamp), request,\"\n             \"       UNIX_TIMESTAMP(leased_until), leased_by \"\n             \"FROM message_handler_requests \"\n             \"ORDER BY timestamp DESC\")\n\n    cursor.execute(query)\n\n    res = []\n    for timestamp, request, leased_until, leased_by in cursor.fetchall():\n      req = rdf_objects.MessageHandlerRequest.FromSerializedString(request)\n      req.timestamp = mysql_utils.TimestampToRDFDatetime(timestamp)\n      req.leased_by = leased_by\n      req.leased_until = mysql_utils.TimestampToRDFDatetime(leased_until)\n      res.append(req)\n    return res", "response": "Reads all message handler requests from the database."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef DeleteMessageHandlerRequests(self, requests, cursor=None):\n\n    query = \"DELETE FROM message_handler_requests WHERE request_id IN ({})\"\n    request_ids = set([r.request_id for r in requests])\n    query = query.format(\",\".join([\"%s\"] * len(request_ids)))\n    cursor.execute(query, request_ids)", "response": "Deletes a list of message handler requests from the database."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nleases a number of message handler requests up to the indicated time.", "response": "def _LeaseMessageHandlerRequests(self, lease_time, limit, cursor=None):\n    \"\"\"Leases a number of message handler requests up to the indicated limit.\"\"\"\n\n    now = rdfvalue.RDFDatetime.Now()\n    now_str = mysql_utils.RDFDatetimeToTimestamp(now)\n\n    expiry = now + lease_time\n    expiry_str = mysql_utils.RDFDatetimeToTimestamp(expiry)\n\n    query = (\"UPDATE message_handler_requests \"\n             \"SET leased_until=FROM_UNIXTIME(%s), leased_by=%s \"\n             \"WHERE leased_until IS NULL OR leased_until < FROM_UNIXTIME(%s) \"\n             \"LIMIT %s\")\n\n    id_str = utils.ProcessIdString()\n    args = (expiry_str, id_str, now_str, limit)\n    updated = cursor.execute(query, args)\n\n    if updated == 0:\n      return []\n\n    cursor.execute(\n        \"SELECT UNIX_TIMESTAMP(timestamp), request \"\n        \"FROM message_handler_requests \"\n        \"WHERE leased_by=%s AND leased_until=FROM_UNIXTIME(%s) LIMIT %s\",\n        (id_str, expiry_str, updated))\n    res = []\n    for timestamp, request in cursor.fetchall():\n      req = rdf_objects.MessageHandlerRequest.FromSerializedString(request)\n      req.timestamp = mysql_utils.TimestampToRDFDatetime(timestamp)\n      req.leased_until = expiry\n      req.leased_by = id_str\n      res.append(req)\n\n    return res"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nread all client messages available for a given client_id.", "response": "def ReadAllClientActionRequests(self, client_id, cursor=None):\n    \"\"\"Reads all client messages available for a given client_id.\"\"\"\n\n    query = (\"SELECT request, UNIX_TIMESTAMP(leased_until), leased_by, \"\n             \"leased_count \"\n             \"FROM client_action_requests \"\n             \"WHERE client_id = %s\")\n\n    cursor.execute(query, [db_utils.ClientIDToInt(client_id)])\n\n    ret = []\n    for req, leased_until, leased_by, leased_count in cursor.fetchall():\n      request = rdf_flows.ClientActionRequest.FromSerializedString(req)\n      if leased_until is not None:\n        request.leased_by = leased_by\n        request.leased_until = mysql_utils.TimestampToRDFDatetime(leased_until)\n      else:\n        request.leased_by = None\n        request.leased_until = None\n      request.ttl = db.Database.CLIENT_MESSAGES_TTL - leased_count\n      ret.append(request)\n\n    return sorted(ret, key=lambda req: (req.flow_id, req.request_id))"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ndelete a list of client messages from the db.", "response": "def DeleteClientActionRequests(self, requests):\n    \"\"\"Deletes a list of client messages from the db.\"\"\"\n    if not requests:\n      return\n\n    to_delete = []\n    for r in requests:\n      to_delete.append((r.client_id, r.flow_id, r.request_id))\n\n    if len(set(to_delete)) != len(to_delete):\n      raise ValueError(\n          \"Received multiple copies of the same message to delete.\")\n\n    self._DeleteClientActionRequest(to_delete)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef LeaseClientActionRequests(self,\n                                client_id,\n                                lease_time=None,\n                                limit=None,\n                                cursor=None):\n    \"\"\"Leases available client messages for the client with the given id.\"\"\"\n\n    now = rdfvalue.RDFDatetime.Now()\n    now_str = mysql_utils.RDFDatetimeToTimestamp(now)\n    expiry = now + lease_time\n    expiry_str = mysql_utils.RDFDatetimeToTimestamp(expiry)\n    proc_id_str = utils.ProcessIdString()\n    client_id_int = db_utils.ClientIDToInt(client_id)\n\n    query = (\"UPDATE client_action_requests \"\n             \"SET leased_until=FROM_UNIXTIME(%s), leased_by=%s, \"\n             \"leased_count=leased_count+1 \"\n             \"WHERE client_id=%s AND \"\n             \"(leased_until IS NULL OR leased_until < FROM_UNIXTIME(%s)) \"\n             \"LIMIT %s\")\n    args = [expiry_str, proc_id_str, client_id_int, now_str, limit]\n\n    num_leased = cursor.execute(query, args)\n    if num_leased == 0:\n      return []\n\n    query = (\"SELECT request, leased_count FROM client_action_requests \"\n             \"WHERE client_id=%s AND leased_until=FROM_UNIXTIME(%s) \"\n             \"AND leased_by=%s\")\n\n    cursor.execute(query, [client_id_int, expiry_str, proc_id_str])\n\n    ret = []\n    expired = []\n    for req, leased_count in cursor.fetchall():\n      request = rdf_flows.ClientActionRequest.FromSerializedString(req)\n      request.leased_by = proc_id_str\n      request.leased_until = expiry\n      request.ttl = db.Database.CLIENT_MESSAGES_TTL - leased_count\n      # > comparison since this check happens after the lease.\n      if leased_count > db.Database.CLIENT_MESSAGES_TTL:\n        expired.append((request.client_id, request.flow_id, request.request_id))\n      else:\n        ret.append(request)\n\n    if expired:\n      self._DeleteClientActionRequest(expired, cursor=cursor)\n\n    return sorted(ret, key=lambda req: (req.flow_id, req.request_id))", "response": "Leases available client messages for the given client with the given id."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef WriteClientActionRequests(self, requests, cursor=None):\n\n    query = (\"INSERT IGNORE INTO client_action_requests \"\n             \"(client_id, flow_id, request_id, timestamp, request) \"\n             \"VALUES %s ON DUPLICATE KEY UPDATE \"\n             \"timestamp=VALUES(timestamp), request=VALUES(request)\")\n    now = mysql_utils.RDFDatetimeToTimestamp(rdfvalue.RDFDatetime.Now())\n\n    value_templates = []\n    args = []\n    for r in requests:\n      args.extend([\n          db_utils.ClientIDToInt(r.client_id),\n          db_utils.FlowIDToInt(r.flow_id), r.request_id, now,\n          r.SerializeToString()\n      ])\n      value_templates.append(\"(%s, %s, %s, FROM_UNIXTIME(%s), %s)\")\n\n    query %= \",\".join(value_templates)\n    try:\n      cursor.execute(query, args)\n    except MySQLdb.IntegrityError as e:\n      request_keys = [(r.client_id, r.flow_id, r.request_id) for r in requests]\n      raise db.AtLeastOneUnknownRequestError(request_keys=request_keys, cause=e)", "response": "Writes the given list of client action requests to the db."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nwrites a single flow object to the database.", "response": "def WriteFlowObject(self, flow_obj, cursor=None):\n    \"\"\"Writes a flow object to the database.\"\"\"\n\n    query = \"\"\"\n    INSERT INTO flows (client_id, flow_id, long_flow_id, parent_flow_id,\n                       parent_hunt_id, flow, flow_state,\n                       next_request_to_process, pending_termination, timestamp,\n                       network_bytes_sent, user_cpu_time_used_micros,\n                       system_cpu_time_used_micros, num_replies_sent, last_update)\n    VALUES (%(client_id)s, %(flow_id)s, %(long_flow_id)s, %(parent_flow_id)s,\n            %(parent_hunt_id)s, %(flow)s, %(flow_state)s,\n            %(next_request_to_process)s, %(pending_termination)s,\n            FROM_UNIXTIME(%(timestamp)s),\n            %(network_bytes_sent)s, %(user_cpu_time_used_micros)s,\n            %(system_cpu_time_used_micros)s, %(num_replies_sent)s, NOW(6))\n    ON DUPLICATE KEY UPDATE\n        flow=VALUES(flow),\n        flow_state=VALUES(flow_state),\n        next_request_to_process=VALUES(next_request_to_process),\n        last_update=VALUES(last_update)\n    \"\"\"\n\n    user_cpu_time_used_micros = db_utils.SecondsToMicros(\n        flow_obj.cpu_time_used.user_cpu_time)\n    system_cpu_time_used_micros = db_utils.SecondsToMicros(\n        flow_obj.cpu_time_used.system_cpu_time)\n\n    args = {\n        \"client_id\": db_utils.ClientIDToInt(flow_obj.client_id),\n        \"flow_id\": db_utils.FlowIDToInt(flow_obj.flow_id),\n        \"long_flow_id\": flow_obj.long_flow_id,\n        \"flow\": flow_obj.SerializeToString(),\n        \"flow_state\": int(flow_obj.flow_state),\n        \"next_request_to_process\": flow_obj.next_request_to_process,\n        \"timestamp\": mysql_utils.RDFDatetimeToTimestamp(flow_obj.create_time),\n        \"network_bytes_sent\": flow_obj.network_bytes_sent,\n        \"num_replies_sent\": flow_obj.num_replies_sent,\n        \"user_cpu_time_used_micros\": user_cpu_time_used_micros,\n        \"system_cpu_time_used_micros\": system_cpu_time_used_micros,\n    }\n\n    if flow_obj.parent_flow_id:\n      args[\"parent_flow_id\"] = db_utils.FlowIDToInt(flow_obj.parent_flow_id)\n    else:\n      args[\"parent_flow_id\"] = None\n\n    if flow_obj.parent_hunt_id:\n      args[\"parent_hunt_id\"] = db_utils.HuntIDToInt(flow_obj.parent_hunt_id)\n    else:\n      args[\"parent_hunt_id\"] = None\n\n    if flow_obj.HasField(\"pending_termination\"):\n      serialized_termination = flow_obj.pending_termination.SerializeToString()\n      args[\"pending_termination\"] = serialized_termination\n    else:\n      args[\"pending_termination\"] = None\n\n    try:\n      cursor.execute(query, args)\n    except MySQLdb.IntegrityError as e:\n      raise db.UnknownClientError(flow_obj.client_id, cause=e)"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _FlowObjectFromRow(self, row):\n\n    flow, fs, cci, pt, nr, pd, po, ps, uct, sct, nbs, nrs, ts, lut = row\n\n    flow_obj = rdf_flow_objects.Flow.FromSerializedString(flow)\n    if fs not in [None, rdf_flow_objects.Flow.FlowState.UNSET]:\n      flow_obj.flow_state = fs\n    if cci is not None:\n      cc_cls = rdf_client.ClientCrash\n      flow_obj.client_crash_info = cc_cls.FromSerializedString(cci)\n    if pt is not None:\n      pt_cls = rdf_flow_objects.PendingFlowTermination\n      flow_obj.pending_termination = pt_cls.FromSerializedString(pt)\n    if nr:\n      flow_obj.next_request_to_process = nr\n    if pd is not None:\n      flow_obj.processing_deadline = mysql_utils.TimestampToRDFDatetime(pd)\n    if po is not None:\n      flow_obj.processing_on = po\n    if ps is not None:\n      flow_obj.processing_since = mysql_utils.TimestampToRDFDatetime(ps)\n    flow_obj.cpu_time_used.user_cpu_time = db_utils.MicrosToSeconds(uct)\n    flow_obj.cpu_time_used.system_cpu_time = db_utils.MicrosToSeconds(sct)\n    flow_obj.network_bytes_sent = nbs\n    if nrs:\n      flow_obj.num_replies_sent = nrs\n    flow_obj.timestamp = mysql_utils.TimestampToRDFDatetime(ts)\n    flow_obj.last_update_time = mysql_utils.TimestampToRDFDatetime(lut)\n\n    return flow_obj", "response": "Generates a flow object from a database row."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nread a single flow object from the database.", "response": "def ReadFlowObject(self, client_id, flow_id, cursor=None):\n    \"\"\"Reads a flow object from the database.\"\"\"\n    query = (\"SELECT \" + self.FLOW_DB_FIELDS +\n             \"FROM flows WHERE client_id=%s AND flow_id=%s\")\n    cursor.execute(\n        query,\n        [db_utils.ClientIDToInt(client_id),\n         db_utils.FlowIDToInt(flow_id)])\n    result = cursor.fetchall()\n    if not result:\n      raise db.UnknownFlowError(client_id, flow_id)\n    row, = result\n    return self._FlowObjectFromRow(row)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef ReadAllFlowObjects(self,\n                         client_id = None,\n                         min_create_time = None,\n                         max_create_time = None,\n                         include_child_flows = True,\n                         cursor=None):\n    \"\"\"Returns all flow objects.\"\"\"\n    conditions = []\n    args = []\n\n    if client_id is not None:\n      conditions.append(\"client_id = %s\")\n      args.append(db_utils.ClientIDToInt(client_id))\n\n    if min_create_time is not None:\n      conditions.append(\"timestamp >= FROM_UNIXTIME(%s)\")\n      args.append(mysql_utils.RDFDatetimeToTimestamp(min_create_time))\n\n    if max_create_time is not None:\n      conditions.append(\"timestamp <= FROM_UNIXTIME(%s)\")\n      args.append(mysql_utils.RDFDatetimeToTimestamp(max_create_time))\n\n    if not include_child_flows:\n      conditions.append(\"parent_flow_id IS NULL\")\n\n    query = \"SELECT {} FROM flows\".format(self.FLOW_DB_FIELDS)\n    if conditions:\n      query += \" WHERE \" + \" AND \".join(conditions)\n\n    cursor.execute(query, args)\n    return [self._FlowObjectFromRow(row) for row in cursor.fetchall()]", "response": "Reads all flows from the database."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef ReadChildFlowObjects(self, client_id, flow_id, cursor=None):\n    query = (\"SELECT \" + self.FLOW_DB_FIELDS +\n             \"FROM flows WHERE client_id=%s AND parent_flow_id=%s\")\n    cursor.execute(\n        query,\n        [db_utils.ClientIDToInt(client_id),\n         db_utils.FlowIDToInt(flow_id)])\n    return [self._FlowObjectFromRow(row) for row in cursor.fetchall()]", "response": "Reads child flows from the database."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef LeaseFlowForProcessing(self,\n                             client_id,\n                             flow_id,\n                             processing_time,\n                             cursor=None):\n    \"\"\"Marks a flow as being processed on this worker and returns it.\"\"\"\n    query = (\"SELECT \" + self.FLOW_DB_FIELDS +\n             \"FROM flows WHERE client_id=%s AND flow_id=%s\")\n    cursor.execute(\n        query,\n        [db_utils.ClientIDToInt(client_id),\n         db_utils.FlowIDToInt(flow_id)])\n    response = cursor.fetchall()\n    if not response:\n      raise db.UnknownFlowError(client_id, flow_id)\n\n    row, = response\n    rdf_flow = self._FlowObjectFromRow(row)\n\n    now = rdfvalue.RDFDatetime.Now()\n    if rdf_flow.processing_on and rdf_flow.processing_deadline > now:\n      raise ValueError(\"Flow %s on client %s is already being processed.\" %\n                       (client_id, flow_id))\n\n    if (rdf_flow.parent_hunt_id is not None and\n        # TODO(user): remove the check for a legacy hunt prefix as soon as\n        # AFF4 is gone.\n        not rdf_flow.parent_hunt_id.startswith(\"H:\")):\n\n      query = \"SELECT hunt_state FROM hunts WHERE hunt_id=%s\"\n      args = [db_utils.HuntIDToInt(rdf_flow.parent_hunt_id)]\n      rows_found = cursor.execute(query, args)\n      if rows_found == 1:\n        hunt_state, = cursor.fetchone()\n        if (hunt_state is not None and\n            not rdf_hunt_objects.IsHuntSuitableForFlowProcessing(hunt_state)):\n          raise db.ParentHuntIsNotRunningError(client_id, flow_id,\n                                               rdf_flow.parent_hunt_id,\n                                               hunt_state)\n\n    update_query = (\"UPDATE flows SET \"\n                    \"processing_on=%s, \"\n                    \"processing_since=FROM_UNIXTIME(%s), \"\n                    \"processing_deadline=FROM_UNIXTIME(%s) \"\n                    \"WHERE client_id=%s and flow_id=%s\")\n    processing_deadline = now + processing_time\n    process_id_string = utils.ProcessIdString()\n\n    args = [\n        process_id_string,\n        mysql_utils.RDFDatetimeToTimestamp(now),\n        mysql_utils.RDFDatetimeToTimestamp(processing_deadline),\n        db_utils.ClientIDToInt(client_id),\n        db_utils.FlowIDToInt(flow_id)\n    ]\n    cursor.execute(update_query, args)\n\n    # This needs to happen after we are sure that the write has succeeded.\n    rdf_flow.processing_on = process_id_string\n    rdf_flow.processing_since = now\n    rdf_flow.processing_deadline = processing_deadline\n    return rdf_flow", "response": "Leases a flow on this worker and returns it."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef UpdateFlow(self,\n                 client_id,\n                 flow_id,\n                 flow_obj=db.Database.unchanged,\n                 flow_state=db.Database.unchanged,\n                 client_crash_info=db.Database.unchanged,\n                 pending_termination=db.Database.unchanged,\n                 processing_on=db.Database.unchanged,\n                 processing_since=db.Database.unchanged,\n                 processing_deadline=db.Database.unchanged,\n                 cursor=None):\n    \"\"\"Updates flow objects in the database.\"\"\"\n    updates = []\n    args = []\n    if flow_obj != db.Database.unchanged:\n      updates.append(\"flow=%s\")\n      args.append(flow_obj.SerializeToString())\n      updates.append(\"flow_state=%s\")\n      args.append(int(flow_obj.flow_state))\n      updates.append(\"user_cpu_time_used_micros=%s\")\n      args.append(\n          db_utils.SecondsToMicros(flow_obj.cpu_time_used.user_cpu_time))\n      updates.append(\"system_cpu_time_used_micros=%s\")\n      args.append(\n          db_utils.SecondsToMicros(flow_obj.cpu_time_used.system_cpu_time))\n      updates.append(\"network_bytes_sent=%s\")\n      args.append(flow_obj.network_bytes_sent)\n      updates.append(\"num_replies_sent=%s\")\n      args.append(flow_obj.num_replies_sent)\n\n    if flow_state != db.Database.unchanged:\n      updates.append(\"flow_state=%s\")\n      args.append(int(flow_state))\n    if client_crash_info != db.Database.unchanged:\n      updates.append(\"client_crash_info=%s\")\n      args.append(client_crash_info.SerializeToString())\n    if pending_termination != db.Database.unchanged:\n      updates.append(\"pending_termination=%s\")\n      args.append(pending_termination.SerializeToString())\n    if processing_on != db.Database.unchanged:\n      updates.append(\"processing_on=%s\")\n      args.append(processing_on)\n    if processing_since != db.Database.unchanged:\n      updates.append(\"processing_since=FROM_UNIXTIME(%s)\")\n      args.append(mysql_utils.RDFDatetimeToTimestamp(processing_since))\n    if processing_deadline != db.Database.unchanged:\n      updates.append(\"processing_deadline=FROM_UNIXTIME(%s)\")\n      args.append(mysql_utils.RDFDatetimeToTimestamp(processing_deadline))\n\n    if not updates:\n      return\n\n    query = \"UPDATE flows SET last_update=NOW(6), \"\n    query += \", \".join(updates)\n    query += \" WHERE client_id=%s AND flow_id=%s\"\n\n    args.append(db_utils.ClientIDToInt(client_id))\n    args.append(db_utils.FlowIDToInt(flow_id))\n    updated = cursor.execute(query, args)\n    if updated == 0:\n      raise db.UnknownFlowError(client_id, flow_id)", "response": "Updates a flow in the database."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nupdating the list of flows in the database.", "response": "def UpdateFlows(self,\n                  client_id_flow_id_pairs,\n                  pending_termination=db.Database.unchanged,\n                  cursor=None):\n    \"\"\"Updates flow objects in the database.\"\"\"\n\n    if pending_termination == db.Database.unchanged:\n      return\n\n    serialized_termination = pending_termination.SerializeToString()\n    query = \"UPDATE flows SET pending_termination=%s WHERE \"\n    args = [serialized_termination]\n    for index, (client_id, flow_id) in enumerate(client_id_flow_id_pairs):\n      query += (\"\" if index == 0 else \" OR \") + \" client_id=%s AND flow_id=%s\"\n      args.extend(\n          [db_utils.ClientIDToInt(client_id),\n           db_utils.FlowIDToInt(flow_id)])\n    cursor.execute(query, args)"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _WriteFlowProcessingRequests(self, requests, cursor):\n    templates = []\n    args = []\n    for req in requests:\n      templates.append(\"(%s, %s, %s, FROM_UNIXTIME(%s))\")\n      args.append(db_utils.ClientIDToInt(req.client_id))\n      args.append(db_utils.FlowIDToInt(req.flow_id))\n      args.append(req.SerializeToString())\n      if req.delivery_time:\n        args.append(mysql_utils.RDFDatetimeToTimestamp(req.delivery_time))\n      else:\n        args.append(None)\n\n    query = (\"INSERT INTO flow_processing_requests \"\n             \"(client_id, flow_id, request, delivery_time) VALUES \")\n    query += \", \".join(templates)\n    cursor.execute(query, args)", "response": "Writes the given requests into the flow processing table."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef WriteFlowRequests(self, requests, cursor=None):\n    args = []\n    templates = []\n    flow_keys = []\n    needs_processing = {}\n\n    for r in requests:\n      if r.needs_processing:\n        needs_processing.setdefault((r.client_id, r.flow_id), []).append(r)\n\n      flow_keys.append((r.client_id, r.flow_id))\n      templates.append(\"(%s, %s, %s, %s, %s)\")\n      args.extend([\n          db_utils.ClientIDToInt(r.client_id),\n          db_utils.FlowIDToInt(r.flow_id), r.request_id, r.needs_processing,\n          r.SerializeToString()\n      ])\n\n    if needs_processing:\n      flow_processing_requests = []\n      nr_conditions = []\n      nr_args = []\n      for client_id, flow_id in needs_processing:\n        nr_conditions.append(\"(client_id=%s AND flow_id=%s)\")\n        nr_args.append(db_utils.ClientIDToInt(client_id))\n        nr_args.append(db_utils.FlowIDToInt(flow_id))\n\n      nr_query = (\"SELECT client_id, flow_id, next_request_to_process \"\n                  \"FROM flows WHERE \")\n      nr_query += \" OR \".join(nr_conditions)\n\n      cursor.execute(nr_query, nr_args)\n\n      db_result = cursor.fetchall()\n      for client_id_int, flow_id_int, next_request_to_process in db_result:\n        client_id = db_utils.IntToClientID(client_id_int)\n        flow_id = db_utils.IntToFlowID(flow_id_int)\n        candidate_requests = needs_processing.get((client_id, flow_id), [])\n        for r in candidate_requests:\n          if next_request_to_process == r.request_id:\n            flow_processing_requests.append(\n                rdf_flows.FlowProcessingRequest(\n                    client_id=client_id,\n                    flow_id=flow_id,\n                    delivery_time=r.start_time))\n\n      if flow_processing_requests:\n        self._WriteFlowProcessingRequests(flow_processing_requests, cursor)\n\n    query = (\"INSERT INTO flow_requests \"\n             \"(client_id, flow_id, request_id, needs_processing, request) \"\n             \"VALUES \")\n    query += \", \".join(templates)\n    try:\n      cursor.execute(query, args)\n    except MySQLdb.IntegrityError as e:\n      raise db.AtLeastOneUnknownFlowError(flow_keys, cause=e)", "response": "Writes a list of flow requests to the database."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nbuilding the writes to store the given responses in the db.", "response": "def _WriteResponses(self, responses, cursor):\n    \"\"\"Builds the writes to store the given responses in the db.\"\"\"\n\n    query = (\"INSERT IGNORE INTO flow_responses \"\n             \"(client_id, flow_id, request_id, response_id, \"\n             \"response, status, iterator, timestamp) VALUES \")\n\n    templates = []\n    args = []\n    for r in responses:\n      templates.append(\"(%s, %s, %s, %s, %s, %s, %s, NOW(6))\")\n      client_id_int = db_utils.ClientIDToInt(r.client_id)\n      flow_id_int = db_utils.FlowIDToInt(r.flow_id)\n\n      args.append(client_id_int)\n      args.append(flow_id_int)\n      args.append(r.request_id)\n      args.append(r.response_id)\n      if isinstance(r, rdf_flow_objects.FlowResponse):\n        args.append(r.SerializeToString())\n        args.append(\"\")\n        args.append(\"\")\n      elif isinstance(r, rdf_flow_objects.FlowStatus):\n        args.append(\"\")\n        args.append(r.SerializeToString())\n        args.append(\"\")\n      elif isinstance(r, rdf_flow_objects.FlowIterator):\n        args.append(\"\")\n        args.append(\"\")\n        args.append(r.SerializeToString())\n      else:\n        # This can't really happen due to db api type checking.\n        raise ValueError(\"Got unexpected response type: %s %s\" % (type(r), r))\n\n    query += \",\".join(templates)\n    try:\n      cursor.execute(query, args)\n    except MySQLdb.IntegrityError:\n      # If we have multiple responses and one of them fails to insert, we try\n      # them one by one so we don't lose any valid replies.\n      if len(responses) > 1:\n        for r in responses:\n          self._WriteResponses([r], cursor)\n      else:\n        logging.warn(\"Response for unknown request: %s\", responses[0])"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ndeletes the given client messages from the database.", "response": "def _DeleteClientActionRequest(self, to_delete, cursor=None):\n    \"\"\"Builds deletes for client messages.\"\"\"\n    query = \"DELETE FROM client_action_requests WHERE \"\n    conditions = []\n    args = []\n\n    for client_id, flow_id, request_id in to_delete:\n      conditions.append(\"(client_id=%s AND flow_id=%s AND request_id=%s)\")\n      args.append(db_utils.ClientIDToInt(client_id))\n      args.append(db_utils.FlowIDToInt(flow_id))\n      args.append(request_id)\n\n    query += \" OR \".join(conditions)\n    cursor.execute(query, args)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _WriteFlowResponsesAndExpectedUpdates(self, responses, cursor=None):\n\n    self._WriteResponses(responses, cursor)\n\n    query = \"\"\"\n      UPDATE flow_requests\n      SET responses_expected=%(responses_expected)s\n      WHERE\n        client_id = %(client_id)s AND\n        flow_id = %(flow_id)s AND\n        request_id = %(request_id)s\n    \"\"\"\n\n    for r in responses:\n      # If the response is a FlowStatus, we have to update the FlowRequest with\n      # the number of expected messages.\n      if isinstance(r, rdf_flow_objects.FlowStatus):\n        args = {\n            \"client_id\": db_utils.ClientIDToInt(r.client_id),\n            \"flow_id\": db_utils.FlowIDToInt(r.flow_id),\n            \"request_id\": r.request_id,\n            \"responses_expected\": r.response_id,\n        }\n        cursor.execute(query, args)", "response": "Writes a flow responses and updates flow requests expected counts."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _ReadFlowResponseCounts(self, request_keys, cursor=None):\n\n    query = \"\"\"\n      SELECT\n        flow_requests.client_id, flow_requests.flow_id,\n        flow_requests.request_id, COUNT(*)\n      FROM flow_responses, flow_requests\n      WHERE ({conditions}) AND\n        flow_requests.client_id = flow_responses.client_id AND\n        flow_requests.flow_id = flow_responses.flow_id AND\n        flow_requests.request_id = flow_responses.request_id AND\n        flow_requests.needs_processing = FALSE\n      GROUP BY\n        flow_requests.client_id,\n        flow_requests.flow_id,\n        flow_requests.request_id\n    \"\"\"\n\n    condition_template = \"\"\"\n      (flow_requests.client_id=%s AND\n       flow_requests.flow_id=%s AND\n       flow_requests.request_id=%s)\"\"\"\n    conditions = [condition_template] * len(request_keys)\n    args = []\n    for client_id, flow_id, request_id in request_keys:\n      args.append(db_utils.ClientIDToInt(client_id))\n      args.append(db_utils.FlowIDToInt(flow_id))\n      args.append(request_id)\n\n    query = query.format(conditions=\" OR \".join(conditions))\n    cursor.execute(query, args)\n    response_counts = {}\n    for (client_id_int, flow_id_int, request_id, count) in cursor.fetchall():\n      request_key = (db_utils.IntToClientID(client_id_int),\n                     db_utils.IntToFlowID(flow_id_int), request_id)\n      response_counts[request_key] = count\n    return response_counts", "response": "Reads the counts of responses for the given requests."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _ReadAndLockNextRequestsToProcess(self, flow_keys, cursor):\n\n    query = \"\"\"\n      SELECT client_id, flow_id, next_request_to_process\n      FROM flows\n      WHERE {conditions}\n      FOR UPDATE\n    \"\"\"\n    condition_template = \"(client_id = %s AND flow_id = %s)\"\n    conditions = [condition_template] * len(flow_keys)\n    query = query.format(conditions=\" OR \".join(conditions))\n    args = []\n    for client_id, flow_id in flow_keys:\n      args.append(db_utils.ClientIDToInt(client_id))\n      args.append(db_utils.FlowIDToInt(flow_id))\n\n    cursor.execute(query, args)\n    next_requests = {}\n    for client_id_int, flow_id_int, next_request in cursor.fetchall():\n      flow_key = (db_utils.IntToClientID(client_id_int),\n                  db_utils.IntToFlowID(flow_id_int))\n      next_requests[flow_key] = next_request\n    return next_requests", "response": "Reads and locks the next_request_to_process for a number of flows."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nread locks and updates completed requests.", "response": "def _ReadLockAndUpdateCompletedRequests(self, request_keys, response_counts,\n                                          cursor):\n    \"\"\"Reads, locks, and updates completed requests.\"\"\"\n\n    condition_template = \"\"\"\n      (flow_requests.client_id = %s AND\n       flow_requests.flow_id = %s AND\n       flow_requests.request_id = %s AND\n       responses_expected = %s)\"\"\"\n\n    args = []\n    conditions = []\n    completed_requests = {}\n\n    for request_key in request_keys:\n      client_id, flow_id, request_id = request_key\n      if request_key in response_counts:\n        conditions.append(condition_template)\n        args.append(db_utils.ClientIDToInt(client_id))\n        args.append(db_utils.FlowIDToInt(flow_id))\n        args.append(request_id)\n        args.append(response_counts[request_key])\n\n    if not args:\n      return completed_requests\n\n    query = \"\"\"\n      SELECT client_id, flow_id, request_id, request\n      FROM flow_requests\n      WHERE ({conditions}) AND NOT needs_processing\n      FOR UPDATE\n    \"\"\"\n\n    query = query.format(conditions=\" OR \".join(conditions))\n    cursor.execute(query, args)\n    for client_id_int, flow_id_int, request_id, request in cursor.fetchall():\n      request_key = (db_utils.IntToClientID(client_id_int),\n                     db_utils.IntToFlowID(flow_id_int), request_id)\n      r = rdf_flow_objects.FlowRequest.FromSerializedString(request)\n      completed_requests[request_key] = r\n\n    query = \"\"\"\n    UPDATE flow_requests\n    SET needs_processing = TRUE\n    WHERE ({conditions}) AND NOT needs_processing\n    \"\"\"\n    query = query.format(conditions=\" OR \".join(conditions))\n    cursor.execute(query, args)\n\n    return completed_requests"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _UpdateRequestsAndScheduleFPRs(self, responses, cursor=None):\n\n    request_keys = set(\n        (r.client_id, r.flow_id, r.request_id) for r in responses)\n    flow_keys = set((r.client_id, r.flow_id) for r in responses)\n\n    response_counts = self._ReadFlowResponseCounts(request_keys, cursor)\n\n    next_requests = self._ReadAndLockNextRequestsToProcess(flow_keys, cursor)\n\n    completed_requests = self._ReadLockAndUpdateCompletedRequests(\n        request_keys, response_counts, cursor)\n\n    if not completed_requests:\n      return completed_requests\n\n    fprs_to_write = []\n    for request_key, r in iteritems(completed_requests):\n      client_id, flow_id, request_id = request_key\n      if next_requests[(client_id, flow_id)] == request_id:\n        fprs_to_write.append(\n            rdf_flows.FlowProcessingRequest(\n                client_id=r.client_id,\n                flow_id=r.flow_id,\n                delivery_time=r.start_time))\n\n    if fprs_to_write:\n      self._WriteFlowProcessingRequests(fprs_to_write, cursor)\n\n    return completed_requests", "response": "Updates requests and writes FlowProcessingRequests if needed."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nwrites FlowMessages and updates corresponding requests.", "response": "def WriteFlowResponses(self, responses):\n    \"\"\"Writes FlowMessages and updates corresponding requests.\"\"\"\n\n    if not responses:\n      return\n\n    for batch in collection.Batch(responses, self._WRITE_ROWS_BATCH_SIZE):\n\n      self._WriteFlowResponsesAndExpectedUpdates(batch)\n\n      completed_requests = self._UpdateRequestsAndScheduleFPRs(batch)\n\n      if completed_requests:\n        self._DeleteClientActionRequest(completed_requests)"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef DeleteFlowRequests(self, requests, cursor=None):\n    if not requests:\n      return\n\n    conditions = []\n    args = []\n    for r in requests:\n      conditions.append(\"(client_id=%s AND flow_id=%s AND request_id=%s)\")\n      args.append(db_utils.ClientIDToInt(r.client_id))\n      args.append(db_utils.FlowIDToInt(r.flow_id))\n      args.append(r.request_id)\n\n    req_query = \"DELETE FROM flow_requests WHERE \" + \" OR \".join(conditions)\n    res_query = \"DELETE FROM flow_responses WHERE \" + \" OR \".join(conditions)\n\n    cursor.execute(res_query, args)\n    cursor.execute(req_query, args)", "response": "Deletes a list of flow requests from the database."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef DeleteAllFlowRequestsAndResponses(self, client_id, flow_id, cursor=None):\n    args = [db_utils.ClientIDToInt(client_id), db_utils.FlowIDToInt(flow_id)]\n    res_query = \"DELETE FROM flow_responses WHERE client_id=%s AND flow_id=%s\"\n    cursor.execute(res_query, args)\n    req_query = \"DELETE FROM flow_requests WHERE client_id=%s AND flow_id=%s\"\n    cursor.execute(req_query, args)", "response": "Deletes all requests and responses for a given flow from the database."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef ReadFlowRequestsReadyForProcessing(self,\n                                         client_id,\n                                         flow_id,\n                                         next_needed_request,\n                                         cursor=None):\n    \"\"\"Reads all requests for a flow that can be processed by the worker.\"\"\"\n    query = (\"SELECT request, needs_processing, responses_expected, \"\n             \"UNIX_TIMESTAMP(timestamp) \"\n             \"FROM flow_requests \"\n             \"WHERE client_id=%s AND flow_id=%s\")\n    args = [db_utils.ClientIDToInt(client_id), db_utils.FlowIDToInt(flow_id)]\n    cursor.execute(query, args)\n\n    requests = {}\n    for req, needs_processing, responses_expected, ts in cursor.fetchall():\n      if not needs_processing:\n        continue\n\n      request = rdf_flow_objects.FlowRequest.FromSerializedString(req)\n      request.needs_processing = needs_processing\n      request.nr_responses_expected = responses_expected\n      request.timestamp = mysql_utils.TimestampToRDFDatetime(ts)\n      requests[request.request_id] = request\n\n    query = (\"SELECT response, status, iterator, UNIX_TIMESTAMP(timestamp) \"\n             \"FROM flow_responses \"\n             \"WHERE client_id=%s AND flow_id=%s\")\n    cursor.execute(query, args)\n\n    responses = {}\n    for res, status, iterator, ts in cursor.fetchall():\n      if status:\n        response = rdf_flow_objects.FlowStatus.FromSerializedString(status)\n      elif iterator:\n        response = rdf_flow_objects.FlowIterator.FromSerializedString(iterator)\n      else:\n        response = rdf_flow_objects.FlowResponse.FromSerializedString(res)\n      response.timestamp = mysql_utils.TimestampToRDFDatetime(ts)\n      responses.setdefault(response.request_id, []).append(response)\n\n    res = {}\n    while next_needed_request in requests:\n      req = requests[next_needed_request]\n      sorted_responses = sorted(\n          responses.get(next_needed_request, []), key=lambda r: r.response_id)\n      res[req.request_id] = (req, sorted_responses)\n      next_needed_request += 1\n\n    return res", "response": "Reads all requests that can be processed by the worker."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreleasing a flow that the worker was processing.", "response": "def ReleaseProcessedFlow(self, flow_obj, cursor=None):\n    \"\"\"Releases a flow that the worker was processing to the database.\"\"\"\n\n    update_query = \"\"\"\n    UPDATE flows\n    LEFT OUTER JOIN (\n      SELECT client_id, flow_id, needs_processing\n      FROM flow_requests\n      WHERE\n        client_id = %(client_id)s AND\n        flow_id = %(flow_id)s AND\n        request_id = %(next_request_to_process)s AND\n        needs_processing\n    ) AS needs_processing\n    ON\n      flows.client_id = needs_processing.client_id AND\n      flows.flow_id = needs_processing.flow_id\n    SET\n      flows.flow = %(flow)s,\n      flows.processing_on = NULL,\n      flows.processing_since = NULL,\n      flows.processing_deadline = NULL,\n      flows.next_request_to_process = %(next_request_to_process)s,\n      flows.flow_state = %(flow_state)s,\n      flows.user_cpu_time_used_micros = %(user_cpu_time_used_micros)s,\n      flows.system_cpu_time_used_micros = %(system_cpu_time_used_micros)s,\n      flows.network_bytes_sent = %(network_bytes_sent)s,\n      flows.num_replies_sent = %(num_replies_sent)s,\n      flows.last_update = NOW(6)\n    WHERE\n      flows.client_id = %(client_id)s AND\n      flows.flow_id = %(flow_id)s AND (\n        needs_processing.needs_processing = FALSE OR\n        needs_processing.needs_processing IS NULL)\n    \"\"\"\n\n    clone = flow_obj.Copy()\n    clone.processing_on = None\n    clone.processing_since = None\n    clone.processing_deadline = None\n    args = {\n        \"client_id\":\n            db_utils.ClientIDToInt(flow_obj.client_id),\n        \"flow\":\n            clone.SerializeToString(),\n        \"flow_id\":\n            db_utils.FlowIDToInt(flow_obj.flow_id),\n        \"flow_state\":\n            int(clone.flow_state),\n        \"network_bytes_sent\":\n            flow_obj.network_bytes_sent,\n        \"next_request_to_process\":\n            flow_obj.next_request_to_process,\n        \"num_replies_sent\":\n            flow_obj.num_replies_sent,\n        \"system_cpu_time_used_micros\":\n            db_utils.SecondsToMicros(flow_obj.cpu_time_used.system_cpu_time),\n        \"user_cpu_time_used_micros\":\n            db_utils.SecondsToMicros(flow_obj.cpu_time_used.user_cpu_time),\n    }\n    rows_updated = cursor.execute(update_query, args)\n    return rows_updated == 1"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef ReadFlowProcessingRequests(self, cursor=None):\n    query = (\"SELECT request, UNIX_TIMESTAMP(timestamp) \"\n             \"FROM flow_processing_requests\")\n    cursor.execute(query)\n\n    res = []\n    for serialized_request, ts in cursor.fetchall():\n      req = rdf_flows.FlowProcessingRequest.FromSerializedString(\n          serialized_request)\n      req.timestamp = mysql_utils.TimestampToRDFDatetime(ts)\n      res.append(req)\n    return res", "response": "Reads all flow processing requests from the database."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef AckFlowProcessingRequests(self, requests, cursor=None):\n    if not requests:\n      return\n\n    query = \"DELETE FROM flow_processing_requests WHERE \"\n\n    conditions = []\n    args = []\n    for r in requests:\n      conditions.append(\n          \"(client_id=%s AND flow_id=%s AND timestamp=FROM_UNIXTIME(%s))\")\n      args.append(db_utils.ClientIDToInt(r.client_id))\n      args.append(db_utils.FlowIDToInt(r.flow_id))\n      args.append(mysql_utils.RDFDatetimeToTimestamp(r.timestamp))\n\n    query += \" OR \".join(conditions)\n    cursor.execute(query, args)", "response": "Acknowledge a list of flow processing requests from the database."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nleasing a number of flow processing requests.", "response": "def _LeaseFlowProcessingReqests(self, cursor=None):\n    \"\"\"Leases a number of flow processing requests.\"\"\"\n    now = rdfvalue.RDFDatetime.Now()\n    expiry = now + rdfvalue.Duration(\"10m\")\n\n    query = \"\"\"\n      UPDATE flow_processing_requests\n      SET leased_until=FROM_UNIXTIME(%(expiry)s), leased_by=%(id)s\n      WHERE\n       (delivery_time IS NULL OR\n        delivery_time <= NOW(6)) AND\n       (leased_until IS NULL OR\n        leased_until < NOW(6))\n      LIMIT %(limit)s\n    \"\"\"\n\n    id_str = utils.ProcessIdString()\n    args = {\n        \"expiry\": mysql_utils.RDFDatetimeToTimestamp(expiry),\n        \"id\": id_str,\n        \"limit\": 50,\n    }\n\n    updated = cursor.execute(query, args)\n\n    if updated == 0:\n      return []\n\n    query = \"\"\"\n      SELECT UNIX_TIMESTAMP(timestamp), request\n      FROM flow_processing_requests\n      FORCE INDEX (flow_processing_requests_by_lease)\n      WHERE leased_by=%(id)s AND leased_until=FROM_UNIXTIME(%(expiry)s)\n      LIMIT %(updated)s\n    \"\"\"\n\n    args = {\n        \"expiry\": mysql_utils.RDFDatetimeToTimestamp(expiry),\n        \"id\": id_str,\n        \"updated\": updated,\n    }\n\n    cursor.execute(query, args)\n\n    res = []\n    for timestamp, request in cursor.fetchall():\n      req = rdf_flows.FlowProcessingRequest.FromSerializedString(request)\n      req.timestamp = mysql_utils.TimestampToRDFDatetime(timestamp)\n      req.leased_until = expiry\n      req.leased_by = id_str\n      res.append(req)\n\n    return res"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef RegisterFlowProcessingHandler(self, handler):\n    self.UnregisterFlowProcessingHandler()\n\n    if handler:\n      self.flow_processing_request_handler_stop = False\n      self.flow_processing_request_handler_thread = threading.Thread(\n          name=\"flow_processing_request_handler\",\n          target=self._FlowProcessingRequestHandlerLoop,\n          args=(handler,))\n      self.flow_processing_request_handler_thread.daemon = True\n      self.flow_processing_request_handler_thread.start()", "response": "Registers a handler to receive flow processing messages."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef WriteFlowResults(self, results, cursor=None):\n    query = (\"INSERT INTO flow_results \"\n             \"(client_id, flow_id, hunt_id, timestamp, payload, type, tag) \"\n             \"VALUES \")\n    templates = []\n\n    args = []\n    for r in results:\n      templates.append(\"(%s, %s, %s, FROM_UNIXTIME(%s), %s, %s, %s)\")\n      args.append(db_utils.ClientIDToInt(r.client_id))\n      args.append(db_utils.FlowIDToInt(r.flow_id))\n      if r.hunt_id:\n        args.append(db_utils.HuntIDToInt(r.hunt_id))\n      else:\n        args.append(0)\n      args.append(\n          mysql_utils.RDFDatetimeToTimestamp(rdfvalue.RDFDatetime.Now()))\n      args.append(r.payload.SerializeToString())\n      args.append(compatibility.GetName(r.payload.__class__))\n      args.append(r.tag)\n\n    query += \",\".join(templates)\n\n    try:\n      cursor.execute(query, args)\n    except MySQLdb.IntegrityError as e:\n      raise db.AtLeastOneUnknownFlowError(\n          [(r.client_id, r.flow_id) for r in results], cause=e)", "response": "Writes the given flow results to the database."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef ReadFlowResults(self,\n                      client_id,\n                      flow_id,\n                      offset,\n                      count,\n                      with_tag=None,\n                      with_type=None,\n                      with_substring=None,\n                      cursor=None):\n    \"\"\"Reads flow results of a given flow using given query options.\"\"\"\n\n    query = (\"SELECT payload, type, UNIX_TIMESTAMP(timestamp), tag \"\n             \"FROM flow_results \"\n             \"FORCE INDEX (flow_results_by_client_id_flow_id_timestamp) \"\n             \"WHERE client_id = %s AND flow_id = %s \")\n    args = [db_utils.ClientIDToInt(client_id), db_utils.FlowIDToInt(flow_id)]\n\n    if with_tag is not None:\n      query += \"AND tag = %s \"\n      args.append(with_tag)\n\n    if with_type is not None:\n      query += \"AND type = %s \"\n      args.append(with_type)\n\n    if with_substring is not None:\n      query += \"AND payload LIKE %s \"\n      args.append(\"%{}%\".format(with_substring))\n\n    query += \"ORDER BY timestamp ASC LIMIT %s OFFSET %s\"\n    args.append(count)\n    args.append(offset)\n\n    cursor.execute(query, args)\n\n    ret = []\n    for serialized_payload, payload_type, ts, tag in cursor.fetchall():\n      if payload_type in rdfvalue.RDFValue.classes:\n        payload = rdfvalue.RDFValue.classes[payload_type]()\n        payload.ParseFromString(serialized_payload)\n      else:\n        payload = rdf_objects.SerializedValueOfUnrecognizedType(\n            type_name=payload_type, value=serialized_payload)\n\n      timestamp = mysql_utils.TimestampToRDFDatetime(ts)\n      result = rdf_flow_objects.FlowResult(payload=payload, timestamp=timestamp)\n      if tag:\n        result.tag = tag\n\n      ret.append(result)\n\n    return ret", "response": "Reads all flow results of a given flow."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ncount the number of flow results of a given client and flow.", "response": "def CountFlowResults(self,\n                       client_id,\n                       flow_id,\n                       with_tag=None,\n                       with_type=None,\n                       cursor=None):\n    \"\"\"Counts flow results of a given flow using given query options.\"\"\"\n    query = (\"SELECT COUNT(*) \"\n             \"FROM flow_results \"\n             \"FORCE INDEX (flow_results_by_client_id_flow_id_timestamp) \"\n             \"WHERE client_id = %s AND flow_id = %s \")\n    args = [db_utils.ClientIDToInt(client_id), db_utils.FlowIDToInt(flow_id)]\n\n    if with_tag is not None:\n      query += \"AND tag = %s \"\n      args.append(with_tag)\n\n    if with_type is not None:\n      query += \"AND type = %s \"\n      args.append(with_type)\n\n    cursor.execute(query, args)\n    return cursor.fetchone()[0]"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef CountFlowResultsByType(self, client_id, flow_id, cursor=None):\n    query = (\"SELECT type, COUNT(*) FROM flow_results \"\n             \"FORCE INDEX (flow_results_by_client_id_flow_id_timestamp) \"\n             \"WHERE client_id = %s AND flow_id = %s \"\n             \"GROUP BY type\")\n    args = [db_utils.ClientIDToInt(client_id), db_utils.FlowIDToInt(flow_id)]\n\n    cursor.execute(query, args)\n\n    return dict(cursor.fetchall())", "response": "Counts the number of flow results grouped by result type."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreads flow log entries of a given flow.", "response": "def ReadFlowLogEntries(self,\n                         client_id,\n                         flow_id,\n                         offset,\n                         count,\n                         with_substring=None,\n                         cursor=None):\n    \"\"\"Reads flow log entries of a given flow using given query options.\"\"\"\n\n    query = (\"SELECT message, UNIX_TIMESTAMP(timestamp) \"\n             \"FROM flow_log_entries \"\n             \"FORCE INDEX (flow_log_entries_by_flow) \"\n             \"WHERE client_id = %s AND flow_id = %s \")\n    args = [db_utils.ClientIDToInt(client_id), db_utils.FlowIDToInt(flow_id)]\n\n    if with_substring is not None:\n      query += \"AND message LIKE %s \"\n      args.append(\"%{}%\".format(with_substring))\n\n    query += \"ORDER BY log_id ASC LIMIT %s OFFSET %s\"\n    args.append(count)\n    args.append(offset)\n\n    cursor.execute(query, args)\n\n    ret = []\n    for message, timestamp in cursor.fetchall():\n      ret.append(\n          rdf_flow_objects.FlowLogEntry(\n              message=message,\n              timestamp=mysql_utils.TimestampToRDFDatetime(timestamp)))\n\n    return ret"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns the number of flow log entries of a given client.", "response": "def CountFlowLogEntries(self, client_id, flow_id, cursor=None):\n    \"\"\"Returns number of flow log entries of a given flow.\"\"\"\n\n    query = (\"SELECT COUNT(*) \"\n             \"FROM flow_log_entries \"\n             \"FORCE INDEX (flow_log_entries_by_flow) \"\n             \"WHERE client_id = %s AND flow_id = %s \")\n    args = [db_utils.ClientIDToInt(client_id), db_utils.FlowIDToInt(flow_id)]\n\n    cursor.execute(query, args)\n    return cursor.fetchone()[0]"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nwrite a list of flow output plugin log entries.", "response": "def WriteFlowOutputPluginLogEntries(self, entries, cursor=None):\n    \"\"\"Writes flow output plugin log entries for a given flow.\"\"\"\n    query = (\"INSERT INTO flow_output_plugin_log_entries \"\n             \"(client_id, flow_id, hunt_id, output_plugin_id, \"\n             \"log_entry_type, message) \"\n             \"VALUES \")\n    templates = []\n    args = []\n    for entry in entries:\n      templates.append(\"(%s, %s, %s, %s, %s, %s)\")\n      args.append(db_utils.ClientIDToInt(entry.client_id))\n      args.append(db_utils.FlowIDToInt(entry.flow_id))\n      if entry.hunt_id:\n        args.append(db_utils.HuntIDToInt(entry.hunt_id))\n      else:\n        args.append(0)\n      args.append(db_utils.OutputPluginIDToInt(entry.output_plugin_id))\n      args.append(int(entry.log_entry_type))\n      args.append(entry.message)\n\n    query += \",\".join(templates)\n\n    try:\n      cursor.execute(query, args)\n    except MySQLdb.IntegrityError as e:\n      raise db.AtLeastOneUnknownFlowError(\n          [(entry.client_id, entry.flow_id) for entry in entries], cause=e)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef ReadFlowOutputPluginLogEntries(self,\n                                     client_id,\n                                     flow_id,\n                                     output_plugin_id,\n                                     offset,\n                                     count,\n                                     with_type=None,\n                                     cursor=None):\n    \"\"\"Reads flow output plugin log entries.\"\"\"\n    query = (\"SELECT log_entry_type, message, UNIX_TIMESTAMP(timestamp) \"\n             \"FROM flow_output_plugin_log_entries \"\n             \"FORCE INDEX (flow_output_plugin_log_entries_by_flow) \"\n             \"WHERE client_id = %s AND flow_id = %s AND output_plugin_id = %s \")\n    args = [\n        db_utils.ClientIDToInt(client_id),\n        db_utils.FlowIDToInt(flow_id),\n        db_utils.OutputPluginIDToInt(output_plugin_id)\n    ]\n\n    if with_type is not None:\n      query += \"AND log_entry_type = %s \"\n      args.append(int(with_type))\n\n    query += \"ORDER BY log_id ASC LIMIT %s OFFSET %s\"\n    args.append(count)\n    args.append(offset)\n\n    cursor.execute(query, args)\n\n    ret = []\n    for log_entry_type, message, timestamp in cursor.fetchall():\n      ret.append(\n          rdf_flow_objects.FlowOutputPluginLogEntry(\n              client_id=client_id,\n              flow_id=flow_id,\n              output_plugin_id=output_plugin_id,\n              log_entry_type=log_entry_type,\n              message=message,\n              timestamp=mysql_utils.TimestampToRDFDatetime(timestamp)))\n\n    return ret", "response": "Reads flow output plugin log entries."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef CountFlowOutputPluginLogEntries(self,\n                                      client_id,\n                                      flow_id,\n                                      output_plugin_id,\n                                      with_type=None,\n                                      cursor=None):\n    \"\"\"Returns number of flow output plugin log entries of a given flow.\"\"\"\n    query = (\"SELECT COUNT(*) \"\n             \"FROM flow_output_plugin_log_entries \"\n             \"FORCE INDEX (flow_output_plugin_log_entries_by_flow) \"\n             \"WHERE client_id = %s AND flow_id = %s AND output_plugin_id = %s \")\n    args = [\n        db_utils.ClientIDToInt(client_id),\n        db_utils.FlowIDToInt(flow_id), output_plugin_id\n    ]\n\n    if with_type is not None:\n      query += \"AND log_entry_type = %s\"\n      args.append(int(with_type))\n\n    cursor.execute(query, args)\n    return cursor.fetchone()[0]", "response": "Counts the number of flow output plugin log entries of a given flow."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nevaluating the rules held in the rule set.", "response": "def Evaluate(self, client_obj):\n    \"\"\"Evaluates rules held in the rule set.\n\n    Args:\n      client_obj: Either an aff4 client object or a client_info dict as returned\n        by ReadFullInfoClient if the relational db is used for reading.\n\n    Returns:\n      A bool value of the evaluation.\n\n    Raises:\n      ValueError: The match mode is of unknown value.\n    \"\"\"\n    if self.match_mode == ForemanClientRuleSet.MatchMode.MATCH_ALL:\n      quantifier = all\n    elif self.match_mode == ForemanClientRuleSet.MatchMode.MATCH_ANY:\n      quantifier = any\n    else:\n      raise ValueError(\"Unexpected match mode value: %s\" % self.match_mode)\n\n    return quantifier(rule.Evaluate(client_obj) for rule in self.rules)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nwriting a GRR user object for a given username.", "response": "def WriteGRRUser(self,\n                   username,\n                   password=None,\n                   ui_mode=None,\n                   canary_mode=None,\n                   user_type=None):\n    \"\"\"Writes user object for a user with a given name.\"\"\"\n    u = self.users.setdefault(username, rdf_objects.GRRUser(username=username))\n    if password is not None:\n      u.password = password\n    if ui_mode is not None:\n      u.ui_mode = ui_mode\n    if canary_mode is not None:\n      u.canary_mode = canary_mode\n    if user_type is not None:\n      u.user_type = user_type"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreads a GRR user object corresponding to a given name.", "response": "def ReadGRRUser(self, username):\n    \"\"\"Reads a user object corresponding to a given name.\"\"\"\n    try:\n      return self.users[username].Copy()\n    except KeyError:\n      raise db.UnknownGRRUserError(username)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef ReadGRRUsers(self, offset=0, count=None):\n    if count is None:\n      count = len(self.users)\n\n    users = sorted(self.users.values(), key=lambda user: user.username)\n    return [user.Copy() for user in users[offset:offset + count]]", "response": "Reads GRR users with optional pagination sorted by username."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef DeleteGRRUser(self, username):\n    try:\n      del self.approvals_by_username[username]\n    except KeyError:\n      pass  # No approvals to delete for this user.\n\n    for approvals in itervalues(self.approvals_by_username):\n      for approval in itervalues(approvals):\n        grants = [g for g in approval.grants if g.grantor_username != username]\n        if len(grants) != len(approval.grants):\n          approval.grants = grants\n\n    try:\n      del self.notifications_by_username[username]\n    except KeyError:\n      pass  # No notifications to delete for this user.\n\n    try:\n      del self.users[username]\n    except KeyError:\n      raise db.UnknownGRRUserError(username)", "response": "Deletes the user and all related metadata with the given username."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nwriting an approval request object.", "response": "def WriteApprovalRequest(self, approval_request):\n    \"\"\"Writes an approval request object.\"\"\"\n    approvals = self.approvals_by_username.setdefault(\n        approval_request.requestor_username, {})\n\n    approval_id = str(os.urandom(16).encode(\"hex\"))\n    cloned_request = approval_request.Copy()\n    cloned_request.timestamp = rdfvalue.RDFDatetime.Now()\n    cloned_request.approval_id = approval_id\n    approvals[approval_id] = cloned_request\n\n    return approval_id"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nread an approval request object with a given id.", "response": "def ReadApprovalRequest(self, requestor_username, approval_id):\n    \"\"\"Reads an approval request object with a given id.\"\"\"\n    try:\n      return self.approvals_by_username[requestor_username][approval_id]\n    except KeyError:\n      raise db.UnknownApprovalRequestError(\"Can't find approval with id: %s\" %\n                                           approval_id)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreading approval requests of a given type for a given user.", "response": "def ReadApprovalRequests(self,\n                           requestor_username,\n                           approval_type,\n                           subject_id=None,\n                           include_expired=False):\n    \"\"\"Reads approval requests of a given type for a given user.\"\"\"\n    now = rdfvalue.RDFDatetime.Now()\n\n    result = []\n    approvals = self.approvals_by_username.get(requestor_username, {})\n    for approval in itervalues(approvals):\n      if approval.approval_type != approval_type:\n        continue\n\n      if subject_id and approval.subject_id != subject_id:\n        continue\n\n      if not include_expired and approval.expiration_time < now:\n        continue\n\n      result.append(approval)\n\n    return result"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef GrantApproval(self, requestor_username, approval_id, grantor_username):\n    try:\n      approval = self.approvals_by_username[requestor_username][approval_id]\n      approval.grants.append(\n          rdf_objects.ApprovalGrant(\n              grantor_username=grantor_username,\n              timestamp=rdfvalue.RDFDatetime.Now()))\n    except KeyError:\n      raise db.UnknownApprovalRequestError(\"Can't find approval with id: %s\" %\n                                           approval_id)", "response": "Grants approval for a given request using given username."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef WriteUserNotification(self, notification):\n    if notification.username not in self.users:\n      raise db.UnknownGRRUserError(notification.username)\n\n    cloned_notification = notification.Copy()\n    if not cloned_notification.timestamp:\n      cloned_notification.timestamp = rdfvalue.RDFDatetime.Now()\n\n    self.notifications_by_username.setdefault(cloned_notification.username,\n                                              []).append(cloned_notification)", "response": "Writes a notification for a given user."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreads notifications scheduled for a user within a given timerange.", "response": "def ReadUserNotifications(self, username, state=None, timerange=None):\n    \"\"\"Reads notifications scheduled for a user within a given timerange.\"\"\"\n    from_time, to_time = self._ParseTimeRange(timerange)\n\n    result = []\n    for n in self.notifications_by_username.get(username, []):\n      if from_time <= n.timestamp <= to_time and (state is None or\n                                                  n.state == state):\n        result.append(n.Copy())\n\n    return sorted(result, key=lambda r: r.timestamp, reverse=True)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef UpdateUserNotifications(self, username, timestamps, state=None):\n    if not timestamps:\n      return\n\n    for n in self.notifications_by_username.get(username, []):\n      if n.timestamp in timestamps:\n        n.state = state", "response": "Updates existing user notifications."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nconverts a list of FileFinderSizeConditions to RegistryFinderConditions.", "response": "def _ConditionsToFileFinderConditions(conditions):\n  \"\"\"Converts FileFinderSizeConditions to RegistryFinderConditions.\"\"\"\n  ff_condition_type_cls = rdf_file_finder.FileFinderCondition.Type\n  result = []\n  for c in conditions:\n    if c.condition_type == RegistryFinderCondition.Type.MODIFICATION_TIME:\n      result.append(\n          rdf_file_finder.FileFinderCondition(\n              condition_type=ff_condition_type_cls.MODIFICATION_TIME,\n              modification_time=c.modification_time))\n    elif c.condition_type == RegistryFinderCondition.Type.VALUE_LITERAL_MATCH:\n      result.append(\n          rdf_file_finder.FileFinderCondition(\n              condition_type=ff_condition_type_cls.CONTENTS_LITERAL_MATCH,\n              contents_literal_match=c.value_literal_match))\n    elif c.condition_type == RegistryFinderCondition.Type.VALUE_REGEX_MATCH:\n      result.append(\n          rdf_file_finder.FileFinderCondition(\n              condition_type=ff_condition_type_cls.CONTENTS_REGEX_MATCH,\n              contents_regex_match=c.value_regex_match))\n    elif c.condition_type == RegistryFinderCondition.Type.SIZE:\n      result.append(\n          rdf_file_finder.FileFinderCondition(\n              condition_type=ff_condition_type_cls.SIZE, size=c.size))\n    else:\n      raise ValueError(\"Unknown condition type: %s\" % c.condition_type)\n\n  return result"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef StripTypeInfo(rendered_data):\n\n  if isinstance(rendered_data, (list, tuple)):\n    return [StripTypeInfo(d) for d in rendered_data]\n  elif isinstance(rendered_data, dict):\n    if \"value\" in rendered_data and \"type\" in rendered_data:\n      return StripTypeInfo(rendered_data[\"value\"])\n    else:\n      result = {}\n      for k, v in iteritems(rendered_data):\n        result[k] = StripTypeInfo(v)\n      return result\n  else:\n    return rendered_data", "response": "Strips type information from rendered data. Useful for debugging."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef RenderValue(value, limit_lists=-1):\n\n  if value is None:\n    return None\n\n  renderer = ApiValueRenderer.GetRendererForValueOrClass(\n      value, limit_lists=limit_lists)\n  return renderer.RenderValue(value)", "response": "Render given RDFValue as plain old python objects."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef GetRendererForValueOrClass(cls, value, limit_lists=-1):\n\n    if inspect.isclass(value):\n      value_cls = value\n    else:\n      value_cls = value.__class__\n\n    cache_key = \"%s_%d\" % (value_cls.__name__, limit_lists)\n    try:\n      renderer_cls = cls._renderers_cache[cache_key]\n    except KeyError:\n      candidates = []\n      for candidate in itervalues(ApiValueRenderer.classes):\n        if candidate.value_class:\n          candidate_class = candidate.value_class\n        else:\n          continue\n\n        if inspect.isclass(value):\n          if issubclass(value_cls, candidate_class):\n            candidates.append((candidate, candidate_class))\n        else:\n          if isinstance(value, candidate_class):\n            candidates.append((candidate, candidate_class))\n\n      if not candidates:\n        raise RuntimeError(\n            \"No renderer found for value %s.\" % value.__class__.__name__)\n\n      candidates = sorted(\n          candidates, key=lambda candidate: len(candidate[1].mro()))\n      renderer_cls = candidates[-1][0]\n      cls._renderers_cache[cache_key] = renderer_cls\n\n    return renderer_cls(limit_lists=limit_lists)", "response": "Returns a renderer corresponding to a given value and rendering args."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef BuildDefaultValue(self, value_cls):\n    try:\n      return value_cls()\n    except Exception as e:  # pylint: disable=broad-except\n      logging.exception(e)\n      raise DefaultValueError(\n          \"Can't create default for value %s: %s\" % (value_cls.__name__, e))", "response": "Builds a default value of a given class."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef BuildTypeDescriptor(self, value_cls):\n    result = ApiRDFValueDescriptor(\n        name=value_cls.__name__,\n        parents=[klass.__name__ for klass in value_cls.__mro__],\n        doc=value_cls.__doc__ or \"\",\n        kind=\"PRIMITIVE\")\n\n    result.default = self.BuildDefaultValue(value_cls)\n\n    return result", "response": "Builds a type descriptor for a given value class."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef RenderPayload(self, result, value):\n    if \"args_rdf_name\" in result:\n      result[\"payload_type\"] = result[\"args_rdf_name\"]\n      del result[\"args_rdf_name\"]\n\n    if \"args\" in result:\n      result[\"payload\"] = self._PassThrough(value.payload)\n      del result[\"args\"]\n\n    return result", "response": "Renders GrrMessage payload and renames args_rdf_name field."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef AdjustDescriptor(self, fields):\n\n    for f in fields:\n      if f.name == \"args_rdf_name\":\n        f.name = \"payload_type\"\n\n      if f.name == \"args\":\n        f.name = \"payload\"\n\n    return fields", "response": "Adjusts the names of the fields in the descriptor to include the payload_type and payload_type_rdf_name fields."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef PlistValueToPlainValue(plist):\n\n  if isinstance(plist, dict):\n    ret_value = dict()\n    for key, value in iteritems(plist):\n      ret_value[key] = PlistValueToPlainValue(value)\n    return ret_value\n  elif isinstance(plist, list):\n    return [PlistValueToPlainValue(value) for value in plist]\n  elif isinstance(plist, datetime.datetime):\n    return (calendar.timegm(plist.utctimetuple()) * 1000000) + plist.microsecond\n  return plist", "response": "Takes the plist contents generated by binplist and returns a plain dict."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nadding a path component to the current attribute.", "response": "def AddAttributePath(self, **_):\n    \"\"\"Adds a path component to the current attribute.\"\"\"\n    attribute_path = self.current_expression.attribute\n    if not attribute_path:\n      attribute_path = []\n\n    attribute_path.append(self.string)\n    self.current_expression.SetAttribute(attribute_path)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _AtNonLeaf(self, attr_value, path):\n    if isinstance(attr_value, dict):\n      for value in self.Expand(attr_value, path[1:]):\n        yield value\n    else:\n      for v in objectfilter.ValueExpander._AtNonLeaf(self, attr_value, path):\n        yield v", "response": "Makes dictionaries expandable when dealing with plists."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn a list of MetricMetadata for GRR server components.", "response": "def GetMetadata():\n  \"\"\"Returns a list of MetricMetadata for GRR server components.\"\"\"\n  return [\n      # GRR user-management metrics.\n      stats_utils.CreateEventMetadata(\n          \"acl_check_time\", fields=[(\"check_type\", str)]),\n      stats_utils.CreateCounterMetadata(\n          \"approval_searches\",\n          fields=[(\"reason_presence\", str), (\"source\", str)]),\n\n      # Cronjob metrics.\n      stats_utils.CreateCounterMetadata(\"cron_internal_error\"),\n      stats_utils.CreateCounterMetadata(\n          \"cron_job_failure\", fields=[(\"cron_job_id\", str)]),\n      stats_utils.CreateCounterMetadata(\n          \"cron_job_timeout\", fields=[(\"cron_job_id\", str)]),\n      stats_utils.CreateEventMetadata(\n          \"cron_job_latency\", fields=[(\"cron_job_id\", str)]),\n\n      # Access-control metrics.\n      stats_utils.CreateCounterMetadata(\"grr_expired_tokens\"),\n\n      # Datastore metrics.\n      stats_utils.CreateCounterMetadata(\"grr_commit_failure\"),\n      stats_utils.CreateCounterMetadata(\"datastore_retries\"),\n      stats_utils.CreateGaugeMetadata(\n          \"datastore_size\",\n          int,\n          docstring=\"Size of data store in bytes\",\n          units=\"BYTES\"),\n      stats_utils.CreateCounterMetadata(\"grr_task_retransmission_count\"),\n      stats_utils.CreateCounterMetadata(\"grr_task_ttl_expired_count\"),\n      stats_utils.CreateEventMetadata(\n          \"db_request_latency\",\n          fields=[(\"call\", str)],\n          bins=[0.05 * 1.2**x for x in range(30)]),  # 50ms to ~10 secs\n      stats_utils.CreateCounterMetadata(\n          \"db_request_errors\", fields=[(\"call\", str), (\"type\", str)]),\n\n      # Threadpool metrics.\n      stats_utils.CreateGaugeMetadata(\n          \"threadpool_outstanding_tasks\", int, fields=[(\"pool_name\", str)]),\n      stats_utils.CreateGaugeMetadata(\n          \"threadpool_threads\", int, fields=[(\"pool_name\", str)]),\n      stats_utils.CreateGaugeMetadata(\n          \"threadpool_cpu_use\", float, fields=[(\"pool_name\", str)]),\n      stats_utils.CreateCounterMetadata(\n          \"threadpool_task_exceptions\", fields=[(\"pool_name\", str)]),\n      stats_utils.CreateEventMetadata(\n          \"threadpool_working_time\", fields=[(\"pool_name\", str)]),\n      stats_utils.CreateEventMetadata(\n          \"threadpool_queueing_time\", fields=[(\"pool_name\", str)]),\n\n      # Worker and flow-related metrics.\n      stats_utils.CreateCounterMetadata(\"grr_flows_stuck\"),\n      stats_utils.CreateCounterMetadata(\n          \"worker_bad_flow_objects\", fields=[(\"type\", str)]),\n      stats_utils.CreateCounterMetadata(\n          \"worker_session_errors\", fields=[(\"type\", str)]),\n      stats_utils.CreateCounterMetadata(\n          \"worker_flow_lock_error\",\n          docstring=\"Worker lock failures. We expect these to be high when the \"\n          \"systemis idle.\"),\n      stats_utils.CreateEventMetadata(\n          \"worker_flow_processing_time\", fields=[(\"flow\", str)]),\n      stats_utils.CreateEventMetadata(\"worker_time_to_retrieve_notifications\"),\n      stats_utils.CreateCounterMetadata(\"grr_flow_completed_count\"),\n      stats_utils.CreateCounterMetadata(\"grr_flow_errors\"),\n      stats_utils.CreateCounterMetadata(\"grr_flow_invalid_flow_count\"),\n      stats_utils.CreateCounterMetadata(\"grr_request_retransmission_count\"),\n      stats_utils.CreateCounterMetadata(\"grr_response_out_of_order\"),\n      stats_utils.CreateCounterMetadata(\"grr_unique_clients\"),\n      stats_utils.CreateCounterMetadata(\"grr_worker_states_run\"),\n      stats_utils.CreateCounterMetadata(\"grr_well_known_flow_requests\"),\n      stats_utils.CreateCounterMetadata(\"flow_starts\", fields=[(\"flow\", str)]),\n      stats_utils.CreateCounterMetadata(\"flow_errors\", fields=[(\"flow\", str)]),\n      stats_utils.CreateCounterMetadata(\n          \"flow_completions\", fields=[(\"flow\", str)]),\n      stats_utils.CreateCounterMetadata(\n          \"well_known_flow_requests\", fields=[(\"flow\", str)]),\n      stats_utils.CreateCounterMetadata(\n          \"well_known_flow_errors\", fields=[(\"flow\", str)]),\n      stats_utils.CreateEventMetadata(\"fleetspeak_last_ping_latency_millis\"),\n\n      # Hunt-related metrics.\n      stats_utils.CreateCounterMetadata(\n          \"hunt_output_plugin_verifications\", fields=[(\"status\", str)]),\n      stats_utils.CreateCounterMetadata(\n          \"hunt_output_plugin_verification_errors\"),\n      stats_utils.CreateCounterMetadata(\n          \"hunt_output_plugin_errors\", fields=[(\"plugin\", str)]),\n      stats_utils.CreateCounterMetadata(\n          \"hunt_results_ran_through_plugin\", fields=[(\"plugin\", str)]),\n      stats_utils.CreateCounterMetadata(\"hunt_results_compacted\"),\n      stats_utils.CreateCounterMetadata(\n          \"hunt_results_compaction_locking_errors\"),\n      stats_utils.CreateCounterMetadata(\"hunt_results_added\"),\n\n      # Metric used to identify the master in a distributed server setup.\n      stats_utils.CreateGaugeMetadata(\"is_master\", int),\n\n      # GRR-API metrics.\n      stats_utils.CreateEventMetadata(\n          \"api_method_latency\",\n          fields=[(\"method_name\", str), (\"protocol\", str), (\"status\", str)]),\n      stats_utils.CreateEventMetadata(\n          \"api_access_probe_latency\",\n          fields=[(\"method_name\", str), (\"protocol\", str), (\"status\", str)]),\n\n      # Client-related metrics.\n      stats_utils.CreateCounterMetadata(\"grr_client_crashes\"),\n      stats_utils.CreateCounterMetadata(\n          \"client_pings_by_label\", fields=[(\"label\", str)]),\n\n      # Metrics specific to GRR frontends.\n      stats_utils.CreateGaugeMetadata(\n          \"frontend_active_count\", int, fields=[(\"source\", str)]),\n      stats_utils.CreateGaugeMetadata(\"frontend_max_active_count\", int),\n      stats_utils.CreateCounterMetadata(\n          \"frontend_http_requests\", fields=[(\"action\", str), (\"protocol\",\n                                                              str)]),\n      stats_utils.CreateCounterMetadata(\n          \"frontend_in_bytes\", fields=[(\"source\", str)]),\n      stats_utils.CreateCounterMetadata(\n          \"frontend_out_bytes\", fields=[(\"source\", str)]),\n      stats_utils.CreateCounterMetadata(\n          \"frontend_request_count\", fields=[(\"source\", str)]),\n      stats_utils.CreateCounterMetadata(\n          \"frontend_inactive_request_count\", fields=[(\"source\", str)]),\n      stats_utils.CreateEventMetadata(\n          \"frontend_request_latency\", fields=[(\"source\", str)]),\n      stats_utils.CreateEventMetadata(\"grr_frontendserver_handle_time\"),\n      stats_utils.CreateCounterMetadata(\"grr_frontendserver_handle_num\"),\n      stats_utils.CreateGaugeMetadata(\"grr_frontendserver_client_cache_size\",\n                                      int),\n      stats_utils.CreateCounterMetadata(\"grr_messages_sent\"),\n      stats_utils.CreateCounterMetadata(\n          \"grr_pub_key_cache\", fields=[(\"type\", str)]),\n  ]"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef Parse(text):\n  precondition.AssertType(text, Text)\n\n  if compatibility.PY2:\n    text = text.encode(\"utf-8\")\n\n  return yaml.safe_load(text)", "response": "Parses a YAML source into a Python object."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef ParseMany(text):\n  precondition.AssertType(text, Text)\n\n  if compatibility.PY2:\n    text = text.encode(\"utf-8\")\n\n  return list(yaml.safe_load_all(text))", "response": "Parses many YAML documents into a list of Python objects."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreads a Python object stored in a specified YAML file.", "response": "def ReadFromPath(filepath):\n  \"\"\"Reads a Python object stored in a specified YAML file.\n\n  Args:\n    filepath: A filepath to the YAML file.\n\n  Returns:\n    A Python data structure corresponding to the YAML in the given file.\n  \"\"\"\n  with io.open(filepath, mode=\"r\", encoding=\"utf-8\") as filedesc:\n    return ReadFromFile(filedesc)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef ReadManyFromPath(filepath):\n  with io.open(filepath, mode=\"r\", encoding=\"utf-8\") as filedesc:\n    return ReadManyFromFile(filedesc)", "response": "Reads a Python object stored in a specified YAML file."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nserialize and writes given Python object to the specified YAML file.", "response": "def WriteToPath(obj, filepath):\n  \"\"\"Serializes and writes given Python object to the specified YAML file.\n\n  Args:\n    obj: A Python object to serialize.\n    filepath: A path to the file into which the object is to be written.\n  \"\"\"\n  with io.open(filepath, mode=\"w\", encoding=\"utf-8\") as filedesc:\n    WriteToFile(obj, filedesc)"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nserializes and writes given Python objects to a multi - document YAML file.", "response": "def WriteManyToPath(objs, filepath):\n  \"\"\"Serializes and writes given Python objects to a multi-document YAML file.\n\n  Args:\n    objs: An iterable of Python objects to serialize.\n    filepath: A path to the file into which the object is to be written.\n  \"\"\"\n  with io.open(filepath, mode=\"w\", encoding=\"utf-8\") as filedesc:\n    WriteManyToFile(objs, filedesc)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncreate Windows Registry executable paths detector.", "response": "def CreateWindowsRegistryExecutablePathsDetector(vars_map=None):\n  \"\"\"Creates Windows paths detector.\n\n  Commandline strings can be space separated and contain options.\n  e.g. C:\\\\Program Files\\\\ACME Corporation\\\\wiz.exe /quiet /blah\n\n  See here for microsoft doco on commandline parsing:\n  http://msdn.microsoft.com/en-us/library/windows/desktop/ms682425(v=vs.85).aspx\n\n  Args:\n    vars_map: Environment variables mapping. Default is None.\n\n  Returns:\n    A detector (core.Detector instance).\n  \"\"\"\n  return core.Detector(\n      extractors=[RunDllExtractor(), ExecutableExtractor()],\n      post_processors=[EnvVarsPostProcessor(vars_map or {})],)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef DetectExecutablePaths(source_values, vars_map=None):\n  detector = CreateWindowsRegistryExecutablePathsDetector(vars_map=vars_map)\n\n  for source_value in source_values:\n    for result in detector.Detect(source_value):\n      yield result", "response": "Detects paths in a list of Windows Registry strings."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef Extract(self, components):\n\n    rundll_index = -1\n    for index, component in enumerate(components):\n      if component.lower().endswith(\"rundll32.exe\"):\n        rundll_index = index\n\n    if rundll_index == -1:\n      return []\n\n    components = components[(rundll_index + 1):]\n\n    # We expect components after \"rundll32.exe\" to point at a DLL and a\n    # function. For example:\n    # rundll32.exe \"C:\\Windows\\system32\\advpack.dll\",DelNodeRunDLL32\n    last_component = components[-1].rsplit(\",\", 1)[0]\n\n    extracted_path = \" \".join(components[0:-1] + [last_component])\n    return [extracted_path]", "response": "Extracts interesting paths from a given list of components."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nextracting interesting paths from a given list of components.", "response": "def Extract(self, components):\n    \"\"\"Extracts interesting paths from a given path.\n\n    Args:\n      components: Source string represented as a list of components.\n\n    Returns:\n      A list of extracted paths (as strings).\n    \"\"\"\n\n    for index, component in enumerate(components):\n      if component.lower().endswith(self.EXECUTABLE_EXTENSIONS):\n        extracted_path = \" \".join(components[0:index + 1])\n        return [extracted_path]\n\n    return []"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nprocess a given path and returns a list of paths with environment variables replaced with their respective values.", "response": "def Process(self, path):\n    \"\"\"Processes a given path.\n\n    Args:\n      path: Path (as a string) to post-process.\n\n    Returns:\n      A list of paths with environment variables replaced with their\n      values. If the mapping had a list of values for a particular variable,\n      instead of just one value, then all possible replacements will be\n      returned.\n    \"\"\"\n    path = re.sub(self.SYSTEMROOT_RE, r\"%systemroot%\", path, count=1)\n    path = re.sub(self.SYSTEM32_RE, r\"%systemroot%\\\\system32\", path, count=1)\n\n    matches_iter = self.WIN_ENVIRON_REGEX.finditer(path)\n    var_names = set(m.group(1).lower() for m in matches_iter)\n\n    results = [path]\n    for var_name in var_names:\n      try:\n        var_regex, var_value = self.vars_map[var_name]\n      except KeyError:\n        continue\n\n      if isinstance(var_value, string_types):\n        replacements = [var_value]\n      else:\n        replacements = var_value\n\n      processed_results = []\n      for result in results:\n        for repl in replacements:\n          # Using lambda here, as otherwise Python interprets \\\\f as a\n          # backreference (same applies to \\\\0 and \\\\1). When using a\n          # function as a replacement argument, backreferences are ignored.\n          # pylint: disable=cell-var-from-loop\n          processed_results.append(var_regex.sub(lambda _: repl, result))\n      results = processed_results\n\n    return results"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef CheckDataStoreAccess(self, token, subjects, requested_access=\"r\"):\n    logging.debug(\"Checking %s: %s for %s\", token, subjects, requested_access)\n    raise NotImplementedError()", "response": "This function is called by the ACLToken object when the user is authorized to access the AFF4 resources."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nwriting hunt results from a given client as part of a given hunt.", "response": "def WriteHuntResults(client_id, hunt_id, responses):\n  \"\"\"Writes hunt results from a given client as part of a given hunt.\"\"\"\n\n  if not hunt.IsLegacyHunt(hunt_id):\n    data_store.REL_DB.WriteFlowResults(responses)\n\n    hunt.StopHuntIfCPUOrNetworkLimitsExceeded(hunt_id)\n    return\n\n  hunt_id_urn = rdfvalue.RDFURN(\"hunts\").Add(hunt_id)\n\n  msgs = []\n  for response in responses:\n    if isinstance(response, rdf_flow_objects.FlowStatus):\n      continue\n\n    msgs.append(\n        rdf_flows.GrrMessage(payload=response.payload, source=client_id))\n\n  with data_store.DB.GetMutationPool() as pool:\n    for msg in msgs:\n      hunts_results.HuntResultCollection.StaticAdd(\n          hunt_id_urn.Add(\"Results\"), msg, mutation_pool=pool)\n\n    for msg in msgs:\n      multi_type_collection.MultiTypeCollection.StaticAdd(\n          hunt_id_urn.Add(\"ResultsPerType\"), msg, mutation_pool=pool)\n\n  stats_collector_instance.Get().IncrementCounter(\n      \"hunt_results_added\", delta=len(responses))"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef ProcessHuntFlowError(flow_obj,\n                         error_message=None,\n                         backtrace=None,\n                         status_msg=None):\n  \"\"\"Processes error and status message for a given hunt-induced flow.\"\"\"\n\n  if not hunt.IsLegacyHunt(flow_obj.parent_hunt_id):\n    hunt.StopHuntIfCPUOrNetworkLimitsExceeded(flow_obj.parent_hunt_id)\n    return\n\n  hunt_urn = rdfvalue.RDFURN(\"hunts\").Add(flow_obj.parent_hunt_id)\n  client_urn = rdf_client.ClientURN(flow_obj.client_id)\n\n  error = rdf_hunts.HuntError(client_id=flow_obj.client_id, backtrace=backtrace)\n  if error_message is not None:\n    error.log_message = error_message\n  with data_store.DB.GetMutationPool() as pool:\n    grr_collections.HuntErrorCollection.StaticAdd(\n        hunt_urn.Add(\"ErrorClients\"), error, mutation_pool=pool)\n    grr_collections.ClientUrnCollection.StaticAdd(\n        hunt_urn.Add(\"CompletedClients\"), client_urn, mutation_pool=pool)\n\n  if status_msg is not None:\n    with aff4.FACTORY.Open(hunt_urn, mode=\"rw\") as fd:\n      # Legacy AFF4 code expects token to be set.\n      fd.token = access_control.ACLToken(username=fd.creator)\n      fd.GetRunner().SaveResourceUsage(flow_obj.client_id, status_msg)", "response": "Processes an error message and status message for a given hunt - induced flow."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef ProcessHuntFlowDone(flow_obj, status_msg=None):\n\n  if not hunt.IsLegacyHunt(flow_obj.parent_hunt_id):\n    hunt_obj = hunt.StopHuntIfCPUOrNetworkLimitsExceeded(\n        flow_obj.parent_hunt_id)\n    hunt.CompleteHuntIfExpirationTimeReached(hunt_obj)\n    return\n\n  hunt_urn = rdfvalue.RDFURN(\"hunts\").Add(flow_obj.parent_hunt_id)\n  client_urn = rdf_client.ClientURN(flow_obj.client_id)\n\n  # Update the counter metrics separately from collections to minimize\n  # contention.\n  with aff4.FACTORY.Open(hunt_urn, mode=\"rw\") as fd:\n    # Legacy AFF4 code expects token to be set.\n    fd.token = access_control.ACLToken(username=fd.creator)\n\n    if flow_obj.num_replies_sent:\n      fd.context.clients_with_results_count += 1\n\n    fd.context.completed_clients_count += 1\n    fd.context.results_count += flow_obj.num_replies_sent\n\n    fd.GetRunner().SaveResourceUsage(flow_obj.client_id, status_msg)\n\n  with aff4.FACTORY.Open(hunt_urn, mode=\"rw\") as fd:\n    # Legacy AFF4 code expects token to be set.\n    fd.token = access_control.ACLToken(username=fd.creator)\n\n    fd.RegisterCompletedClient(client_urn)\n    if flow_obj.num_replies_sent:\n      fd.RegisterClientWithResults(client_urn)\n\n    fd.StopHuntIfAverageLimitsExceeded()", "response": "Processes a given hunt - induced flow."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef ProcessHuntFlowLog(flow_obj, log_msg):\n\n  if not hunt.IsLegacyHunt(flow_obj.parent_hunt_id):\n    return\n\n  hunt_urn = rdfvalue.RDFURN(\"hunts\").Add(flow_obj.parent_hunt_id)\n  flow_urn = hunt_urn.Add(flow_obj.flow_id)\n  log_entry = rdf_flows.FlowLog(\n      client_id=flow_obj.client_id,\n      urn=flow_urn,\n      flow_name=flow_obj.flow_class_name,\n      log_message=log_msg)\n  with data_store.DB.GetMutationPool() as pool:\n    grr_collections.LogCollection.StaticAdd(\n        hunt_urn.Add(\"Logs\"), log_entry, mutation_pool=pool)", "response": "Processes a given hunt - induced flow log message."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef ProcessHuntClientCrash(flow_obj, client_crash_info):\n\n  if not hunt.IsLegacyHunt(flow_obj.parent_hunt_id):\n    hunt.StopHuntIfCrashLimitExceeded(flow_obj.parent_hunt_id)\n    return\n\n  hunt_urn = rdfvalue.RDFURN(\"hunts\").Add(flow_obj.parent_hunt_id)\n\n  with aff4.FACTORY.Open(hunt_urn, mode=\"rw\") as fd:\n    # Legacy AFF4 code expects token to be set.\n    fd.token = access_control.ACLToken(username=fd.creator)\n    fd.RegisterCrash(client_crash_info)", "response": "Processes a client crash triggered by a given hunt - induced flow."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef ValidateVfsPath(path):\n\n  components = (path or \"\").lstrip(\"/\").split(\"/\")\n  if not components:\n    raise ValueError(\"Empty path is not a valid path: %s.\" %\n                     utils.SmartStr(path))\n\n  if components[0] not in ROOT_FILES_WHITELIST:\n    raise ValueError(\n        \"First path component was '%s', but has to be one of %s\" %\n        (utils.SmartStr(components[0]), \", \".join(ROOT_FILES_WHITELIST)))\n\n  return True", "response": "Validates a VFS path."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _GenerateApiFileDetails(path_infos):\n\n  type_attrs = []\n  hash_attrs = []\n  size_attrs = []\n  stat_attrs = []\n  pathspec_attrs = []\n\n  def _Value(age, value):\n    \"\"\"Generate ApiAff4ObjectAttributeValue from an age and a value.\"\"\"\n\n    v = ApiAff4ObjectAttributeValue()\n    # TODO(user): get rid of RDF builtin \"age\" property.\n    v.Set(\"age\", age)\n    # With dynamic values we first have to set the type and\n    # then the value itself.\n    # TODO(user): refactor dynamic values logic so that it's not needed,\n    # possibly just removing the \"type\" attribute completely.\n    v.Set(\"type\", compatibility.GetName(value.__class__))\n    v.value = value\n    return v\n\n  for pi in path_infos:\n    if pi.directory:\n      object_type = \"VFSDirectory\"\n    else:\n      object_type = \"VFSFile\"\n\n    type_attrs.append(_Value(pi.timestamp, rdfvalue.RDFString(object_type)))\n\n    if pi.hash_entry:\n      hash_attrs.append(_Value(pi.timestamp, pi.hash_entry))\n      size_attrs.append(\n          _Value(pi.timestamp, rdfvalue.RDFInteger(pi.hash_entry.num_bytes)))\n    if pi.stat_entry:\n      stat_attrs.append(_Value(pi.timestamp, pi.stat_entry))\n\n      if pi.stat_entry.pathspec:\n        pathspec_attrs.append(_Value(pi.timestamp, pi.stat_entry.pathspec))\n\n  return ApiAff4ObjectRepresentation(types=[\n      ApiAff4ObjectType(\n          name=\"AFF4Object\",\n          attributes=[\n              ApiAff4ObjectAttribute(\n                  name=\"TYPE\",\n                  values=type_attrs,\n              ),\n          ],\n      ),\n      ApiAff4ObjectType(\n          name=\"AFF4Stream\",\n          attributes=[\n              ApiAff4ObjectAttribute(\n                  name=\"HASH\",\n                  values=hash_attrs,\n              ),\n              ApiAff4ObjectAttribute(\n                  name=\"SIZE\",\n                  values=size_attrs,\n              ),\n          ],\n      ),\n      ApiAff4ObjectType(\n          name=\"VFSFile\",\n          attributes=[\n              ApiAff4ObjectAttribute(\n                  name=\"PATHSPEC\",\n                  values=pathspec_attrs,\n              ),\n              ApiAff4ObjectAttribute(\n                  name=\"STAT\",\n                  values=stat_attrs,\n              ),\n          ],\n      ),\n  ])", "response": "Generates ApiAff4ObjectRepresentation for a list of path infos."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nretrieves the total size in bytes of an AFF4 object.", "response": "def _Aff4Size(aff4_obj):\n  \"\"\"Retrieves the total size in bytes of an AFF4 object.\n\n  Args:\n    aff4_obj: An AFF4 stream instance to retrieve size for.\n\n  Returns:\n    An integer representing number of bytes.\n\n  Raises:\n    TypeError: If `aff4_obj` is not an instance of AFF4 stream.\n  \"\"\"\n  if not isinstance(aff4_obj, aff4.AFF4Stream):\n    message = \"Expected an instance of `%s` but received `%s`\"\n    raise TypeError(message % (aff4.AFF4Stream, type(aff4_obj)))\n\n  return int(aff4_obj.Get(aff4_obj.Schema.SIZE))"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nread contents of given AFF4 stream.", "response": "def _Aff4Read(aff4_obj, offset, length):\n  \"\"\"Reads contents of given AFF4 file.\n\n  Args:\n    aff4_obj: An AFF4 stream instance to retrieve contents for.\n    offset: An offset to start the reading from.\n    length: A number of bytes to read. Reads the whole file if 0.\n\n  Returns:\n    Contents of specified AFF4 stream.\n\n  Raises:\n    TypeError: If `aff4_obj` is not an instance of AFF4 stream.\n  \"\"\"\n  length = length or (_Aff4Size(aff4_obj) - offset)\n\n  aff4_obj.Seek(offset)\n  return aff4_obj.Read(length)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nretrieve timeline entries from AFF4.", "response": "def _GetTimelineStatEntriesLegacy(client_id, file_path, with_history=True):\n  \"\"\"Gets timeline entries from AFF4.\"\"\"\n\n  folder_urn = aff4.ROOT_URN.Add(str(client_id)).Add(file_path)\n\n  child_urns = []\n  for _, children in aff4.FACTORY.RecursiveMultiListChildren([folder_urn]):\n    child_urns.extend(children)\n\n  if with_history:\n    timestamp = aff4.ALL_TIMES\n  else:\n    timestamp = aff4.NEWEST_TIME\n\n  for fd in aff4.FACTORY.MultiOpen(child_urns, age=timestamp):\n    file_path = \"/\".join(str(fd.urn).split(\"/\")[2:])\n\n    if not with_history:\n      yield file_path, fd.Get(fd.Schema.STAT), fd.Get(fd.Schema.HASH)\n      continue\n\n    result = {}\n\n    stats = fd.GetValuesForAttribute(fd.Schema.STAT)\n    for s in stats:\n      result[s.age] = [s, None]\n\n    hashes = fd.GetValuesForAttribute(fd.Schema.HASH)\n    for h in hashes:\n      prev = result.setdefault(h.age, [None, None])\n      prev[1] = h\n\n    for ts in sorted(result):\n      v = result[ts]\n      yield file_path, v[0], v[1]"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ngets timeline entries from REL_DB.", "response": "def _GetTimelineStatEntriesRelDB(api_client_id, file_path, with_history=True):\n  \"\"\"Gets timeline entries from REL_DB.\"\"\"\n  path_type, components = rdf_objects.ParseCategorizedPath(file_path)\n\n  client_id = str(api_client_id)\n\n  try:\n    root_path_info = data_store.REL_DB.ReadPathInfo(client_id, path_type,\n                                                    components)\n  except db.UnknownPathError:\n    return\n\n  path_infos = []\n  for path_info in itertools.chain(\n      [root_path_info],\n      data_store.REL_DB.ListDescendentPathInfos(client_id, path_type,\n                                                components),\n  ):\n    # TODO(user): this is to keep the compatibility with current\n    # AFF4 implementation. Check if this check is needed.\n    if path_info.directory:\n      continue\n\n    categorized_path = rdf_objects.ToCategorizedPath(path_info.path_type,\n                                                     path_info.components)\n    if with_history:\n      path_infos.append(path_info)\n    else:\n      yield categorized_path, path_info.stat_entry, path_info.hash_entry\n\n  if with_history:\n    hist_path_infos = data_store.REL_DB.ReadPathInfosHistories(\n        client_id, path_type, [tuple(pi.components) for pi in path_infos])\n    for path_info in itertools.chain(*hist_path_infos.itervalues()):\n      categorized_path = rdf_objects.ToCategorizedPath(path_info.path_type,\n                                                       path_info.components)\n      yield categorized_path, path_info.stat_entry, path_info.hash_entry"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nretrieving timeline entries from the appropriate data source.", "response": "def _GetTimelineStatEntries(client_id, file_path, with_history=True):\n  \"\"\"Gets timeline entries from the appropriate data source (AFF4 or REL_DB).\"\"\"\n\n  if data_store.RelationalDBEnabled():\n    fn = _GetTimelineStatEntriesRelDB\n  else:\n    fn = _GetTimelineStatEntriesLegacy\n\n  for v in fn(client_id, file_path, with_history=with_history):\n    yield v"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns a list of ApiVfsTimelineItem objects for a given client id and path.", "response": "def _GetTimelineItems(client_id, file_path):\n  \"\"\"Gets timeline items for a given client id and path.\"\"\"\n\n  items = []\n\n  for file_path, stat_entry, _ in _GetTimelineStatEntries(\n      client_id, file_path, with_history=True):\n\n    # It may be that for a given timestamp only hash entry is available, we're\n    # skipping those.\n    if stat_entry is None:\n      continue\n\n    # Add a new event for each MAC time if it exists.\n    for c in \"mac\":\n      timestamp = getattr(stat_entry, \"st_%stime\" % c)\n      if timestamp is None:\n        continue\n\n      item = ApiVfsTimelineItem()\n      item.timestamp = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(timestamp)\n\n      # Remove aff4:/<client_id> to have a more concise path to the\n      # subject.\n      item.file_path = file_path\n      if c == \"m\":\n        item.action = ApiVfsTimelineItem.FileActionType.MODIFICATION\n      elif c == \"a\":\n        item.action = ApiVfsTimelineItem.FileActionType.ACCESS\n      elif c == \"c\":\n        item.action = ApiVfsTimelineItem.FileActionType.METADATA_CHANGED\n\n      items.append(item)\n\n  return sorted(items, key=lambda x: x.timestamp, reverse=True)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef InitFromAff4Object(self, aff4_obj, aff4_cls, attr_blacklist):\n    self.name = str(aff4_cls.__name__)\n    self.attributes = []\n\n    schema = aff4_cls.SchemaCls\n    for name, attribute in sorted(iteritems(schema.__dict__)):\n      if not isinstance(attribute, aff4.Attribute):\n        continue\n\n      if name in attr_blacklist:\n        continue\n\n      attr_repr = ApiAff4ObjectAttribute()\n      attr_repr.name = name\n      attr_repr.description = attribute.description\n      attr_repr.values = []\n\n      values = list(aff4_obj.GetValuesForAttribute(attribute))\n      for value in values:\n        # This value is really a LazyDecoder() instance. We need to get at the\n        # real data here.\n        # TODO(user): Change GetValuesForAttribute to resolve\n        # lazy decoders and directly yield the rdf value.\n        if hasattr(value, \"ToRDFValue\"):\n          value = value.ToRDFValue()\n\n        value_repr = ApiAff4ObjectAttributeValue()\n        value_repr.Set(\"type\", compatibility.GetName(value.__class__))\n        value_repr.Set(\"age\", value.age)\n        value_repr.value = value\n        attr_repr.values.append(value_repr)\n\n      if attr_repr.values:\n        self.attributes.append(attr_repr)\n\n    return self", "response": "Initializes the current instance from an Aff4Object."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef InitFromAff4Object(self, aff4_obj):\n    attr_blacklist = []  # We use this to show attributes only once.\n\n    self.types = []\n    for aff4_cls in aff4_obj.__class__.__mro__:\n      if not hasattr(aff4_cls, \"SchemaCls\"):\n        continue\n\n      type_repr = ApiAff4ObjectType().InitFromAff4Object(\n          aff4_obj, aff4_cls, attr_blacklist)\n\n      if type_repr.attributes:\n        self.types.append(type_repr)\n\n      # Add all attribute names from this type representation to the\n      # blacklist to not add them to the result again.\n      attr_blacklist.extend([attr.name for attr in type_repr.attributes])\n\n    return self", "response": "Initializes the current instance from an Aff4Object."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ninitialize the current instance from an Aff4Object.", "response": "def InitFromAff4Object(self,\n                         file_obj,\n                         stat_entry=None,\n                         hash_entry=None,\n                         with_details=False):\n    \"\"\"Initializes the current instance from an Aff4Stream.\n\n    Args:\n      file_obj: An Aff4Stream representing a file.\n      stat_entry: An optional stat entry object to be used. If none is provided,\n        the one stored in the AFF4 data store is used.\n      hash_entry: An optional hash entry object to be used. If none is provided,\n        the one stored in the AFF4 data store is used.\n      with_details: True if all details of the Aff4Object should be included,\n        false otherwise.\n\n    Returns:\n      A reference to the current instance.\n    \"\"\"\n    self.name = file_obj.urn.Basename()\n    self.path = \"/\".join(file_obj.urn.Path().split(\"/\")[2:])\n    self.is_directory = \"Container\" in file_obj.behaviours\n\n    self.stat = stat_entry or file_obj.Get(file_obj.Schema.STAT)\n    self.hash = hash_entry or file_obj.Get(file_obj.Schema.HASH, None)\n\n    if not self.is_directory:\n      try:\n        self.last_collected = file_obj.GetContentAge()\n      except AttributeError:\n        # Defensive approach - in case file-like object doesn't have\n        # GetContentAge defined.\n        logging.debug(\"File-like object %s doesn't have GetContentAge defined.\",\n                      file_obj.__class__.__name__)\n\n      if self.last_collected:\n        self.last_collected_size = file_obj.Get(file_obj.Schema.SIZE)\n\n    type_obj = file_obj.Get(file_obj.Schema.TYPE)\n    if type_obj is not None:\n      self.age = type_obj.age\n\n    if with_details:\n      self.details = ApiAff4ObjectRepresentation().InitFromAff4Object(file_obj)\n\n    return self"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _Decode(self, codec_name, data):\n    try:\n      return data.decode(codec_name, \"replace\")\n    except LookupError:\n      raise RuntimeError(\"Codec could not be found.\")\n    except AssertionError:\n      raise RuntimeError(\"Codec failed to decode\")", "response": "Decode data with the given codec name."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nsigns multiple files at once.", "response": "def SignFiles(self, filenames):\n    \"\"\"Signs multiple files at once.\"\"\"\n    file_list = \" \".join(filenames)\n    subprocess.check_call(\"%s %s\" % (self._signing_cmdline, file_list))\n    if self._verification_cmdline:\n      subprocess.check_call(\"%s %s\" % (self._verification_cmdline, file_list))"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nsign a buffer via temp files.", "response": "def SignBuffer(self, in_buffer):\n    \"\"\"Sign a buffer via temp files.\n\n    Our signing tool can't sign a buffer, so we work around it using temporary\n    files.\n\n    Args:\n      in_buffer: data to sign\n\n    Returns:\n      signed data\n    \"\"\"\n    precondition.AssertType(in_buffer, bytes)\n    with tempfile.NamedTemporaryFile() as temp_in:\n      temp_in.write(in_buffer)\n      temp_in.seek(0)\n      outfile = self.SignFile(temp_in.name)\n      with io.open(outfile, \"rb\") as filedesc:\n        return filedesc.read()"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef SignFile(self, in_filename, out_filename=None):\n    if out_filename is None:\n      out_filename = \"%s.signed\" % in_filename\n\n    args = [\n        \"-certs\", self.cert, \"-key\", self.key, \"-n\", self.application, \"-t\",\n        \"http://timestamp.verisign.com/scripts/timestamp.dll\", \"-h\", \"sha1\",\n        \"-in\", in_filename, \"-out\", out_filename\n    ]\n\n    try:\n      output_log = io.StringIO()\n      ossl = pexpect.spawn(\"osslsigncode\", args)\n      # Use logfile_read so we don't get the password we're injecting.\n      ossl.logfile_read = output_log\n      ossl.expect(\"Enter PEM pass phrase\")\n      ossl.sendline(self.password)\n      ossl.wait()\n    except pexpect.ExceptionPexpect:\n      output_log.seek(0)\n      logging.exception(output_log.read())\n      raise\n\n    if not os.path.exists(out_filename):\n      raise SigningError(\"Expected output %s not created\" % out_filename)\n\n    try:\n      subprocess.check_call([\"osslsigncode\", \"verify\", \"-in\", out_filename])\n    except subprocess.CalledProcessError:\n      logging.exception(\"Bad signature verification on %s\", out_filename)\n      raise SigningError(\"Bad signature verification on %s\" % out_filename)\n\n    return out_filename", "response": "Signs a file using osslsigncode."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef AddSignatureToRPMs(self, rpm_filenames):\n    # The horrible second argument here is necessary to get a V3 signature to\n    # support CentOS 5 systems. See:\n    # http://ilostmynotes.blogspot.com/2016/03/the-horror-of-signing-rpms-that-support.html\n    args = [\n        \"--define=%%_gpg_name %s\" % self.gpg_name,\n        (\"--define=__gpg_sign_cmd %%{__gpg} gpg --force-v3-sigs --yes \"\n         \"--digest-algo=sha1 --no-verbose --no-armor --pinentry-mode loopback \"\n         \"--no-secmem-warning -u '%s' -sbo %%{__signature_filename} \"\n         \"%%{__plaintext_filename}\" % self.gpg_name), \"--addsign\"\n    ] + rpm_filenames\n\n    try:\n      output_log = io.StringIO()\n      rpmsign = pexpect.spawn(\"rpmsign\", args, timeout=1000)\n      # Use logfile_read so we don't get the password we're injecting.\n      rpmsign.logfile_read = output_log\n      rpmsign.expect(\"phrase:\")\n      rpmsign.sendline(self.password)\n      rpmsign.wait()\n    except pexpect.exceptions.EOF:\n      # This could have worked using a cached passphrase, we check for the\n      # actual signature below and raise if the package wasn't signed after all.\n      pass\n    except pexpect.ExceptionPexpect:\n      logging.error(output_log.getvalue())\n      raise\n\n    for rpm_filename in rpm_filenames:\n      try:\n        # Expected output is: filename.rpm: rsa sha1 (md5) pgp md5 OK\n        output = subprocess.check_output([\"rpm\", \"--checksig\", rpm_filename])\n        if \"pgp\" not in output:\n          raise SigningError(\"PGP missing checksig %s\" % rpm_filename)\n      except subprocess.CalledProcessError:\n        logging.exception(\"Bad signature verification on %s\", rpm_filename)\n        raise SigningError(\"Bad signature verification on %s\" % rpm_filename)", "response": "Signs a set of RPM files with the gpg signature."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef SearchClients(query_str, token=None, limit=1000):\n  client_schema = aff4.AFF4Object.classes[\"VFSGRRClient\"].SchemaCls\n  index = client_index.CreateClientIndex(token=token)\n\n  client_list = index.LookupClients([query_str])\n  result_set = aff4.FACTORY.MultiOpen(client_list, token=token)\n  results = []\n  for result in result_set:\n    results.append((result, str(result.Get(client_schema.HOSTNAME)),\n                    str(result.Get(client_schema.OS_VERSION)),\n                    str(result.Get(client_schema.PING))))\n    if len(results) >= limit:\n      break\n\n  return results", "response": "Search indexes for clients. Returns list ( client hostname os version ping )."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef DownloadDir(aff4_path, output_dir, bufsize=8192, preserve_path=True):\n  if not os.path.isdir(output_dir):\n    os.makedirs(output_dir)\n  fd = aff4.FACTORY.Open(aff4_path)\n  for child in fd.OpenChildren():\n    if preserve_path:\n      # Get a full path without the aff4:\n      full_dir = utils.JoinPath(output_dir, child.urn.Path())\n      full_dir = os.path.dirname(full_dir)\n      if not os.path.isdir(full_dir):\n        os.makedirs(full_dir)\n      outfile = os.path.join(full_dir, child.urn.Basename())\n    else:\n      outfile = os.path.join(output_dir, child.urn.Basename())\n    logging.info(u\"Downloading %s to %s\", child.urn, outfile)\n    with open(outfile, \"wb\") as out_fd:\n      try:\n        buf = child.Read(bufsize)\n        while buf:\n          out_fd.write(buf)\n          buf = child.Read(bufsize)\n      except IOError as e:\n        logging.error(\"Failed to read %s. Err: %s\", child.urn, e)", "response": "Download all files in a directory."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nopens the client with the given id and token.", "response": "def OpenClient(client_id=None, token=None):\n  \"\"\"Opens the client, getting potential approval tokens.\n\n  Args:\n    client_id: The client id that should be opened.\n    token: Token to use to open the client\n\n  Returns:\n    tuple containing (client, token) objects or (None, None) on if\n    no appropriate aproval tokens were found.\n  \"\"\"\n  if not token:\n    try:\n      token = ApprovalFind(client_id, token=token)\n    except access_control.UnauthorizedAccess as e:\n      logging.debug(\"No authorization found for access to client: %s\", e)\n\n  try:\n    # Try and open with the token we managed to retrieve or the default.\n    client = aff4.FACTORY.Open(\n        rdfvalue.RDFURN(client_id), mode=\"r\", token=token)\n    return client, token\n  except access_control.UnauthorizedAccess:\n    logging.warning(\n        \"Unable to find a valid reason for client %s. You may need \"\n        \"to request approval.\", client_id)\n    return None, None"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef GetNotifications(user=None, token=None):\n  if not user:\n    user = getpass.getuser()\n  user_obj = aff4.FACTORY.Open(\n      aff4.ROOT_URN.Add(\"users\").Add(user), token=token)\n  return list(user_obj.Get(user_obj.Schema.PENDING_NOTIFICATIONS))", "response": "Show pending notifications for a user."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef ApprovalGrant(token=None):\n  user = getpass.getuser()\n  notifications = GetNotifications(user=user, token=token)\n  requests = [n for n in notifications if n.type == \"GrantAccess\"]\n  for request in requests:\n    _, client_id, user, reason = rdfvalue.RDFURN(request.subject).Split()\n    reason = utils.DecodeReasonString(reason)\n    print(request)\n    print(\"Reason: %s\" % reason)\n    if input(\"Do you approve this request? [y/N] \").lower() == \"y\":\n      security.ClientApprovalGrantor(\n          subject_urn=client_id, reason=reason, delegate=user,\n          token=token).Grant()\n      # TODO(user): Remove the notification.\n    else:\n      print(\"skipping request\")\n    print(\"Approval sent\")", "response": "Iterate through requested access approving or not."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nfinds approvals issued for a specific client.", "response": "def ApprovalFind(object_id, token=None):\n  \"\"\"Find approvals issued for a specific client.\"\"\"\n  user = getpass.getuser()\n  object_id = rdfvalue.RDFURN(object_id)\n  try:\n    approved_token = security.Approval.GetApprovalForObject(\n        object_id, token=token, username=user)\n    print(\"Found token %s\" % str(approved_token))\n    return approved_token\n  except access_control.UnauthorizedAccess:\n    print(\"No token available for access to %s\" % object_id)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncreating an approval with raw datastore access.", "response": "def ApprovalCreateRaw(aff4_path,\n                      reason=\"\",\n                      expire_in=60 * 60 * 24 * 7,\n                      token=None,\n                      approval_type=\"ClientApproval\"):\n  \"\"\"Creates an approval with raw access.\n\n  This method requires raw datastore access to manipulate approvals directly.\n  This currently doesn't work for hunt or cron approvals, because they check\n  that each approver has the admin label.  Since the fake users don't exist the\n  check fails.\n\n  Args:\n    aff4_path: The aff4_path or client id the approval should be created for.\n    reason: The reason to put in the token.\n    expire_in: Expiry in seconds to use in the token.\n    token: The token that will be used. If this is specified reason and expiry\n      are ignored.\n    approval_type: The type of the approval to create.\n\n  Returns:\n    The token.\n\n  Raises:\n    RuntimeError: On bad token.\n  \"\"\"\n  if approval_type in [\"ClientApproval\", security.ClientApproval]:\n    urn = rdf_client.ClientURN(aff4_path)\n  else:\n    urn = rdfvalue.RDFURN(aff4_path)\n\n  if not token:\n    expiry = time.time() + expire_in\n    token = access_control.ACLToken(reason=reason, expiry=expiry)\n\n  if not token.reason:\n    raise RuntimeError(\"Cannot create approval with empty reason\")\n  if not token.username:\n    token.username = getpass.getuser()\n  approval_urn = security.ApprovalRequestor.ApprovalUrnBuilder(\n      urn.Path(), token.username, token.reason)\n  super_token = access_control.ACLToken(username=\"raw-approval-superuser\")\n  super_token.supervisor = True\n\n  if isinstance(approval_type, string_types):\n    approval_type_cls = aff4.AFF4Object.classes[approval_type]\n  else:\n    approval_type_cls = approval_type\n\n  approval_request = aff4.FACTORY.Create(\n      approval_urn, approval_type_cls, mode=\"rw\", token=super_token)\n\n  # Add approvals indicating they were approved by fake \"raw\" mode users.\n  approval_request.AddAttribute(\n      approval_request.Schema.APPROVER(\"%s1-raw\" % token.username))\n  approval_request.AddAttribute(\n      approval_request.Schema.APPROVER(\"%s-raw2\" % token.username))\n  approval_request.Close()"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef ApprovalRevokeRaw(aff4_path, token):\n  try:\n    urn = rdf_client.ClientURN(aff4_path)\n  except type_info.TypeValueError:\n    urn = rdfvalue.RDFURN(aff4_path)\n\n  approval_urn = aff4.ROOT_URN.Add(\"ACL\").Add(urn.Path()).Add(\n      token.username).Add(utils.EncodeReasonString(token.reason))\n\n  super_token = access_control.ACLToken(username=\"raw-approval-superuser\")\n  super_token.supervisor = True\n\n  approval_request = aff4.FACTORY.Open(\n      approval_urn, mode=\"rw\", token=super_token)\n  approval_request.DeleteAttribute(approval_request.Schema.APPROVER)\n  approval_request.Close()", "response": "Revokes an approval for a given token."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nopening the given clients in batches and returns hardware information.", "response": "def _GetHWInfos(client_list, batch_size=10000, token=None):\n  \"\"\"Opens the given clients in batches and returns hardware information.\"\"\"\n\n  # This function returns a dict mapping each client_id to a set of reported\n  # hardware serial numbers reported by this client.\n  hw_infos = {}\n\n  logging.info(\"%d clients to process.\", len(client_list))\n\n  c = 0\n\n  for batch in collection.Batch(client_list, batch_size):\n    logging.info(\"Processing batch: %d-%d\", c, c + batch_size)\n    c += len(batch)\n\n    client_objs = aff4.FACTORY.MultiOpen(batch, age=aff4.ALL_TIMES, token=token)\n\n    for client in client_objs:\n      hwi = client.GetValuesForAttribute(client.Schema.HARDWARE_INFO)\n\n      hw_infos[client.urn] = set([\"%s\" % x.serial_number for x in hwi])\n\n  return hw_infos"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef CleanClientVersions(clients=None, dry_run=True, token=None):\n  if not clients:\n    index = client_index.CreateClientIndex(token=token)\n    clients = index.LookupClients([\".\"])\n  clients.sort()\n  with data_store.DB.GetMutationPool() as pool:\n\n    logging.info(\"checking %d clients\", len(clients))\n\n    # TODO(amoser): This only works on datastores that use the Bigtable scheme.\n    client_infos = data_store.DB.MultiResolvePrefix(\n        clients, \"aff4:type\", data_store.DB.ALL_TIMESTAMPS)\n\n    for client, type_list in client_infos:\n      logging.info(\"%s: has %d versions\", client, len(type_list))\n      cleared = 0\n      kept = 1\n      last_kept = type_list[0][2]\n      for _, _, ts in type_list[1:]:\n        if last_kept - ts > 60 * 60 * 1000000:  # 1 hour\n          last_kept = ts\n          kept += 1\n        else:\n          if not dry_run:\n            pool.DeleteAttributes(client, [\"aff4:type\"], start=ts, end=ts)\n          cleared += 1\n          if pool.Size() > 10000:\n            pool.Flush()\n      logging.info(\"%s: kept %d and cleared %d\", client, kept, cleared)", "response": "This function cleans up excessive client versions."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nstart a flow and a worker.", "response": "def StartFlowAndWorker(client_id, flow_name, **kwargs):\n  \"\"\"Launches the flow and worker and waits for it to finish.\n\n  Args:\n     client_id: The client common name we issue the request.\n     flow_name: The name of the flow to launch.\n     **kwargs: passthrough to flow.\n\n  Returns:\n     A flow session id.\n\n  Note: you need raw access to run this flow as it requires running a worker.\n  \"\"\"\n  # Empty token, only works with raw access.\n  queue = rdfvalue.RDFURN(\"DEBUG-%s-\" % getpass.getuser())\n  if \"token\" in kwargs:\n    token = kwargs.pop(\"token\")\n  else:\n    token = access_control.ACLToken(username=\"GRRConsole\")\n\n  session_id = flow.StartAFF4Flow(\n      client_id=client_id,\n      flow_name=flow_name,\n      queue=queue,\n      token=token,\n      **kwargs)\n  worker_thrd = worker_lib.GRRWorker(\n      queues=[queue], token=token, threadpool_size=1)\n  while True:\n    try:\n      worker_thrd.RunOnce()\n    except KeyboardInterrupt:\n      print(\"exiting\")\n      worker_thrd.thread_pool.Join()\n      break\n\n    time.sleep(2)\n    with aff4.FACTORY.Open(session_id, token=token) as flow_obj:\n      if not flow_obj.GetRunner().IsRunning():\n        break\n\n  # Terminate the worker threads\n  worker_thrd.thread_pool.Join()\n\n  return session_id"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nwaking up stuck flows.", "response": "def WakeStuckFlow(session_id):\n  \"\"\"Wake up stuck flows.\n\n  A stuck flow is one which is waiting for the client to do something, but the\n  client requests have been removed from the client queue. This can happen if\n  the system is too loaded and the client messages have TTLed out. In this case\n  we reschedule the client requests for this session.\n\n  Args:\n    session_id: The session for the flow to wake.\n\n  Returns:\n    The total number of client messages re-queued.\n  \"\"\"\n  session_id = rdfvalue.SessionID(session_id)\n  woken = 0\n  checked_pending = False\n\n  with queue_manager.QueueManager() as manager:\n    for request, responses in manager.FetchRequestsAndResponses(session_id):\n      # We need to check if there are client requests pending.\n      if not checked_pending:\n        task = manager.Query(\n            request.client_id, task_id=\"task:%s\" % request.request.task_id)\n\n        if task:\n          # Client has tasks pending already.\n          return\n\n        checked_pending = True\n\n      if (not responses or\n          responses[-1].type != rdf_flows.GrrMessage.Type.STATUS):\n        manager.QueueClientMessage(request.request)\n        woken += 1\n\n      if responses and responses[-1].type == rdf_flows.GrrMessage.Type.STATUS:\n        manager.QueueNotification(session_id)\n\n  return woken"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef RunOnce(self):\n\n    # Figure out which port to use.\n    port = config.CONFIG[\"Monitoring.http_port\"]\n    if not port:\n      logging.info(\"Monitoring server disabled.\")\n      return\n\n    # TODO(user): Implement __contains__ for GrrConfigManager.\n    max_port = config.CONFIG.Get(\"Monitoring.http_port_max\", None)\n    if max_port is None:\n      # Use the same number of available ports as the adminui is using. If we\n      # have 10 available for adminui we will need 10 for the stats server.\n      adminui_max_port = config.CONFIG.Get(\"AdminUI.port_max\",\n                                           config.CONFIG[\"AdminUI.port\"])\n      max_port = port + adminui_max_port - config.CONFIG[\"AdminUI.port\"]\n\n    try:\n      # pylint: disable=g-import-not-at-top\n      from grr_response_server.local import stats_server\n      # pylint: enable=g-import-not-at-top\n      server_cls = stats_server.StatsServer\n      logging.debug(\"Using local StatsServer\")\n    except ImportError:\n      logging.debug(\"Using default StatsServer\")\n      server_cls = StatsServer\n\n    for port in range(port, max_port + 1):\n      try:\n        logging.info(\"Starting monitoring server on port %d.\", port)\n        server_obj = server_cls(port)\n        server_obj.Start()\n        return\n      except base_stats_server.PortInUseError as e:\n        if e.port < max_port:\n          logging.info(e)\n          continue\n        raise", "response": "This method is called by the base class to start monitoring server."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef Parse(self, stat, file_object, knowledge_base):\n    _, _ = stat, knowledge_base\n\n    packages = []\n    sw_data = utils.ReadFileBytesAsUnicode(file_object)\n    try:\n      for pkg in self._deb822.Packages.iter_paragraphs(sw_data.splitlines()):\n        if self.installed_re.match(pkg[\"Status\"]):\n          packages.append(\n              rdf_client.SoftwarePackage(\n                  name=pkg[\"Package\"],\n                  description=pkg[\"Description\"],\n                  version=pkg[\"Version\"],\n                  architecture=pkg[\"Architecture\"],\n                  publisher=pkg[\"Maintainer\"],\n                  install_state=\"INSTALLED\"))\n    except SystemError:\n      yield rdf_anomaly.Anomaly(\n          type=\"PARSER_ANOMALY\", symptom=\"Invalid dpkg status file\")\n    finally:\n      if packages:\n        yield rdf_client.SoftwarePackages(packages=packages)", "response": "Parse the status file."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef Is64bit(self):\n    if \"64\" not in platform.machine():\n      return False\n    iswow64 = ctypes.c_bool(False)\n    if IsWow64Process is None:\n      return False\n    if not IsWow64Process(self.h_process, ctypes.byref(iswow64)):\n      raise process_error.ProcessError(\"Error while calling IsWow64Process.\")\n    return not iswow64.value", "response": "Returns true if this is a 64 bit process."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nopening the process for reading.", "response": "def Open(self):\n    \"\"\"Opens the process for reading.\"\"\"\n\n    self.h_process = kernel32.OpenProcess(\n        PROCESS_VM_READ | PROCESS_QUERY_INFORMATION, 0, self.pid)\n    if not self.h_process:\n      raise process_error.ProcessError(\n          \"Failed to open process (pid %d).\" % self.pid)\n\n    if self.Is64bit():\n      si = self.GetNativeSystemInfo()\n      self.max_addr = si.lpMaximumApplicationAddress\n    else:\n      si = self.GetSystemInfo()\n      self.max_addr = 2147418111\n\n    self.min_addr = si.lpMinimumApplicationAddress"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreads at most num_bytes starting from offset <address >.", "response": "def ReadBytes(self, address, num_bytes):\n    \"\"\"Reads at most num_bytes starting from offset <address>.\"\"\"\n    address = int(address)\n    buf = ctypes.create_string_buffer(num_bytes)\n    bytesread = ctypes.c_size_t(0)\n    res = ReadProcessMemory(self.h_process, address, buf, num_bytes,\n                            ctypes.byref(bytesread))\n    if res == 0:\n      err = wintypes.GetLastError()\n      if err == 299:\n        # Only part of ReadProcessMemory has been done, let's return it.\n        return buf.raw[:bytesread.value]\n      raise process_error.ProcessError(\"Error in ReadProcessMemory: %d\" % err)\n\n    return buf.raw[:bytesread.value]"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _MigrationFilenameToInt(fname):\n  base, _ = os.path.splitext(fname)\n  return int(base)", "response": "Converts a migration filename to a migration number."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nlisting filenames of migrations with numbers bigger than a given one.", "response": "def ListMigrationsToProcess(migrations_root,\n                            current_migration_number\n                           ):\n  \"\"\"Lists filenames of migrations with numbers bigger than a given one.\"\"\"\n  migrations = []\n  for m in os.listdir(migrations_root):\n    if (current_migration_number is None or\n        _MigrationFilenameToInt(m) > current_migration_number):\n      migrations.append(m)\n\n  return sorted(migrations, key=_MigrationFilenameToInt)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef ProcessMigrations(open_conn_fn,\n                      migrations_root):\n  \"\"\"Processes migrations from a given folder.\n\n  This function uses LOCK TABLE MySQL command on _migrations\n  table to ensure that only one GRR process is actually\n  performing the migration.\n\n  We have to use open_conn_fn to open 2 connections to the database,\n  since LOCK TABLE command is per-connection and it's not allowed\n  to modify non-locked tables if LOCK TABLE was called within a\n  connection. To overcome this limitation we use one connection\n  to lock _migrations and perform its updates and one connection\n  to do actual migrations.\n\n  Args:\n    open_conn_fn: A function to open new database connection.\n    migrations_root: A path to folder with migration files.\n  \"\"\"\n  with contextlib.closing(open_conn_fn()) as conn:\n    conn.autocommit(True)\n\n    with contextlib.closing(conn.cursor()) as cursor:\n      cursor.execute(\"\"\"CREATE TABLE IF NOT EXISTS _migrations(\n        migration_id INT UNSIGNED PRIMARY KEY,\n        timestamp TIMESTAMP(6) NOT NULL DEFAULT NOW(6)\n        )\"\"\")\n\n    with contextlib.closing(conn.cursor()) as cursor:\n      cursor.execute('SELECT GET_LOCK(\"grr_migration\", 3600)')\n\n    try:\n      with contextlib.closing(conn.cursor()) as cursor:\n        current_migration = GetLatestMigrationNumber(cursor)\n\n      to_process = ListMigrationsToProcess(migrations_root, current_migration)\n      logging.info(\"Will execute following DB migrations: %s\",\n                   \", \".join(to_process))\n\n      for fname in to_process:\n        start_time = time.time()\n        logging.info(\"Starting migration %s\", fname)\n\n        with open(os.path.join(migrations_root, fname)) as fd:\n          sql = fd.read()\n          with contextlib.closing(conn.cursor()) as cursor:\n            cursor.execute(sql)\n\n        logging.info(\"Migration %s is done. Took %.2fs\", fname,\n                     time.time() - start_time)\n\n        # Update _migrations table with the latest migration.\n        with contextlib.closing(conn.cursor()) as cursor:\n          cursor.execute(\"INSERT INTO _migrations (migration_id) VALUES (%s)\",\n                         [_MigrationFilenameToInt(fname)])\n    finally:\n      with contextlib.closing(conn.cursor()) as cursor:\n        cursor.execute('SELECT RELEASE_LOCK(\"grr_migration\")')", "response": "Processes migrations from a given folder."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ndump the current database schema.", "response": "def DumpCurrentSchema(cursor):\n  \"\"\"Dumps current database schema.\"\"\"\n  cursor.execute(\"SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES \"\n                 \"WHERE table_schema = (SELECT DATABASE())\")\n  defs = []\n  for table, in sorted(cursor.fetchall()):\n    cursor.execute(\"SHOW CREATE TABLE `{}`\".format(table))\n    rows = cursor.fetchall()\n    defs.append(rows[0][1])\n\n  return \"\\n\\n\".join(defs)"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nfilters a list by filtering elements starting at offset.", "response": "def FilterList(l, offset, count=0, filter_value=None):\n  \"\"\"Filters a list, getting count elements, starting at offset.\"\"\"\n\n  if offset < 0:\n    raise ValueError(\"Offset needs to be greater than or equal to zero\")\n\n  if count < 0:\n    raise ValueError(\"Count needs to be greater than or equal to zero\")\n\n  count = count or sys.maxsize\n  if not filter_value:\n    return l[offset:offset + count]\n\n  index = 0\n  items = []\n  for item in l:\n    serialized_item = item.SerializeToString()\n    if re.search(re.escape(filter_value), serialized_item, re.I):\n      if index >= offset:\n        items.append(item)\n      index += 1\n\n      if len(items) >= count:\n        break\n\n  return items"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef FilterCollection(aff4_collection, offset, count=0, filter_value=None):\n\n  if offset < 0:\n    raise ValueError(\"Offset needs to be greater than or equal to zero\")\n\n  if count < 0:\n    raise ValueError(\"Count needs to be greater than or equal to zero\")\n\n  count = count or sys.maxsize\n  if filter_value:\n    index = 0\n    items = []\n    for item in aff4_collection.GenerateItems():\n      serialized_item = item.SerializeToString()\n      if re.search(re.escape(filter_value), serialized_item, re.I):\n        if index >= offset:\n          items.append(item)\n        index += 1\n\n        if len(items) >= count:\n          break\n  else:\n    items = list(itertools.islice(aff4_collection.GenerateItems(offset), count))\n\n  return items", "response": "Filters an aff4 collection starting at offset."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_config():\n  # TODO(hanuszczak): See comment in `setup.py` for `grr-response-proto`.\n  ini_path = os.path.join(THIS_DIRECTORY, \"version.ini\")\n  if not os.path.exists(ini_path):\n    ini_path = os.path.join(THIS_DIRECTORY, \"../../version.ini\")\n    if not os.path.exists(ini_path):\n      raise RuntimeError(\"Couldn't find version.ini\")\n\n  config = configparser.SafeConfigParser()\n  config.read(ini_path)\n  return config", "response": "Get INI parser with version. ini data."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef CopyFiles(self):\n    if self.fleetspeak_enabled:\n      # Copy files needed for dpkg-buildpackage.\n      shutil.copytree(\n          config_lib.Resource().Filter(\n              \"install_data/debian/dpkg_client/fleetspeak-debian\"),\n          os.path.join(self.package_dir, \"debian/debian.in\"))\n      return\n\n    # Copy the nanny binary.\n    shutil.copy(\n        package.ResourcePath(\"grr-response-core\",\n                             \"install_data/debian/dpkg_client/nanny.sh.in\"),\n        self.output_dir)\n\n    # Copy files needed for dpkg-buildpackage.\n    shutil.copytree(\n        config_lib.Resource().Filter(\"install_data/debian/dpkg_client/debian\"),\n        os.path.join(self.package_dir, \"debian/debian.in\"))\n\n    # Copy upstart files\n    outdir = os.path.join(self.package_dir, \"debian/upstart.in\")\n    utils.EnsureDirExists(outdir)\n    shutil.copy(\n        config_lib.Resource().Filter(\n            \"install_data/debian/dpkg_client/upstart/grr-client.conf\"), outdir)\n\n    # Copy init files\n    outdir = os.path.join(self.package_dir, \"debian/initd.in\")\n    utils.EnsureDirExists(outdir)\n    shutil.copy(\n        config_lib.Resource().Filter(\n            \"install_data/debian/dpkg_client/initd/grr-client\"), outdir)\n\n    # Copy systemd unit file\n    outdir = os.path.join(self.package_dir, \"debian/systemd.in\")\n    utils.EnsureDirExists(outdir)\n    shutil.copy(\n        config_lib.Resource().Filter(\n            \"install_data/systemd/client/grr-client.service\"), outdir)", "response": "This copies the files needed for the debian and nanny binary."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ncreate a ZIP archive of the files in the input directory.", "response": "def MakeZip(self, input_dir, output_file):\n    \"\"\"Creates a ZIP archive of the files in the input directory.\n\n    Args:\n      input_dir: the name of the input directory.\n      output_file: the name of the output ZIP archive without extension.\n    \"\"\"\n\n    logging.info(\"Generating zip template file at %s\", output_file)\n    zf = zipfile.ZipFile(output_file, \"w\")\n    oldwd = os.getcwd()\n    os.chdir(input_dir)\n    for path in [\"debian\", \"rpmbuild\", \"fleetspeak\"]:\n      for root, _, files in os.walk(path):\n        for f in files:\n          zf.write(os.path.join(root, f))\n    zf.close()\n    os.chdir(oldwd)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef CopyFiles(self):\n\n    shutil.move(\n        os.path.join(self.package_dir, \"debian\"),\n        os.path.join(self.package_dir, \"rpmbuild\"))\n    if self.fleetspeak_enabled:\n      rpm_spec_filename = \"fleetspeak.grr.spec.in\"\n    else:\n      rpm_spec_filename = \"grr.spec.in\"\n      shutil.copy(\n          config_lib.Resource().Filter(\n              \"install_data/centos/grr-client.initd.in\"),\n          os.path.join(self.package_dir, \"rpmbuild/grr-client.initd.in\"))\n      shutil.copy(\n          config_lib.Resource().Filter(\n              \"install_data/systemd/client/grr-client.service\"),\n          os.path.join(self.package_dir, \"rpmbuild/grr-client.service.in\"))\n\n    shutil.copy(\n        config_lib.Resource().Filter(\"install_data/centos/\" +\n                                     rpm_spec_filename),\n        os.path.join(self.package_dir, \"rpmbuild/grr.spec.in\"))\n\n    # Copy prelink blacklist file. Without this file, prelink will mangle\n    # the GRR binary.\n    shutil.copy(\n        config_lib.Resource().Filter(\n            \"install_data/centos/prelink_blacklist.conf.in\"),\n        os.path.join(self.package_dir, \"rpmbuild/prelink_blacklist.conf.in\"))", "response": "This copies the files from the build directory to the build directory."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef SecurityCheck(func):\n\n  def Wrapper(request, *args, **kwargs):\n    \"\"\"Wrapping function.\"\"\"\n    if WEBAUTH_MANAGER is None:\n      raise RuntimeError(\"Attempt to initialize before WEBAUTH_MANAGER set.\")\n    return WEBAUTH_MANAGER.SecurityCheck(func, request, *args, **kwargs)\n\n  return Wrapper", "response": "A decorator applied to protected web handlers."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncheck if access should be allowed for the request.", "response": "def SecurityCheck(self, func, request, *args, **kwargs):\n    \"\"\"Check if access should be allowed for the request.\"\"\"\n\n    try:\n      auth_header = request.headers.get(\"Authorization\", \"\")\n      if not auth_header.startswith(self.BEARER_PREFIX):\n        raise ValueError(\"JWT token is missing.\")\n\n      token = auth_header[len(self.BEARER_PREFIX):]\n\n      auth_domain = config.CONFIG[\"AdminUI.firebase_auth_domain\"]\n      project_id = auth_domain.split(\".\")[0]\n\n      idinfo = id_token.verify_firebase_token(\n          token, request, audience=project_id)\n\n      if idinfo[\"iss\"] != self.SECURE_TOKEN_PREFIX + project_id:\n        raise ValueError(\"Wrong issuer.\")\n\n      request.user = idinfo[\"email\"]\n    except ValueError as e:\n      # For a homepage, just do a pass-through, otherwise JS code responsible\n      # for the Firebase auth won't ever get executed. This approach is safe,\n      # because wsgiapp.HttpRequest object will raise on any attempt to\n      # access uninitialized HttpRequest.user attribute.\n      if request.path != \"/\":\n        return self.AuthError(\"JWT token validation failed: %s\" % e)\n\n    return func(request, *args, **kwargs)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nrun this once on init.", "response": "def RunOnce(self):\n    \"\"\"Run this once on init.\"\"\"\n    global WEBAUTH_MANAGER  # pylint: disable=global-statement\n\n    # pylint: disable=g-bad-name\n    WEBAUTH_MANAGER = BaseWebAuthManager.GetPlugin(\n        config.CONFIG[\"AdminUI.webauth_manager\"])()\n\n    # pylint: enable=g-bad-name\n    logging.info(\"Using webauth manager %s\", WEBAUTH_MANAGER)"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns a dictionary of annotated router methods.", "response": "def GetAnnotatedMethods(cls):\n    \"\"\"Returns a dictionary of annotated router methods.\"\"\"\n\n    result = {}\n\n    # We want methods with the highest call-order to be processed last,\n    # so that their annotations have precedence.\n    for i_cls in reversed(inspect.getmro(cls)):\n      for name in compatibility.ListAttrs(i_cls):\n        cls_method = getattr(i_cls, name)\n\n        if not callable(cls_method):\n          continue\n\n        if not hasattr(cls_method, \"__http_methods__\"):\n          continue\n\n        result[name] = RouterMethodMetadata(\n            name=name,\n            doc=cls_method.__doc__,\n            args_type=getattr(cls_method, \"__args_type__\", None),\n            result_type=getattr(cls_method, \"__result_type__\", None),\n            category=getattr(cls_method, \"__category__\", None),\n            http_methods=getattr(cls_method, \"__http_methods__\", set()),\n            no_audit_log_required=getattr(cls_method,\n                                          \"__no_audit_log_required__\", False))\n\n    return result"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\npublishes a message into all listeners of the event.", "response": "def PublishEvent(cls, event_name, msg, token=None):\n    \"\"\"Publish the message into all listeners of the event.\n\n    We send the message to all event handlers which contain this\n    string in their EVENT static member. This allows the event to be\n    sent to multiple interested listeners.\n\n    Args:\n      event_name: An event name.\n      msg: The message to send to the event handler.\n      token: ACL token.\n\n    Raises:\n      ValueError: If the message is invalid. The message must be a Semantic\n        Value (instance of RDFValue) or a full GrrMessage.\n    \"\"\"\n    cls.PublishMultipleEvents({event_name: [msg]}, token=token)"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\npublishes multiple messages at once.", "response": "def PublishMultipleEvents(cls, events, token=None):\n    \"\"\"Publishes multiple messages at once.\n\n    Args:\n      events: A dict with keys being event names and values being lists of\n        messages.\n      token: ACL token.\n\n    Raises:\n      ValueError: If the message is invalid. The message must be a Semantic\n        Value (instance of RDFValue) or a full GrrMessage.\n    \"\"\"\n    event_name_map = registry.EventRegistry.EVENT_NAME_MAP\n    for event_name, messages in iteritems(events):\n      if not isinstance(event_name, string_types):\n        raise ValueError(\n            \"Event names should be string, got: %s\" % type(event_name))\n      for msg in messages:\n        if not isinstance(msg, rdfvalue.RDFValue):\n          raise ValueError(\"Can only publish RDFValue instances.\")\n\n      for event_cls in event_name_map.get(event_name, []):\n        event_cls().ProcessMessages(messages, token=token)"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\noverlays a child hint with a parent hint.", "response": "def Overlay(child, parent):\n  \"\"\"Adds hint attributes to a child hint if they are not defined.\"\"\"\n  for arg in child, parent:\n    if not isinstance(arg, collections.Mapping):\n      raise DefinitionError(\"Trying to merge badly defined hints. Child: %s, \"\n                            \"Parent: %s\" % (type(child), type(parent)))\n  for attr in [\"fix\", \"format\", \"problem\", \"summary\"]:\n    if not child.get(attr):\n      child[attr] = parent.get(attr, \"\").strip()\n  return child"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef FanOut(self, obj, parent=None):\n    # Catch cases where RDFs are iterable but return themselves.\n    if parent and obj == parent:\n      results = [utils.SmartUnicode(obj).strip()]\n    elif isinstance(obj, (string_types, rdf_structs.EnumNamedValue)):\n      results = [utils.SmartUnicode(obj).strip()]\n    elif isinstance(obj, rdf_protodict.DataBlob):\n      results = self.FanOut(obj.GetValue())\n    elif isinstance(obj, (collections.Mapping, rdf_protodict.Dict)):\n      results = []\n      # rdf_protodict.Dict only has items, not iteritems.\n      for k, v in iteritems(obj):\n        expanded_v = [utils.SmartUnicode(r) for r in self.FanOut(v)]\n        results.append(\"%s:%s\" % (utils.SmartUnicode(k), \",\".join(expanded_v)))\n    elif isinstance(obj, (collections.Iterable,\n                          rdf_structs.RepeatedFieldHelper)):\n      results = []\n      for rslt in [self.FanOut(o, obj) for o in obj]:\n        results.extend(rslt)\n    else:\n      results = [utils.SmartUnicode(obj).strip()]\n    return results", "response": "Expand values from various attribute types."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef Format(self, format_string, rdf):\n    result = []\n    for literal_text, field_name, _, _ in self.parse(format_string):\n      # output the literal text\n      if literal_text:\n        result.append(literal_text)\n      # if there's a field, output it\n      if field_name is not None:\n        rslts = []\n        objs = self.expander(rdf, field_name)\n        for o in objs:\n          rslts.extend(self.FanOut(o))\n        # format the objects and append to the result\n        result.append(\",\".join(rslts))\n    return \"\".join(result)", "response": "Apply string formatting templates to rdf data."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nrefreshes an attribute of an object.", "response": "def Update(self, attribute=None):\n    \"\"\"Refresh an old attribute.\n\n    Note that refreshing the attribute is asynchronous. It does not change\n    anything about the current object - you need to reopen the same URN some\n    time later to get fresh data.\n\n    Attributes: CONTAINS - Refresh the content of the directory listing.\n    Args:\n       attribute: An attribute object as listed above.\n\n    Returns:\n       The Flow ID that is pending\n\n    Raises:\n       IOError: If there has been an error starting the flow.\n    \"\"\"\n    # client id is the first path element\n    client_id = self.urn.Split()[0]\n\n    if attribute == \"CONTAINS\":\n      # Get the pathspec for this object\n      flow_id = flow.StartAFF4Flow(\n          client_id=client_id,\n          # Dependency loop: aff4_objects/aff4_grr.py depends on\n          # aff4_objects/standard.py that depends on flows/general/filesystem.py\n          # that eventually depends on aff4_objects/aff4_grr.py\n          # flow_name=filesystem.ListDirectory.__name__,\n          flow_name=\"ListDirectory\",\n          pathspec=self.real_pathspec,\n          notify_to_user=False,\n          token=self.token)\n\n      return flow_id"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _GetChunkForReading(self, chunk):\n    try:\n      return self.chunk_cache.Get(chunk)\n    except KeyError:\n      pass\n\n    # We don't have this chunk already cached. The most common read\n    # access pattern is contiguous reading so since we have to go to\n    # the data store already, we read ahead to reduce round trips.\n\n    missing_chunks = []\n    for chunk_number in range(chunk, chunk + 10):\n      if chunk_number not in self.chunk_cache:\n        missing_chunks.append(chunk_number)\n\n    self._ReadChunks(missing_chunks)\n    # This should work now - otherwise we just give up.\n    try:\n      return self.chunk_cache.Get(chunk)\n    except KeyError:\n      raise aff4.ChunkNotFoundError(\"Cannot open chunk %s\" % chunk)", "response": "Returns the relevant chunk from the datastore and reads ahead."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn the relevant chunk from the datastore.", "response": "def _GetChunkForWriting(self, chunk):\n    \"\"\"Returns the relevant chunk from the datastore.\"\"\"\n    try:\n      chunk = self.chunk_cache.Get(chunk)\n      chunk.dirty = True\n      return chunk\n    except KeyError:\n      pass\n\n    try:\n      chunk = self._ReadChunk(chunk)\n      chunk.dirty = True\n      return chunk\n    except KeyError:\n      pass\n\n    fd = io.BytesIO()\n    fd.chunk = chunk\n    fd.dirty = True\n    self.chunk_cache.Put(chunk, fd)\n\n    # Keep track of the biggest chunk_number we've seen so far.\n    if chunk > self.last_chunk:\n      self.last_chunk = chunk\n      self._dirty = True\n\n    return fd"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _ReadPartial(self, length):\n    chunk = self.offset // self.chunksize\n    chunk_offset = self.offset % self.chunksize\n\n    # If we're past the end of the file, we don't have a chunk to read from, so\n    # we can't read anymore. We return the empty string here so we can read off\n    # the end of a file without raising, and get as much data as is there.\n    if chunk > self.last_chunk:\n      return \"\"\n\n    available_to_read = min(length, self.chunksize - chunk_offset)\n\n    fd = self._GetChunkForReading(chunk)\n\n    fd.seek(chunk_offset)\n\n    result = fd.read(available_to_read)\n    self.offset += len(result)\n\n    return result", "response": "Read as much as possible but not more than length."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef AddBlob(self, blob_hash, length, chunk_number):\n    if len(blob_hash.AsBytes()) != self._HASH_SIZE:\n      raise ValueError(\"Hash '%s' doesn't have correct length (%d).\" %\n                       (blob_hash, self._HASH_SIZE))\n\n    # If we're adding a new blob, we should increase the size. If we're just\n    # updating an existing blob, the size should stay the same.\n    # That is, if we read the index at the right offset and no hash is there, we\n    # must not have seen this blob before, so we say we're adding a new one and\n    # increase in size.\n    if not self.ChunkExists(chunk_number):\n      # We say that we've increased in size by the size of the blob,\n      # but really we only store its hash in the AFF4SparseImage.\n      self.size += length\n      self._dirty = True\n      # Keep track of the biggest chunk_number we've seen so far.\n      if chunk_number > self.last_chunk:\n        self.last_chunk = chunk_number\n        self._dirty = True\n\n    index_urn = self.urn.Add(self.CHUNK_ID_TEMPLATE % chunk_number)\n    # TODO(amoser): This opens a subobject for each AddBlob call :/\n    with aff4.FACTORY.Create(\n        index_urn, aff4.AFF4MemoryStream, token=self.token) as fd:\n      fd.write(blob_hash.AsBytes())\n    if chunk_number in self.chunk_cache:\n      self.chunk_cache.Pop(chunk_number)", "response": "Adds a blob to this image using its hash and length."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nchecking if the given chunk numbers exist in the index.", "response": "def ChunksExist(self, chunk_numbers):\n    \"\"\"Do we have this chunk in the index?\"\"\"\n    index_urns = {\n        self.urn.Add(self.CHUNK_ID_TEMPLATE % chunk_number): chunk_number\n        for chunk_number in chunk_numbers\n    }\n\n    res = {chunk_number: False for chunk_number in chunk_numbers}\n\n    for metadata in aff4.FACTORY.Stat(index_urns):\n      res[index_urns[metadata[\"urn\"]]] = True\n\n    return res"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef Flush(self):\n    super(LabelSet, self).Flush()\n\n    self.to_delete = self.to_delete.difference(self.to_set)\n\n    with data_store.DB.GetMutationPool() as mutation_pool:\n      mutation_pool.LabelUpdateLabels(\n          self.urn, self.to_set, to_delete=self.to_delete)\n    self.to_set = set()\n    self.to_delete = set()", "response": "Flush the data to the index."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef WriteCronJob(self, cronjob, cursor=None):\n    query = (\"INSERT INTO cron_jobs \"\n             \"(job_id, job, create_time, enabled) \"\n             \"VALUES (%s, %s, FROM_UNIXTIME(%s), %s) \"\n             \"ON DUPLICATE KEY UPDATE \"\n             \"enabled=VALUES(enabled)\")\n\n    create_time_str = mysql_utils.RDFDatetimeToTimestamp(\n        cronjob.created_at or rdfvalue.RDFDatetime.Now())\n    cursor.execute(query, [\n        cronjob.cron_job_id,\n        cronjob.SerializeToString(), create_time_str, cronjob.enabled\n    ])", "response": "Writes a cronjob to the database."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _CronJobFromRow(self, row):\n    (job, create_time, enabled, forced_run_requested, last_run_status,\n     last_run_time, current_run_id, state, leased_until, leased_by) = row\n\n    job = rdf_cronjobs.CronJob.FromSerializedString(job)\n    job.current_run_id = db_utils.IntToCronJobRunID(current_run_id)\n    job.enabled = enabled\n    job.forced_run_requested = forced_run_requested\n    job.last_run_status = last_run_status\n    job.last_run_time = mysql_utils.TimestampToRDFDatetime(last_run_time)\n    if state:\n      job.state = rdf_protodict.AttributedDict.FromSerializedString(state)\n    job.created_at = mysql_utils.TimestampToRDFDatetime(create_time)\n    job.leased_until = mysql_utils.TimestampToRDFDatetime(leased_until)\n    job.leased_by = leased_by\n    return job", "response": "Creates a cronjob object from a database result row."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef ReadCronJobs(self, cronjob_ids=None, cursor=None):\n    query = (\"SELECT job, UNIX_TIMESTAMP(create_time), enabled, \"\n             \"forced_run_requested, last_run_status, \"\n             \"UNIX_TIMESTAMP(last_run_time), current_run_id, state, \"\n             \"UNIX_TIMESTAMP(leased_until), leased_by \"\n             \"FROM cron_jobs\")\n    if cronjob_ids is None:\n      cursor.execute(query)\n      return [self._CronJobFromRow(row) for row in cursor.fetchall()]\n\n    query += \" WHERE job_id IN (%s)\" % \", \".join([\"%s\"] * len(cronjob_ids))\n    cursor.execute(query, cronjob_ids)\n    res = []\n    for row in cursor.fetchall():\n      res.append(self._CronJobFromRow(row))\n\n    if len(res) != len(cronjob_ids):\n      missing = set(cronjob_ids) - set([c.cron_job_id for c in res])\n      raise db.UnknownCronJobError(\"CronJob(s) with id(s) %s not found.\" %\n                                   missing)\n    return res", "response": "Reads all cronjobs from the database."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nupdates the information for an existing cron job.", "response": "def UpdateCronJob(self,\n                    cronjob_id,\n                    last_run_status=db.Database.unchanged,\n                    last_run_time=db.Database.unchanged,\n                    current_run_id=db.Database.unchanged,\n                    state=db.Database.unchanged,\n                    forced_run_requested=db.Database.unchanged,\n                    cursor=None):\n    \"\"\"Updates run information for an existing cron job.\"\"\"\n    updates = []\n    args = []\n    if last_run_status != db.Database.unchanged:\n      updates.append(\"last_run_status=%s\")\n      args.append(int(last_run_status))\n    if last_run_time != db.Database.unchanged:\n      updates.append(\"last_run_time=FROM_UNIXTIME(%s)\")\n      args.append(mysql_utils.RDFDatetimeToTimestamp(last_run_time))\n    if current_run_id != db.Database.unchanged:\n      updates.append(\"current_run_id=%s\")\n      args.append(db_utils.CronJobRunIDToInt(current_run_id))\n    if state != db.Database.unchanged:\n      updates.append(\"state=%s\")\n      args.append(state.SerializeToString())\n    if forced_run_requested != db.Database.unchanged:\n      updates.append(\"forced_run_requested=%s\")\n      args.append(forced_run_requested)\n\n    if not updates:\n      return\n\n    query = \"UPDATE cron_jobs SET \"\n    query += \", \".join(updates)\n    query += \" WHERE job_id=%s\"\n    res = cursor.execute(query, args + [cronjob_id])\n    if res != 1:\n      raise db.UnknownCronJobError(\"CronJob with id %s not found.\" % cronjob_id)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef LeaseCronJobs(self, cronjob_ids=None, lease_time=None, cursor=None):\n    now = rdfvalue.RDFDatetime.Now()\n    now_str = mysql_utils.RDFDatetimeToTimestamp(now)\n    expiry_str = mysql_utils.RDFDatetimeToTimestamp(now + lease_time)\n    id_str = utils.ProcessIdString()\n\n    query = (\"UPDATE cron_jobs \"\n             \"SET leased_until=FROM_UNIXTIME(%s), leased_by=%s \"\n             \"WHERE (leased_until IS NULL OR leased_until < FROM_UNIXTIME(%s))\")\n    args = [expiry_str, id_str, now_str]\n\n    if cronjob_ids:\n      query += \" AND job_id in (%s)\" % \", \".join([\"%s\"] * len(cronjob_ids))\n      args += cronjob_ids\n\n    updated = cursor.execute(query, args)\n\n    if updated == 0:\n      return []\n\n    cursor.execute(\n        \"SELECT job, UNIX_TIMESTAMP(create_time), enabled,\"\n        \"forced_run_requested, last_run_status, UNIX_TIMESTAMP(last_run_time), \"\n        \"current_run_id, state, UNIX_TIMESTAMP(leased_until), leased_by \"\n        \"FROM cron_jobs \"\n        \"FORCE INDEX (cron_jobs_by_lease) \"\n        \"WHERE leased_until=FROM_UNIXTIME(%s) AND leased_by=%s\",\n        [expiry_str, id_str])\n    return [self._CronJobFromRow(row) for row in cursor.fetchall()]", "response": "Leases all available cron jobs."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef ReturnLeasedCronJobs(self, jobs, cursor=None):\n    if not jobs:\n      return\n\n    unleased_jobs = []\n\n    conditions = []\n    args = []\n    for job in jobs:\n      if not job.leased_by or not job.leased_until:\n        unleased_jobs.append(job)\n        continue\n\n      conditions.append(\"\"\"\n      (job_id = %s AND leased_until = FROM_UNIXTIME(%s) AND leased_by = %s)\n      \"\"\")\n      args += [\n          job.cron_job_id,\n          mysql_utils.RDFDatetimeToTimestamp(job.leased_until), job.leased_by\n      ]\n\n    if conditions:\n      query = (\"UPDATE cron_jobs \"\n               \"SET leased_until=NULL, leased_by=NULL \"\n               \"WHERE \") + \" OR \".join(conditions)\n      returned = cursor.execute(query, args)\n\n    if unleased_jobs:\n      raise ValueError(\"CronJobs to return are not leased: %s\" % unleased_jobs)\n    if returned != len(jobs):\n      raise ValueError(\"%d cronjobs in %s could not be returned.\" %\n                       ((len(jobs) - returned), jobs))", "response": "Returns the leased cron jobs."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef WriteCronJobRun(self, run_object, cursor=None):\n    query = (\"INSERT INTO cron_job_runs \"\n             \"(job_id, run_id, write_time, run) \"\n             \"VALUES (%s, %s, FROM_UNIXTIME(%s), %s) \"\n             \"ON DUPLICATE KEY UPDATE \"\n             \"run=VALUES(run), write_time=VALUES(write_time)\")\n\n    write_time_str = mysql_utils.RDFDatetimeToTimestamp(\n        rdfvalue.RDFDatetime.Now())\n    try:\n      cursor.execute(query, [\n          run_object.cron_job_id,\n          db_utils.CronJobRunIDToInt(run_object.run_id),\n          write_time_str,\n          run_object.SerializeToString(),\n      ])\n    except MySQLdb.IntegrityError as e:\n      raise db.UnknownCronJobError(\n          \"CronJob with id %s not found.\" % run_object.cron_job_id, cause=e)", "response": "Writes a cron job run object to the database."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreading all cron job runs for a given job id.", "response": "def ReadCronJobRuns(self, job_id, cursor=None):\n    \"\"\"Reads all cron job runs for a given job id.\"\"\"\n    query = \"\"\"\n    SELECT run, UNIX_TIMESTAMP(write_time)\n      FROM cron_job_runs\n     WHERE job_id = %s\n    \"\"\"\n    cursor.execute(query, [job_id])\n    runs = [self._CronJobRunFromRow(row) for row in cursor.fetchall()]\n    return sorted(runs, key=lambda run: run.started_at, reverse=True)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef ReadCronJobRun(self, job_id, run_id, cursor=None):\n    query = (\"SELECT run, UNIX_TIMESTAMP(write_time) FROM cron_job_runs \"\n             \"WHERE job_id = %s AND run_id = %s\")\n    num_runs = cursor.execute(\n        query, [job_id, db_utils.CronJobRunIDToInt(run_id)])\n    if num_runs == 0:\n      raise db.UnknownCronJobRunError(\n          \"Run with job id %s and run id %s not found.\" % (job_id, run_id))\n\n    return self._CronJobRunFromRow(cursor.fetchall()[0])", "response": "Reads a single cron job run from the db."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ndeletes cron job runs that are older than the given timestamp.", "response": "def DeleteOldCronJobRuns(self, cutoff_timestamp, cursor=None):\n    \"\"\"Deletes cron job runs that are older then the given timestamp.\"\"\"\n    query = \"DELETE FROM cron_job_runs WHERE write_time < FROM_UNIXTIME(%s)\"\n    cursor.execute(query,\n                   [mysql_utils.RDFDatetimeToTimestamp(cutoff_timestamp)])"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _Insert(cursor, table, values):\n  precondition.AssertIterableType(values, dict)\n\n  if not values:  # Nothing can be INSERTed with empty `values` list.\n    return\n\n  column_names = list(sorted(values[0]))\n  for value_dict in values:\n    if set(column_names) != set(value_dict):\n      raise ValueError(\"Given value dictionaries must have identical keys. \"\n                       \"Expecting columns {!r}, but got value {!r}\".format(\n                           column_names, value_dict))\n\n  query = \"INSERT IGNORE INTO %s {cols} VALUES {vals}\" % table\n  query = query.format(\n      cols=mysql_utils.Columns(column_names),\n      vals=mysql_utils.Placeholders(num=len(column_names), values=len(values)))\n\n  values_list = []\n  for values_dict in values:\n    values_list.extend(values_dict[column] for column in column_names)\n\n  cursor.execute(query, values_list)", "response": "Insert one or multiple rows into the given table."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _BlobToChunks(blob_id, blob):\n  #  In case of empty blob (with empty range), use [0].\n  chunk_begins = list(range(0, len(blob), BLOB_CHUNK_SIZE)) or [0]\n  chunks = []\n  for i, chunk_begin in enumerate(chunk_begins):\n    chunks.append({\n        \"blob_id\": blob_id,\n        \"chunk_index\": i,\n        \"blob_chunk\": blob[chunk_begin:chunk_begin + BLOB_CHUNK_SIZE]\n    })\n  return chunks", "response": "Splits a Blob into chunks of size BLOB_CHUNK_SIZE."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ngroup chunks into partitions of size safe for a single INSERT.", "response": "def _PartitionChunks(chunks):\n  \"\"\"Groups chunks into partitions of size safe for a single INSERT.\"\"\"\n  partitions = [[]]\n  partition_size = 0\n\n  for chunk in chunks:\n    cursize = len(chunk[\"blob_chunk\"])\n    if (cursize + partition_size > BLOB_CHUNK_SIZE or\n        len(partitions[-1]) >= CHUNKS_PER_INSERT):\n      partitions.append([])\n      partition_size = 0\n    partitions[-1].append(chunk)\n    partition_size += cursize\n\n  return partitions"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nchecks if given blobs exist.", "response": "def CheckBlobsExist(self, blob_ids, cursor=None):\n    \"\"\"Checks if given blobs exist.\"\"\"\n    if not blob_ids:\n      return {}\n\n    exists = {blob_id: False for blob_id in blob_ids}\n    query = (\"SELECT blob_id \"\n             \"FROM blobs \"\n             \"FORCE INDEX (PRIMARY) \"\n             \"WHERE blob_id IN {}\".format(\n                 mysql_utils.Placeholders(len(blob_ids))))\n    cursor.execute(query, [blob_id.AsBytes() for blob_id in blob_ids])\n    for blob_id, in cursor.fetchall():\n      exists[rdf_objects.BlobID.FromBytes(blob_id)] = True\n    return exists"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nwrites blob references for a given set of hashes.", "response": "def WriteHashBlobReferences(self, references_by_hash, cursor):\n    \"\"\"Writes blob references for a given set of hashes.\"\"\"\n    values = []\n    for hash_id, blob_refs in iteritems(references_by_hash):\n      refs = rdf_objects.BlobReferences(items=blob_refs).SerializeToString()\n      values.append({\n          \"hash_id\": hash_id.AsBytes(),\n          \"blob_references\": refs,\n      })\n    _Insert(cursor, \"hash_blob_references\", values)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef ReadHashBlobReferences(self, hashes, cursor):\n    query = (\"SELECT hash_id, blob_references FROM hash_blob_references WHERE \"\n             \"hash_id IN {}\").format(mysql_utils.Placeholders(len(hashes)))\n    cursor.execute(query, [hash_id.AsBytes() for hash_id in hashes])\n    results = {hash_id: None for hash_id in hashes}\n    for hash_id, blob_references in cursor.fetchall():\n      sha_hash_id = rdf_objects.SHA256HashID.FromBytes(hash_id)\n      refs = rdf_objects.BlobReferences.FromSerializedString(blob_references)\n      results[sha_hash_id] = list(refs.items)\n    return results", "response": "Reads blob references of a given set of hashes."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef WriteBlobsWithUnknownHashes(\n      self, blobs_data):\n    \"\"\"Calculates hash ids and writes contents of given data blobs.\n\n    Args:\n      blobs_data: An iterable of bytes.\n\n    Returns:\n      A list of rdf_objects.BlobID objects with each blob id corresponding\n      to an element in the original blobs_data argument.\n    \"\"\"\n    blobs_ids = [rdf_objects.BlobID.FromBlobData(d) for d in blobs_data]\n    self.WriteBlobs(dict(zip(blobs_ids, blobs_data)))\n    return blobs_ids", "response": "Calculates hash ids and writes contents of given blobs."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning the client id from the queue q.", "response": "def _GetClientIdFromQueue(q):\n  \"\"\"Returns q's client id, if q is a client task queue, otherwise None.\n\n  Args:\n    q: rdfvalue.RDFURN\n\n  Returns:\n    string or None\n  \"\"\"\n  split = q.Split()\n  if not split or len(split) < 2:\n    return None\n\n  # Normalize to lowercase.\n  split = [s.lower() for s in split]\n\n  str_client_id, tasks_marker = split\n\n  if not str_client_id.startswith(\"c.\") or tasks_marker != \"tasks\":\n    return None\n\n  # The \"C.\" prefix should be uppercase.\n  str_client_id = \"C\" + str_client_id[1:]\n\n  return str_client_id"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ngetting a single shard for a given queue.", "response": "def GetNotificationShard(self, queue):\n    \"\"\"Gets a single shard for a given queue.\"\"\"\n    queue_name = str(queue)\n    QueueManager.notification_shard_counters.setdefault(queue_name, 0)\n    QueueManager.notification_shard_counters[queue_name] += 1\n    notification_shard_index = (\n        QueueManager.notification_shard_counters[queue_name] %\n        self.num_notification_shards)\n    if notification_shard_index > 0:\n      return queue.Add(str(notification_shard_index))\n    else:\n      return queue"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning a copy of the QueueManager object.", "response": "def Copy(self):\n    \"\"\"Return a copy of the queue manager.\n\n    Returns:\n    Copy of the QueueManager object.\n    NOTE: pending writes/deletions are not copied. On the other hand, if the\n    original object has a frozen timestamp, a copy will have it as well.\n    \"\"\"\n    result = QueueManager(store=self.data_store, token=self.token)\n    result.prev_frozen_timestamps = self.prev_frozen_timestamps\n    result.frozen_timestamp = self.frozen_timestamp\n    return result"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef FreezeTimestamp(self):\n    self.prev_frozen_timestamps.append(self.frozen_timestamp)\n    self.frozen_timestamp = rdfvalue.RDFDatetime.Now()", "response": "Freeze the timestamp used for resolve and delete database queries."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nfetching all the requests with a status message queued for them.", "response": "def FetchCompletedRequests(self, session_id, timestamp=None):\n    \"\"\"Fetch all the requests with a status message queued for them.\"\"\"\n    if timestamp is None:\n      timestamp = (0, self.frozen_timestamp or rdfvalue.RDFDatetime.Now())\n\n    for request, status in self.data_store.ReadCompletedRequests(\n        session_id, timestamp=timestamp, limit=self.request_limit):\n      yield request, status"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef FetchCompletedResponses(self, session_id, timestamp=None, limit=10000):\n    if timestamp is None:\n      timestamp = (0, self.frozen_timestamp or rdfvalue.RDFDatetime.Now())\n\n    completed_requests = collections.deque(\n        self.FetchCompletedRequests(session_id, timestamp=timestamp))\n\n    total_size = 0\n    while completed_requests:\n\n      # Size reported in the status messages may be different from actual\n      # number of responses read from the database. Example: hunt responses\n      # may get deleted from the database and then worker may die before\n      # deleting the request. Then status.response_id will be >0, but no\n      # responses will be read from the DB.\n      projected_total_size = total_size\n      request_list = []\n      while completed_requests:\n        request, status = completed_requests.popleft()\n\n        # Make sure at least one response is fetched.\n        request_list.append(request)\n\n        # Quit if there are too many responses.\n        projected_total_size += status.response_id\n        if projected_total_size > limit:\n          break\n\n      for request, responses in self.data_store.ReadResponses(request_list):\n\n        yield (request, responses)\n        total_size += len(responses)\n        if total_size > limit:\n          raise MoreDataException()", "response": "Fetch only completed requests and responses up to a limit."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nfetch all outstanding requests and responses for this flow.", "response": "def FetchRequestsAndResponses(self, session_id, timestamp=None):\n    \"\"\"Fetches all outstanding requests and responses for this flow.\n\n    We first cache all requests and responses for this flow in memory to\n    prevent round trips.\n\n    Args:\n      session_id: The session_id to get the requests/responses for.\n      timestamp: Tuple (start, end) with a time range. Fetched requests and\n        responses will have timestamp in this range.\n\n    Yields:\n      an tuple (request protobufs, list of responses messages) in ascending\n      order of request ids.\n\n    Raises:\n      MoreDataException: When there is more data available than read by the\n                         limited query.\n    \"\"\"\n    if timestamp is None:\n      timestamp = (0, self.frozen_timestamp or rdfvalue.RDFDatetime.Now())\n\n    num_requests = 0\n    for request, responses in self.data_store.ReadRequestsAndResponses(\n        session_id,\n        timestamp=timestamp,\n        request_limit=self.request_limit,\n        response_limit=self.response_limit):\n      yield (request, responses)\n      num_requests += 1\n\n    if num_requests >= self.request_limit:\n      raise MoreDataException()"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ndelete the request and all its responses from the flow state queue.", "response": "def DeleteRequest(self, request):\n    \"\"\"Deletes the request and all its responses from the flow state queue.\"\"\"\n    self.requests_to_delete.append(request)\n\n    if request and request.HasField(\"request\"):\n      self.DeQueueClientRequest(request.request)\n\n    data_store.DB.DeleteRequest(request)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef MultiDestroyFlowStates(self, session_ids):\n    deleted_requests = self.data_store.MultiDestroyFlowStates(\n        session_ids, request_limit=self.request_limit)\n\n    for request in deleted_requests:\n      if request.HasField(\"request\"):\n        # Client request dequeueing is cached so we can call it directly.\n        self.DeQueueClientRequest(request.request)", "response": "Deletes all states in multiple flows and dequeues all client messages."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef Flush(self):\n\n    self.data_store.StoreRequestsAndResponses(\n        new_requests=self.request_queue,\n        new_responses=self.response_queue,\n        requests_to_delete=self.requests_to_delete)\n\n    # We need to make sure that notifications are written after the requests so\n    # we flush after writing all requests and only notify afterwards.\n    mutation_pool = self.data_store.GetMutationPool()\n    with mutation_pool:\n\n      messages_by_queue = collection.Group(\n          list(itervalues(\n              self.client_messages_to_delete)), lambda request: request.queue)\n      for queue, messages in iteritems(messages_by_queue):\n        self.Delete(queue, messages, mutation_pool=mutation_pool)\n\n      if self.new_client_messages:\n        for timestamp, messages in iteritems(\n            collection.Group(self.new_client_messages, lambda x: x[1])):\n\n          self.Schedule([x[0] for x in messages],\n                        timestamp=timestamp,\n                        mutation_pool=mutation_pool)\n\n    if self.notifications:\n      for notification in itervalues(self.notifications):\n        self.NotifyQueue(notification, mutation_pool=mutation_pool)\n\n      mutation_pool.Flush()\n\n    self.request_queue = []\n    self.response_queue = []\n    self.requests_to_delete = []\n\n    self.client_messages_to_delete = {}\n    self.notifications = {}\n    self.new_client_messages = []", "response": "Flushes all the changes in this object to the datastore."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nqueue the message on the flow s state.", "response": "def QueueResponse(self, response, timestamp=None):\n    \"\"\"Queues the message on the flow's state.\"\"\"\n    if timestamp is None:\n      timestamp = self.frozen_timestamp\n    self.response_queue.append((response, timestamp))"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nqueues a notification for a flow.", "response": "def QueueNotification(self, notification=None, timestamp=None, **kw):\n    \"\"\"Queues a notification for a flow.\"\"\"\n\n    if notification is None:\n      notification = rdf_flows.GrrNotification(**kw)\n\n    session_id = notification.session_id\n\n    if session_id:\n      if timestamp is None:\n        timestamp = self.frozen_timestamp\n\n      notification.timestamp = timestamp\n\n      # We must not store more than one notification per session id and\n      # timestamp or there will be race conditions. We therefore only keep\n      # the one with the highest request number (indicated by last_status).\n      # Note that timestamp might be None. In that case we also only want\n      # to keep the latest.\n      if timestamp is None:\n        ts_str = \"None\"\n      else:\n        ts_str = int(timestamp)\n      key = \"%s!%s\" % (session_id, ts_str)\n      existing = self.notifications.get(key)\n      if existing is not None:\n        if existing.last_status < notification.last_status:\n          self.notifications[key] = notification\n      else:\n        self.notifications[key] = notification"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef Delete(self, queue, tasks, mutation_pool=None):\n    if queue is None:\n      return\n    if mutation_pool is None:\n      raise ValueError(\"Mutation pool can't be none.\")\n    mutation_pool.QueueDeleteTasks(queue, tasks)", "response": "Deletes the tasks from the queue."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nschedules a set of Task instances.", "response": "def Schedule(self, tasks, mutation_pool, timestamp=None):\n    \"\"\"Schedule a set of Task() instances.\"\"\"\n    non_fleetspeak_tasks = []\n    for queue, queued_tasks in iteritems(\n        collection.Group(tasks, lambda x: x.queue)):\n      if not queue:\n        continue\n\n      client_id = _GetClientIdFromQueue(queue)\n      if fleetspeak_utils.IsFleetspeakEnabledClient(\n          client_id, token=self.token):\n        for task in queued_tasks:\n          fleetspeak_utils.SendGrrMessageThroughFleetspeak(client_id, task)\n        continue\n      non_fleetspeak_tasks.extend(queued_tasks)\n\n    timestamp = timestamp or self.frozen_timestamp\n    mutation_pool.QueueScheduleTasks(non_fleetspeak_tasks, timestamp)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn all queue notifications.", "response": "def GetNotifications(self, queue):\n    \"\"\"Returns all queue notifications.\"\"\"\n    queue_shard = self.GetNotificationShard(queue)\n    return self._GetUnsortedNotifications(queue_shard).values()"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef GetNotificationsForAllShards(self, queue):\n    notifications_by_session_id = {}\n    for queue_shard in self.GetAllNotificationShards(queue):\n      self._GetUnsortedNotifications(\n          queue_shard, notifications_by_session_id=notifications_by_session_id)\n\n    return notifications_by_session_id.values()", "response": "Returns all notifications for all shards of a queue at once."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns all the unsorted notifications for a queue_shard.", "response": "def _GetUnsortedNotifications(self,\n                                queue_shard,\n                                notifications_by_session_id=None):\n    \"\"\"Returns all the available notifications for a queue_shard.\n\n    Args:\n      queue_shard: urn of queue shard\n      notifications_by_session_id: store notifications in this dict rather than\n        creating a new one\n\n    Returns:\n      dict of notifications. keys are session ids.\n    \"\"\"\n    if notifications_by_session_id is None:\n      notifications_by_session_id = {}\n    end_time = self.frozen_timestamp or rdfvalue.RDFDatetime.Now()\n    for notification in self.data_store.GetNotifications(queue_shard, end_time):\n\n      existing = notifications_by_session_id.get(notification.session_id)\n      if existing:\n        # If we have a notification for this session_id already, we only store\n        # the one that was scheduled last.\n        if notification.first_queued > existing.first_queued:\n          notifications_by_session_id[notification.session_id] = notification\n        elif notification.first_queued == existing.first_queued and (\n            notification.last_status > existing.last_status):\n          # Multiple notifications with the same timestamp should not happen.\n          # We can still do the correct thing and use the latest one.\n          logging.warning(\n              \"Notifications with equal first_queued fields detected: %s %s\",\n              notification, existing)\n          notifications_by_session_id[notification.session_id] = notification\n      else:\n        notifications_by_session_id[notification.session_id] = notification\n\n    return notifications_by_session_id"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef MultiNotifyQueue(self, notifications, mutation_pool=None):\n    extract_queue = lambda notification: notification.session_id.Queue()\n    for queue, notifications in iteritems(\n        collection.Group(notifications, extract_queue)):\n      self._MultiNotifyQueue(queue, notifications, mutation_pool=mutation_pool)", "response": "This is the same as NotifyQueue but for several session_ids at once."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _MultiNotifyQueue(self, queue, notifications, mutation_pool=None):\n    notification_list = []\n    now = rdfvalue.RDFDatetime.Now()\n    for notification in notifications:\n      if not notification.first_queued:\n        notification.first_queued = (\n            self.frozen_timestamp or rdfvalue.RDFDatetime.Now())\n      else:\n        diff = now - notification.first_queued\n        if diff.seconds >= self.notification_expiry_time:\n          # This notification has been around for too long, we drop it.\n          logging.debug(\"Dropping notification: %s\", str(notification))\n          continue\n\n      notification_list.append(notification)\n\n    mutation_pool.CreateNotifications(\n        self.GetNotificationShard(queue), notification_list)", "response": "Does the actual queuing."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nretrieves all the tasks from a queue without leasing them.", "response": "def Query(self, queue, limit=1):\n    \"\"\"Retrieves tasks from a queue without leasing them.\n\n    This is good for a read only snapshot of the tasks.\n\n    Args:\n       queue: The task queue that this task belongs to, usually client.Queue()\n         where client is the ClientURN object you want to schedule msgs on.\n       limit: Number of values to fetch.\n\n    Returns:\n        A list of Task() objects.\n    \"\"\"\n    # This function is usually used for manual testing so we also accept client\n    # ids and get the queue from it.\n    if isinstance(queue, rdf_client.ClientURN):\n      queue = queue.Queue()\n\n    return self.data_store.QueueQueryTasks(queue, limit=limit)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef QueryAndOwn(self, queue, lease_seconds=10, limit=1):\n    with self.data_store.GetMutationPool() as mutation_pool:\n      return mutation_pool.QueueQueryAndOwn(queue, lease_seconds, limit,\n                                            self.frozen_timestamp)", "response": "Returns a list of GrrMessage objects leased for a certain time."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef FetchResponses(self, session_id):\n    timestamp = (0, self.frozen_timestamp or rdfvalue.RDFDatetime.Now())\n\n    for response in self.data_store.FetchResponsesForWellKnownFlow(\n        session_id, self.response_limit, timestamp=timestamp):\n      yield response", "response": "Retrieves the responses for a well known flow."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nmigrating one Artifact from AFF4 to REL_DB.", "response": "def _MigrateArtifact(artifact):\n  \"\"\"Migrate one Artifact from AFF4 to REL_DB.\"\"\"\n  name = Text(artifact.name)\n\n  try:\n    logging.info(\"Migating %s\", name)\n    data_store.REL_DB.WriteArtifact(artifact)\n    logging.info(\"  Wrote %s\", name)\n  except db.DuplicatedArtifactError:\n    logging.info(\"  Skipped %s, because artifact already exists.\", name)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef MigrateArtifacts():\n\n  # First, delete all existing artifacts in REL_DB.\n  artifacts = data_store.REL_DB.ReadAllArtifacts()\n  if artifacts:\n    logging.info(\"Deleting %d artifacts from REL_DB.\", len(artifacts))\n    for artifact in data_store.REL_DB.ReadAllArtifacts():\n      data_store.REL_DB.DeleteArtifact(Text(artifact.name))\n  else:\n    logging.info(\"No artifacts found in REL_DB.\")\n\n  artifacts = artifact_registry.REGISTRY.GetArtifacts(\n      reload_datastore_artifacts=True)\n\n  logging.info(\"Found %d artifacts in AFF4.\", len(artifacts))\n\n  # Only migrate user-created artifacts.\n  artifacts = list(filter(_IsCustom, artifacts))\n\n  logging.info(\"Migrating %d user-created artifacts.\", len(artifacts))\n\n  for artifact in artifacts:\n    _MigrateArtifact(artifact)", "response": "Migrates Artifacts from AFF4 to REL_DB."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _GetBaseURLs(self):\n    result = config.CONFIG[\"Client.server_urls\"]\n    if not result:\n      # Backwards compatibility - deduce server_urls from Client.control_urls.\n      for control_url in config.CONFIG[\"Client.control_urls\"]:\n        result.append(posixpath.dirname(control_url) + \"/\")\n\n    # Check the URLs for trailing /. This traps configuration errors.\n    for url in result:\n      if not url.endswith(\"/\"):\n        raise RuntimeError(\"Bad URL: %s URLs must end with /\" % url)\n\n    return result", "response": "Gathers a list of base URLs we will try."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _GetProxies(self):\n    # Detect proxies from the OS environment.\n    result = client_utils.FindProxies()\n\n    # Also try to connect directly if all proxies fail.\n    result.append(\"\")\n\n    # Also try all proxies configured in the config system.\n    result.extend(config.CONFIG[\"Client.proxy_servers\"])\n\n    return result", "response": "Gather a list of proxies to use."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nopening a server endpoint.", "response": "def OpenServerEndpoint(self,\n                         path,\n                         verify_cb=lambda x: True,\n                         data=None,\n                         params=None,\n                         headers=None,\n                         method=\"GET\",\n                         timeout=None):\n    \"\"\"Search through all the base URLs to connect to one that works.\n\n    This is a thin wrapper around requests.request() so most parameters are\n    documented there.\n\n    Args:\n      path: The URL path to access in this endpoint.\n      verify_cb: A callback which should return True if the response is\n        reasonable. This is used to detect if we are able to talk to the correct\n        endpoint. If not we try a different endpoint/proxy combination.\n      data: Parameters to send in POST bodies (See Requests documentation).\n      params: Parameters to send in GET URLs (See Requests documentation).\n      headers: Additional headers (See Requests documentation)\n      method: The HTTP method to use. If not set we select one automatically.\n      timeout: See Requests documentation.\n\n    Returns:\n      an HTTPObject() instance with the correct error code set.\n    \"\"\"\n    tries = 0\n    last_error = HTTPObject(code=404)\n\n    while tries < len(self.base_urls):\n      base_url_index = self.last_base_url_index % len(self.base_urls)\n      active_base_url = self.base_urls[base_url_index]\n\n      result = self.OpenURL(\n          self._ConcatenateURL(active_base_url, path),\n          data=data,\n          params=params,\n          headers=headers,\n          method=method,\n          timeout=timeout,\n          verify_cb=verify_cb,\n      )\n\n      if not result.Success():\n        tries += 1\n        self.last_base_url_index += 1\n        last_error = result\n        continue\n\n      # The URL worked - we record that.\n      self.active_base_url = active_base_url\n\n      return result\n\n    # No connection is possible at all.\n    logging.info(\n        \"Could not connect to GRR servers %s, directly or through \"\n        \"these proxies: %s.\", self.base_urls, self.proxies)\n\n    return last_error"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nopens a URL and return an HTTPObject.", "response": "def OpenURL(self,\n              url,\n              verify_cb=lambda x: True,\n              data=None,\n              params=None,\n              headers=None,\n              method=\"GET\",\n              timeout=None):\n    \"\"\"Get the requested URL.\n\n    Note that we do not have any concept of timing here - we try to connect\n    through all proxies as fast as possible until one works. Timing and poll\n    frequency is left to the calling code.\n\n    Args:\n      url: The URL to fetch\n      verify_cb: An optional callback which can be used to validate the URL. It\n        receives the HTTPObject and return True if this seems OK, False\n        otherwise. For example, if we are behind a captive portal we might\n        receive invalid object even though HTTP status is 200.\n      data: Parameters to send in POST bodies (See Requests documentation).\n      params: Parameters to send in GET URLs (See Requests documentation).\n      headers: Additional headers (See Requests documentation)\n      method: The HTTP method to use. If not set we select one automatically.\n      timeout: See Requests documentation.\n\n    Returns:\n      An HTTPObject instance or None if a connection could not be made.\n\n    \"\"\"\n    # Start checking the proxy from the last value found.\n    tries = 0\n    last_error = 500\n\n    while tries < len(self.proxies):\n      proxy_index = self.last_proxy_index % len(self.proxies)\n      proxy = self.proxies[proxy_index]\n      try:\n        proxydict = {}\n        if proxy:\n          proxydict[\"http\"] = proxy\n          proxydict[\"https\"] = proxy\n\n        headers = (headers or {}).copy()\n        headers[\"Cache-Control\"] = \"no-cache\"\n\n        if data:\n          method = \"POST\"\n\n        duration, handle = self._RetryRequest(\n            url=url,\n            data=data,\n            params=params,\n            headers=headers,\n            method=method,\n            timeout=timeout,\n            proxies=proxydict,\n        )\n\n        data = handle.content\n\n        result = HTTPObject(\n            url=url, data=data, proxy=proxy, code=200, duration=duration)\n\n        if not verify_cb(result):\n          raise IOError(\"Data not verified.\")\n\n        # The last connection worked.\n        self.consecutive_connection_errors = 0\n        return result\n\n      except requests.RequestException as e:\n        # Especially trap a 406 error message - it means the client needs to\n        # enroll.\n        if e.response is not None:\n          last_error = e.response.status_code\n          if last_error == 406:\n            # A 406 is not considered an error as the frontend is reachable. If\n            # we considered it as an error the client would be unable to send\n            # the enrollment request since connection errors disable message\n            # draining.\n            self.consecutive_connection_errors = 0\n            return HTTPObject(code=406)\n\n        # Try the next proxy\n        self.last_proxy_index = proxy_index + 1\n        tries += 1\n\n      # Catch any exceptions that dont have a code (e.g. socket.error).\n      except IOError:\n        # Try the next proxy\n        self.last_proxy_index = proxy_index + 1\n        tries += 1\n        last_error = 500\n      # Catch unexpected exceptions. If the error is proxy related it makes\n      # sense to cycle the proxy before reraising. One error we have seen here\n      # is ProxySchemeUnknown but urllib can raise many different exceptions, it\n      # doesn't make sense to enumerate them all.\n      except Exception:  # pylint: disable=broad-except\n        logging.exception(\n            \"Got an unexpected exception while connecting to the server.\")\n        # Try the next proxy\n        self.last_proxy_index = proxy_index + 1\n        tries += 1\n        last_error = 500\n\n    # We failed to connect at all here.\n    return HTTPObject(code=last_error)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _RetryRequest(self, timeout=None, **request_args):\n    while True:\n      try:\n        now = time.time()\n        if not timeout:\n          timeout = config.CONFIG[\"Client.http_timeout\"]\n\n        result = requests.request(**request_args)\n        # By default requests doesn't raise on HTTP error codes.\n        result.raise_for_status()\n\n        # Requests does not always raise an exception when an incorrect response\n        # is received. This fixes that behaviour.\n        if not result.ok:\n          raise requests.RequestException(response=result)\n\n        return time.time() - now, result\n\n      # Catch any exceptions that dont have a code (e.g. socket.error).\n      except IOError as e:\n        self.consecutive_connection_errors += 1\n        # Request failed. If we connected successfully before we attempt a few\n        # connections before we determine that it really failed. This might\n        # happen if the front end is loaded and returns a few throttling 500\n        # messages.\n        if self.active_base_url is not None:\n          # Propagate 406 immediately without retrying, as 406 is a valid\n          # response that indicates a need for enrollment.\n          response = getattr(e, \"response\", None)\n          if getattr(response, \"status_code\", None) == 406:\n            raise\n\n          if self.consecutive_connection_errors >= self.retry_error_limit:\n            # We tried several times but this really did not work, just fail it.\n            logging.info(\n                \"Too many connection errors to %s, retrying another URL\",\n                self.active_base_url)\n            self.active_base_url = None\n            raise e\n\n          # Back off hard to allow the front end to recover.\n          logging.debug(\n              \"Unable to connect to frontend. Backing off %s seconds.\",\n              self.error_poll_min)\n          self.Wait(self.error_poll_min)\n\n        # We never previously connected, maybe the URL/proxy is wrong? Just fail\n        # right away to allow callers to try a different URL.\n        else:\n          raise e", "response": "Retry the request until the timeout is reached."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef Wait(self, timeout):\n    time.sleep(timeout - int(timeout))\n\n    # Split a long sleep interval into 1 second intervals so we can heartbeat.\n    for _ in range(int(timeout)):\n      time.sleep(1)\n\n      if self.heart_beat_cb:\n        self.heart_beat_cb()", "response": "Wait for the specified timeout."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nwaiting until the next action is needed.", "response": "def Wait(self):\n    \"\"\"Wait until the next action is needed.\"\"\"\n    time.sleep(self.sleep_time - int(self.sleep_time))\n\n    # Split a long sleep interval into 1 second intervals so we can heartbeat.\n    for _ in range(int(self.sleep_time)):\n      time.sleep(1)\n\n    # Back off slowly at first and fast if no answer.\n    self.sleep_time = min(self.poll_max,\n                          max(self.poll_min, self.sleep_time) * self.poll_slew)"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\npushes the Serialized Message on the output queue.", "response": "def QueueResponse(self, message, blocking=True):\n    \"\"\"Pushes the Serialized Message on the output queue.\"\"\"\n    self._out_queue.Put(message, block=blocking)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef QueueMessages(self, messages):\n    # Push all the messages to our input queue\n    for message in messages:\n      self._in_queue.put(message, block=True)", "response": "Pushes a list of messages to the input queue."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef SendReply(self,\n                rdf_value=None,\n                request_id=None,\n                response_id=None,\n                session_id=\"W:0\",\n                message_type=None,\n                name=None,\n                require_fastpoll=None,\n                ttl=None,\n                blocking=True,\n                task_id=None):\n    \"\"\"Send the protobuf to the server.\n\n    Args:\n      rdf_value: The RDFvalue to return.\n      request_id: The id of the request this is a response to.\n      response_id: The id of this response.\n      session_id: The session id of the flow.\n      message_type: The contents of this message, MESSAGE, STATUS or\n        RDF_VALUE.\n      name: The name of the client action that sends this response.\n      require_fastpoll: If set, this will set the client to fastpoll mode after\n        sending this message.\n      ttl: The time to live of this message.\n      blocking: If the output queue is full, block until there is space.\n      task_id: The task ID that the request was queued at. We send this back to\n        the server so it can de-queue the request.\n\n    Raises:\n      RuntimeError: An object other than an RDFValue was passed for sending.\n    \"\"\"\n    if not isinstance(rdf_value, rdfvalue.RDFValue):\n      raise RuntimeError(\"Sending objects other than RDFValues not supported.\")\n\n    message = rdf_flows.GrrMessage(\n        session_id=session_id,\n        task_id=task_id,\n        name=name,\n        response_id=response_id,\n        request_id=request_id,\n        require_fastpoll=require_fastpoll,\n        ttl=ttl,\n        type=message_type)\n\n    if rdf_value is not None:\n      message.payload = rdf_value\n\n    serialized_message = message.SerializeToString()\n\n    self.ChargeBytesToSession(session_id, len(serialized_message))\n\n    if message.type == rdf_flows.GrrMessage.Type.STATUS:\n      rdf_value.network_bytes_sent = self.sent_bytes_per_flow[session_id]\n      del self.sent_bytes_per_flow[session_id]\n      message.payload = rdf_value\n\n    try:\n      self.QueueResponse(message, blocking=blocking)\n    except queue.Full:\n      # In the case of a non blocking send, we reraise the exception to notify\n      # the caller that something went wrong.\n      if not blocking:\n        raise\n\n      # There is nothing we can do about it here - we just lose the message and\n      # keep going.\n      logging.info(\"Queue is full, dropping messages.\")", "response": "Sends a reply to the server."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef HandleMessage(self, message):\n    self._is_active = True\n    try:\n      action_cls = actions.ActionPlugin.classes.get(message.name)\n      if action_cls is None:\n        raise RuntimeError(\"Client action %r not known\" % message.name)\n\n      action = action_cls(grr_worker=self)\n\n      # Write the message to the transaction log.\n      self.transaction_log.Write(message)\n\n      # Heartbeat so we have the full period to work on this message.\n      action.Progress()\n      action.Execute(message)\n\n      # If we get here without exception, we can remove the transaction.\n      self.transaction_log.Clear()\n    finally:\n      self._is_active = False\n      # We want to send ClientStats when client action is complete.\n      self.stats_collector.RequestSend()", "response": "Handles a GrrMessage received from the server."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef MemoryExceeded(self):\n    rss_size = self.proc.memory_info().rss\n    return rss_size // 1024 // 1024 > config.CONFIG[\"Client.rss_max\"]", "response": "Returns True if our memory footprint is too large."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nsleeps the calling thread with heartbeat.", "response": "def Sleep(self, timeout):\n    \"\"\"Sleeps the calling thread with heartbeat.\"\"\"\n    if self.nanny_controller:\n      self.nanny_controller.Heartbeat()\n\n    # Split a long sleep interval into 1 second intervals so we can heartbeat.\n    while timeout > 0:\n      time.sleep(min(1., timeout))\n      timeout -= 1\n      # If the output queue is full, we are ready to do a post - no\n      # point in waiting.\n      if self._out_queue.Full():\n        return\n\n      if self.nanny_controller:\n        self.nanny_controller.Heartbeat()"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef OnStartup(self):\n    # We read the transaction log and fail any requests that are in it. If there\n    # is anything in the transaction log we assume its there because we crashed\n    # last time and let the server know.\n\n    last_request = self.transaction_log.Get()\n    if last_request:\n      status = rdf_flows.GrrStatus(\n          status=rdf_flows.GrrStatus.ReturnedStatus.CLIENT_KILLED,\n          error_message=\"Client killed during transaction\")\n      if self.nanny_controller:\n        nanny_status = self.nanny_controller.GetNannyStatus()\n        if nanny_status:\n          status.nanny_status = nanny_status\n\n      self.SendReply(\n          status,\n          request_id=last_request.request_id,\n          response_id=1,\n          session_id=last_request.session_id,\n          message_type=rdf_flows.GrrMessage.Type.STATUS)\n\n    self.transaction_log.Clear()\n\n    # Inform the server that we started.\n    action = admin.SendStartupInfo(grr_worker=self)\n    action.Run(None, ttl=1)", "response": "Called when the client starts up."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nput a message on the queue.", "response": "def Put(self, message, block=True, timeout=1000):\n    \"\"\"Put a message on the queue, blocking if it is too full.\n\n    Blocks when the queue contains more than the threshold.\n\n    Args:\n      message: rdf_flows.GrrMessage The message to put.\n      block: bool If True, we block and wait for the queue to have more space.\n        Otherwise, if the queue is full, we raise.\n      timeout: int Maximum time (in seconds, with 1 sec resolution) we spend\n        waiting on the queue.\n\n    Raises:\n      queue.Full: if the queue is full and block is False, or\n        timeout is exceeded.\n    \"\"\"\n    # We only queue already serialized objects so we know how large they are.\n    message = message.SerializeToString()\n\n    if not block:\n      if self.Full():\n        raise queue.Full\n\n    else:\n      t0 = time.time()\n      while self.Full():\n        time.sleep(1)\n        self._heart_beat_cb()\n\n        if time.time() - t0 > timeout:\n          raise queue.Full\n\n    with self._lock:\n      self._queue.appendleft(message)\n      self._total_size += len(message)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nretrieving and removes the messages from the queue.", "response": "def GetMessages(self, soft_size_limit=None):\n    \"\"\"Retrieves and removes the messages from the queue.\n\n    Args:\n      soft_size_limit: int If there is more data in the queue than\n        soft_size_limit bytes, the returned list of messages will be\n        approximately this large. If None (default), returns all messages\n        currently on the queue.\n\n    Returns:\n      rdf_flows.MessageList A list of messages that were .Put on the queue\n      earlier.\n    \"\"\"\n    with self._lock:\n      ret = rdf_flows.MessageList()\n      ret_size = 0\n      for message in self._Generate():\n        self._total_size -= len(message)\n        ret.job.append(rdf_flows.GrrMessage.FromSerializedString(message))\n        ret_size += len(message)\n        if soft_size_limit is not None and ret_size > soft_size_limit:\n          break\n\n      return ret"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nverify the server PEM for validity.", "response": "def VerifyServerPEM(self, http_object):\n    \"\"\"Check the server PEM for validity.\n\n    This is used to determine connectivity to the server. Sometimes captive\n    portals return a valid HTTP status, but the data is corrupted.\n\n    Args:\n      http_object: The response received from the server.\n\n    Returns:\n      True if the response contains a valid server certificate.\n    \"\"\"\n    try:\n      server_pem = http_object.data\n      server_url = http_object.url\n\n      if b\"BEGIN CERTIFICATE\" in server_pem:\n        # Now we know that this proxy is working. We still have to verify the\n        # certificate. This will raise if the server cert is invalid.\n        server_certificate = rdf_crypto.RDFX509Cert(server_pem)\n        self.communicator.LoadServerCertificate(\n            server_certificate=server_certificate, ca_certificate=self.ca_cert)\n\n        logging.info(\"Server PEM re-keyed.\")\n        return True\n    except Exception as e:  # pylint: disable=broad-except\n      logging.info(\"Unable to verify server certificate at %s: %s\", server_url,\n                   e)\n\n      return False"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef VerifyServerControlResponse(self, http_object):\n    if http_object.code != 200:\n      return False\n\n    # Try to decrypt the message into the http_object.\n    try:\n      http_object.messages, http_object.source, http_object.nonce = (\n          self.communicator.DecryptMessage(http_object.data))\n\n      return True\n\n    # Something went wrong - the response seems invalid!\n    except communicator.DecodingError as e:\n      logging.info(\"Protobuf decode error: %s.\", e)\n      return False", "response": "Verifies the server response to a control endpoint POST message."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef MakeRequest(self, data):\n    stats_collector_instance.Get().IncrementCounter(\"grr_client_sent_bytes\",\n                                                    len(data))\n\n    # Verify the response is as it should be from the control endpoint.\n    response = self.http_manager.OpenServerEndpoint(\n        path=\"control?api=%s\" % config.CONFIG[\"Network.api\"],\n        verify_cb=self.VerifyServerControlResponse,\n        data=data,\n        headers={\"Content-Type\": \"binary/octet-stream\"})\n\n    if response.code == 406:\n      self.InitiateEnrolment()\n      return response\n\n    if response.code == 200:\n      stats_collector_instance.Get().IncrementCounter(\n          \"grr_client_received_bytes\", len(response.data))\n      return response\n\n    # An unspecified error occured.\n    return response", "response": "Make a HTTP POST request to the server control endpoint."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef RunOnce(self):\n    # Attempt to fetch and load server certificate.\n    if not self._FetchServerCertificate():\n      self.timer.Wait()\n      return HTTPObject(code=500)\n\n    # Here we only drain messages if we were able to connect to the server in\n    # the last poll request. Otherwise we just wait until the connection comes\n    # back so we don't expire our messages too fast.\n    if self.http_manager.consecutive_connection_errors == 0:\n      # Grab some messages to send\n      message_list = self.client_worker.Drain(\n          max_size=config.CONFIG[\"Client.max_post_size\"])\n    else:\n      message_list = rdf_flows.MessageList()\n\n    # If any outbound messages require fast poll we switch to fast poll mode.\n    for message in message_list.job:\n      if message.require_fastpoll:\n        self.timer.FastPoll()\n        break\n\n    # Make new encrypted ClientCommunication rdfvalue.\n    payload = rdf_flows.ClientCommunication()\n\n    # If our memory footprint is too large, we advertise that our input queue\n    # is full. This will prevent the server from sending us any messages, and\n    # hopefully allow us to work down our memory usage, by processing any\n    # outstanding messages.\n    if self.client_worker.MemoryExceeded():\n      logging.info(\"Memory exceeded, will not retrieve jobs.\")\n      payload.queue_size = 1000000\n    else:\n      # Let the server know how many messages are currently queued in\n      # the input queue.\n      payload.queue_size = self.client_worker.InQueueSize()\n\n    nonce = self.communicator.EncodeMessages(message_list, payload)\n    payload_data = payload.SerializeToString()\n    response = self.MakeRequest(payload_data)\n\n    # Unable to decode response or response not valid.\n    if response.code != 200 or response.messages is None:\n      # We don't print response here since it should be encrypted and will\n      # cause ascii conversion errors.\n      logging.info(\"%s: Could not connect to server at %s, status %s\",\n                   self.communicator.common_name,\n                   self.http_manager.active_base_url, response.code)\n\n      # Force the server pem to be reparsed on the next connection.\n      self.server_certificate = None\n\n      # Reschedule the tasks back on the queue so they get retried next time.\n      messages = list(message_list.job)\n      for message in messages:\n        message.require_fastpoll = False\n        message.ttl -= 1\n        if message.ttl > 0:\n          self.client_worker.QueueResponse(message)\n        else:\n          logging.info(\"Dropped message due to retransmissions.\")\n\n      return response\n\n    # Check the decoded nonce was as expected.\n    if response.nonce != nonce:\n      logging.info(\"Nonce not matched.\")\n      response.code = 500\n      return response\n\n    if response.source != self.communicator.server_name:\n      logging.info(\"Received a message not from the server \"\n                   \"%s, expected %s.\", response.source,\n                   self.communicator.server_name)\n      response.code = 500\n      return response\n\n    # Check to see if any inbound messages want us to fastpoll. This means we\n    # drop to fastpoll immediately on a new request rather than waiting for the\n    # next beacon to report results.\n    for message in response.messages:\n      if message.require_fastpoll:\n        self.timer.FastPoll()\n        break\n\n    # Process all messages. Messages can be processed by clients in\n    # any order since clients do not have state.\n    self.client_worker.QueueMessages(response.messages)\n\n    cn = self.communicator.common_name\n    logging.info(\n        \"%s: Sending %s(%s), Received %s messages in %s sec. \"\n        \"Sleeping for %s sec.\", cn, len(message_list), len(payload_data),\n        len(response.messages), response.duration, self.timer.sleep_time)\n\n    return response", "response": "Makes a single request to the GRR server."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nattempting to fetch the server certificate. Returns True if we succeed.", "response": "def _FetchServerCertificate(self):\n    \"\"\"Attempts to fetch the server cert.\n\n    Returns:\n      True if we succeed.\n    \"\"\"\n    # Certificate is loaded and still valid.\n    if self.server_certificate:\n      return True\n\n    response = self.http_manager.OpenServerEndpoint(\n        \"server.pem\", verify_cb=self.VerifyServerPEM)\n\n    if response.Success():\n      self.server_certificate = response.data\n      return True\n\n    # We failed to fetch the cert, switch to slow poll mode.\n    self.timer.SlowPoll()\n    return False"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef Run(self):\n    while True:\n      if self.http_manager.ErrorLimitReached():\n        return\n\n      # Check if there is a message from the nanny to be sent.\n      self.client_worker.SendNannyMessage()\n\n      now = time.time()\n      # Check with the foreman if we need to\n      if (now > self.last_foreman_check +\n          config.CONFIG[\"Client.foreman_check_frequency\"]):\n        # We must not queue messages from the comms thread with blocking=True\n        # or we might deadlock. If the output queue is full, we can't accept\n        # more work from the foreman anyways so it's ok to drop the message.\n        try:\n          self.client_worker.SendReply(\n              rdf_protodict.DataBlob(),\n              session_id=rdfvalue.FlowSessionID(flow_name=\"Foreman\"),\n              require_fastpoll=False,\n              blocking=False)\n          self.last_foreman_check = now\n        except queue.Full:\n          pass\n\n      try:\n        self.RunOnce()\n      except Exception:  # pylint: disable=broad-except\n        # Catch everything, yes, this is terrible but necessary\n        logging.warning(\"Uncaught exception caught: %s\", traceback.format_exc())\n        if flags.FLAGS.pdb_post_mortem:\n          pdb.post_mortem()\n\n      # We suicide if our memory is exceeded, and there is no more work to do\n      # right now. Our death should not result in loss of messages since we are\n      # not holding any requests in our input queues.\n      if (self.client_worker.MemoryExceeded() and\n          not self.client_worker.IsActive() and\n          self.client_worker.InQueueSize() == 0 and\n          self.client_worker.OutQueueSize() == 0):\n        logging.warning(\"Memory exceeded - exiting.\")\n        self.client_worker.SendClientAlert(\"Memory limit exceeded, exiting.\")\n        # Make sure this will return True so we don't get more work.\n        # pylint: disable=g-bad-name\n        self.client_worker.MemoryExceeded = lambda: True\n        # pylint: enable=g-bad-name\n        # Now send back the client message.\n        self.RunOnce()\n        # And done for now.\n        sys.exit(-1)\n\n      self.timer.Wait()", "response": "This method is called by the client thread to send a message to the nanny server."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef InitiateEnrolment(self):\n    logging.debug(\"sending enrollment request\")\n    now = time.time()\n    if now > self.last_enrollment_time + 10 * 60:\n      if not self.last_enrollment_time:\n        # This is the first enrollment request - we should enter fastpoll mode.\n        self.timer.FastPoll()\n\n      self.last_enrollment_time = now\n      # Send registration request:\n      self.client_worker.SendReply(\n          rdf_crypto.Certificate(\n              type=rdf_crypto.Certificate.Type.CSR,\n              pem=self.communicator.GetCSRAsPem()),\n          session_id=rdfvalue.SessionID(\n              queue=queues.ENROLLMENT, flow_name=\"Enrol\"))", "response": "Initiate the enrollment process."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ninitializes a new RSA key for this client.", "response": "def InitPrivateKey(self):\n    \"\"\"Makes sure this client has a private key set.\n\n    It first tries to load an RSA key from the certificate.\n\n    If no certificate is found, or it is invalid, we make a new random RSA key,\n    and store it as our certificate.\n\n    Returns:\n      An RSA key - either from the certificate or a new random key.\n    \"\"\"\n    if self.private_key:\n      try:\n        self.common_name = rdf_client.ClientURN.FromPrivateKey(self.private_key)\n\n        logging.info(\"Starting client %s\", self.common_name)\n\n        return self.private_key\n\n      except type_info.TypeValueError:\n        pass\n\n    # We either have an invalid key or no key. We just generate a new one.\n    key = rdf_crypto.RSAPrivateKey.GenerateKey(\n        bits=config.CONFIG[\"Client.rsa_key_length\"])\n\n    self.common_name = rdf_client.ClientURN.FromPrivateKey(key)\n    logging.info(\"Client pending enrolment %s\", self.common_name)\n\n    # Save the keys\n    self.SavePrivateKey(key)\n\n    return key"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nstores the new private key on disk.", "response": "def SavePrivateKey(self, private_key):\n    \"\"\"Store the new private key on disk.\"\"\"\n    self.private_key = private_key\n    config.CONFIG.Set(\"Client.private_key\",\n                      self.private_key.SerializeToString())\n    config.CONFIG.Write()"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nload and verifies the server certificate.", "response": "def LoadServerCertificate(self, server_certificate=None, ca_certificate=None):\n    \"\"\"Loads and verifies the server certificate.\"\"\"\n    # Check that the server certificate verifies\n    try:\n      server_certificate.Verify(ca_certificate.GetPublicKey())\n    except rdf_crypto.VerificationError as e:\n      self.server_name = None\n      raise IOError(\"Server cert is invalid: %s\" % e)\n\n    # Make sure that the serial number is higher.\n    server_cert_serial = server_certificate.GetSerialNumber()\n\n    if server_cert_serial < config.CONFIG[\"Client.server_serial_number\"]:\n      # We can not accept this serial number...\n      raise IOError(\"Server certificate serial number is too old.\")\n    elif server_cert_serial > config.CONFIG[\"Client.server_serial_number\"]:\n      logging.info(\"Server serial number updated to %s\", server_cert_serial)\n      config.CONFIG.Set(\"Client.server_serial_number\", server_cert_serial)\n\n      # Save the new data to the config file.\n      config.CONFIG.Write()\n\n    self.server_name = server_certificate.GetCN()\n    self.server_certificate = server_certificate\n    self.ca_certificate = ca_certificate\n    self.server_public_key = server_certificate.GetPublicKey()\n    # If we still have a cached session key, we need to remove it.\n    self._ClearServerCipherCache()"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef ClaimNotificationsForCollection(cls,\n                                      token=None,\n                                      start_time=None,\n                                      lease_time=200,\n                                      collection=None):\n    \"\"\"Return unclaimed hunt result notifications for collection.\n\n    Args:\n      token: The security token to perform database operations with.\n      start_time: If set, an RDFDateTime indicating at what point to start\n        claiming notifications. Only notifications with a timestamp after this\n        point will be claimed.\n      lease_time: How long to claim the notifications for.\n      collection: The urn of the collection to find notifications for. If unset,\n        the earliest (unclaimed) notification will determine the collection.\n\n    Returns:\n      A pair (collection, results) where collection is the collection\n      that notifications were retrieved for and results is a list of\n      Record objects which identify GrrMessage within the result\n      collection.\n    \"\"\"\n\n    class CollectionFilter(object):\n\n      def __init__(self, collection):\n        self.collection = collection\n\n      def FilterRecord(self, notification):\n        if self.collection is None:\n          self.collection = notification.result_collection_urn\n        return self.collection != notification.result_collection_urn\n\n    f = CollectionFilter(collection)\n    results = []\n    with aff4.FACTORY.OpenWithLock(\n        RESULT_NOTIFICATION_QUEUE,\n        aff4_type=HuntResultQueue,\n        lease_time=300,\n        blocking=True,\n        blocking_sleep_interval=15,\n        blocking_lock_timeout=600,\n        token=token) as queue:\n      for record in queue.ClaimRecords(\n          record_filter=f.FilterRecord,\n          start_time=start_time,\n          timeout=lease_time,\n          limit=100000):\n        results.append(record)\n    return (f.collection, results)", "response": "Return unclaimed hunt result notifications for a given collection."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef ChunkTable(table,\n               max_chunk_size):\n  \"\"\"Chunks given table into multiple smaller ones.\n\n  Tables that osquery yields can be arbitrarily large. Because GRR's messages\n  cannot be arbitrarily large, it might happen that the table has to be split\n  into multiple smaller ones.\n\n  Note that that serialized response protos are going to be slightly bigger than\n  the specified limit. For regular values the additional payload should be\n  negligible.\n\n  Args:\n    table: A table to split into multiple smaller ones.\n    max_chunk_size: A maximum size of the returned table in bytes.\n\n  Yields:\n    Tables with the same query and headers as the input table and a subset of\n    rows.\n  \"\"\"\n\n  def ByteLength(string):\n    return len(string.encode(\"utf-8\"))\n\n  def Chunk():\n    result = rdf_osquery.OsqueryTable()\n    result.query = table.query\n    result.header = table.header\n    return result\n\n  chunk = Chunk()\n  chunk_size = 0\n\n  for row in table.rows:\n    row_size = sum(map(ByteLength, row.values))\n\n    if chunk_size + row_size > max_chunk_size:\n      yield chunk\n\n      chunk = Chunk()\n      chunk_size = 0\n\n    chunk.rows.append(row)\n    chunk_size += row_size\n\n  # We want to yield extra chunk in two cases:\n  # * there are some rows that did not cause the chunk to overflow so it has not\n  #   been yielded as part of the loop.\n  # * the initial table has no rows but we still need to yield some table even\n  #   if it is empty.\n  if chunk.rows or not table.rows:\n    yield chunk", "response": "Yields a chunk of osquery tables into multiple smaller ones."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef ParseTable(table):\n  precondition.AssertIterableType(table, dict)\n\n  result = rdf_osquery.OsqueryTable()\n  result.header = ParseHeader(table)\n  for row in table:\n    result.rows.append(ParseRow(result.header, row))\n  return result", "response": "Parses a table of osquery output."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef ParseHeader(table):\n  precondition.AssertIterableType(table, dict)\n\n  prototype = None  # type: List[Text]\n\n  for row in table:\n    columns = list(iterkeys(row))\n    if prototype is None:\n      prototype = columns\n    elif prototype != columns:\n      message = \"Expected columns '{expected}', got '{actual}' for table {json}\"\n      message = message.format(expected=prototype, actual=columns, json=table)\n      raise ValueError(message)\n\n  result = rdf_osquery.OsqueryHeader()\n  for name in prototype or []:\n    result.columns.append(rdf_osquery.OsqueryColumn(name=name))\n  return result", "response": "Parses header of osquery output.\n\n  Args:\n    table: A table in a \"parsed JSON\" representation.\n\n  Returns:\n    A parsed `rdf_osquery.OsqueryHeader` instance."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef ParseRow(header,\n             row):\n  \"\"\"Parses a single row of osquery output.\n\n  Args:\n    header: A parsed header describing the row format.\n    row: A row in a \"parsed JSON\" representation.\n\n  Returns:\n    A parsed `rdf_osquery.OsqueryRow` instance.\n  \"\"\"\n  precondition.AssertDictType(row, Text, Text)\n\n  result = rdf_osquery.OsqueryRow()\n  for column in header.columns:\n    result.values.append(row[column.name])\n  return result", "response": "Parses a single row of osquery output."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef Query(args):\n  query = args.query.encode(\"utf-8\")\n  timeout = args.timeout_millis / 1000  # `subprocess.run` uses seconds.\n  # TODO: pytype is not aware of the backport.\n  # pytype: disable=module-attr\n  try:\n    # We use `--S` to enforce shell execution. This is because on Windows there\n    # is only `osqueryd` and `osqueryi` is not available. However, by passing\n    # `--S` we can make `osqueryd` behave like `osqueryi`. Since this flag also\n    # works with `osqueryi`, by passing it we simply expand number of supported\n    # executable types.\n    command = [config.CONFIG[\"Osquery.path\"], \"--S\", \"--json\", query]\n    proc = subprocess.run(\n        command,\n        timeout=timeout,\n        check=True,\n        stdout=subprocess.PIPE,\n        stderr=subprocess.PIPE)\n  # TODO: Since we use a backported API, `SubprocessError` is hard\n  # to work with. Until support for Python 2 is dropped we re-raise with simpler\n  # exception type because we don't really care that much (the exception message\n  # should be detailed enough anyway).\n  except subprocess.TimeoutExpired as error:\n    raise TimeoutError(cause=error)\n  except subprocess.CalledProcessError as error:\n    raise Error(\"osquery invocation error\", cause=error)\n  # pytype: enable=module-attr\n\n  stdout = proc.stdout.decode(\"utf-8\")\n  stderr = proc.stderr.decode(\"utf-8\").strip()\n  return ProcOutput(stdout=stdout, stderr=stderr)", "response": "Calls osquery with given query and returns its output."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nsearch the text for our value.", "response": "def Search(self, text):\n    \"\"\"Search the text for our value.\"\"\"\n    if isinstance(text, rdfvalue.RDFString):\n      text = str(text)\n\n    return self._regex.search(text)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef SplitEmailsAndAppendEmailDomain(self, address_list):\n    result = []\n    # Process email addresses, and build up a list.\n    if isinstance(address_list, rdf_standard.DomainEmailAddress):\n      address_list = [str(address_list)]\n    elif isinstance(address_list, string_types):\n      address_list = [address for address in address_list.split(\",\") if address]\n    for address in address_list:\n      result.append(self.AddEmailDomain(address))\n    return result", "response": "Splits a string of comma - separated emails appending default domain."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nschedule all the SystemCronFlows found.", "response": "def ScheduleSystemCronFlows(names=None, token=None):\n  \"\"\"Schedule all the SystemCronFlows found.\"\"\"\n\n  if data_store.RelationalDBEnabled():\n    return cronjobs.ScheduleSystemCronJobs(names=names)\n\n  errors = []\n  for name in config.CONFIG[\"Cron.disabled_system_jobs\"]:\n    try:\n      cls = registry.AFF4FlowRegistry.FlowClassByName(name)\n    except ValueError:\n      errors.append(\"No such flow: %s.\" % name)\n      continue\n\n    if not issubclass(cls, SystemCronFlow):\n      errors.append(\"Disabled system cron job name doesn't correspond to \"\n                    \"a flow inherited from SystemCronFlow: %s\" % name)\n\n  if names is None:\n    names = iterkeys(registry.AFF4FlowRegistry.FLOW_REGISTRY)\n\n  for name in names:\n    cls = registry.AFF4FlowRegistry.FlowClassByName(name)\n\n    if not issubclass(cls, SystemCronFlow):\n      continue\n\n    cron_args = rdf_cronjobs.CreateCronJobFlowArgs(\n        periodicity=cls.frequency,\n        lifetime=cls.lifetime,\n        allow_overruns=cls.allow_overruns)\n    cron_args.flow_runner_args.flow_name = name\n\n    if cls.enabled:\n      enabled = name not in config.CONFIG[\"Cron.disabled_system_jobs\"]\n    else:\n      enabled = False\n\n    job_urn = CronManager.CRON_JOBS_PATH.Add(name)\n    with aff4.FACTORY.Create(\n        job_urn,\n        aff4_type=CronJob,\n        mode=\"rw\",\n        token=token,\n        force_new_version=False) as cron_job:\n\n      # If the cronjob was already present we don't want to overwrite the\n      # original start_time.\n      existing_cron_args = cron_job.Get(cron_job.Schema.CRON_ARGS)\n\n      if cron_args != existing_cron_args:\n        cron_job.Set(cron_job.Schema.CRON_ARGS(cron_args))\n\n      cron_job.Set(cron_job.Schema.DISABLED(not enabled))\n\n  if errors:\n    raise ValueError(\"Error(s) while parsing Cron.disabled_system_jobs: %s\" %\n                     errors)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncreating a new cron job object.", "response": "def CreateJob(self, cron_args=None, job_id=None, token=None, enabled=True):\n    \"\"\"Creates a cron job that runs given flow with a given frequency.\n\n    Args:\n      cron_args: A protobuf of type rdf_cronjobs.CreateCronJobArgs.\n      job_id: Use this job_id instead of an autogenerated unique name (used for\n        system cron jobs - we want them to have well-defined persistent name).\n      token: Security token used for data store access.\n      enabled: If False, the job object will be created, but will be disabled.\n\n    Returns:\n      Name of the cron job created.\n    \"\"\"\n    if not job_id:\n      uid = random.UInt16()\n      job_id = \"%s_%s\" % (cron_args.flow_name, uid)\n\n    flow_runner_args = rdf_flow_runner.FlowRunnerArgs(\n        flow_name=\"CreateAndRunGenericHuntFlow\")\n\n    flow_args = rdf_hunts.CreateGenericHuntFlowArgs()\n    flow_args.hunt_args.flow_args = cron_args.flow_args\n    flow_args.hunt_args.flow_runner_args.flow_name = cron_args.flow_name\n    flow_args.hunt_runner_args = cron_args.hunt_runner_args\n    flow_args.hunt_runner_args.hunt_name = \"GenericHunt\"\n\n    create_cron_args = rdf_cronjobs.CreateCronJobFlowArgs(\n        description=cron_args.description,\n        periodicity=cron_args.frequency,\n        flow_runner_args=flow_runner_args,\n        flow_args=flow_args,\n        allow_overruns=cron_args.allow_overruns,\n        lifetime=cron_args.lifetime)\n\n    cron_job_urn = self.CRON_JOBS_PATH.Add(job_id)\n    with aff4.FACTORY.Create(\n        cron_job_urn,\n        aff4_type=CronJob,\n        mode=\"rw\",\n        token=token,\n        force_new_version=False) as cron_job:\n\n      # If the cronjob was already present we don't want to overwrite the\n      # original start_time.\n      existing_cron_args = cron_job.Get(cron_job.Schema.CRON_ARGS)\n      if existing_cron_args and existing_cron_args.start_time:\n        create_cron_args.start_time = existing_cron_args.start_time\n\n      if create_cron_args != existing_cron_args:\n        cron_job.Set(cron_job.Schema.CRON_ARGS(create_cron_args))\n\n      cron_job.Set(cron_job.Schema.DISABLED(not enabled))\n\n    return job_id"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns a list of all currently running cron jobs.", "response": "def ListJobs(self, token=None):\n    \"\"\"Returns a list of all currently running cron jobs.\"\"\"\n    job_root = aff4.FACTORY.Open(self.CRON_JOBS_PATH, token=token)\n    return [urn.Basename() for urn in job_root.ListChildren()]"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef EnableJob(self, job_id, token=None):\n    job_urn = self.CRON_JOBS_PATH.Add(job_id)\n    cron_job = aff4.FACTORY.Open(\n        job_urn, mode=\"rw\", aff4_type=CronJob, token=token)\n    cron_job.Set(cron_job.Schema.DISABLED(0))\n    cron_job.Close()", "response": "Enable a cron job with the given URN."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef DeleteJob(self, job_id, token=None):\n    job_urn = self.CRON_JOBS_PATH.Add(job_id)\n    aff4.FACTORY.Delete(job_urn, token=token)", "response": "Deletes the given job."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nruns one or more cron jobs.", "response": "def RunOnce(self, token=None, force=False, names=None):\n    \"\"\"Tries to lock and run cron jobs.\n\n    Args:\n      token: security token\n      force: If True, force a run\n      names: List of job names to run.  If unset, run them all\n    \"\"\"\n    names = names or self.ListJobs(token=token)\n    urns = [self.CRON_JOBS_PATH.Add(name) for name in names]\n\n    for cron_job_urn in urns:\n      try:\n        with aff4.FACTORY.OpenWithLock(\n            cron_job_urn, blocking=False, token=token,\n            lease_time=600) as cron_job:\n          try:\n            logging.info(\"Running cron job: %s\", cron_job.urn)\n            cron_job.Run(force=force)\n          except Exception as e:  # pylint: disable=broad-except\n            logging.exception(\"Error processing cron job %s: %s\", cron_job.urn,\n                              e)\n            stats_collector_instance.Get().IncrementCounter(\n                \"cron_internal_error\")\n\n      except aff4.LockError:\n        pass"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef DeleteOldRuns(self, job, cutoff_timestamp=None, token=None):\n    if cutoff_timestamp is None:\n      raise ValueError(\"cutoff_timestamp can't be None\")\n\n    child_flows = list(job.ListChildren(age=cutoff_timestamp))\n    with queue_manager.QueueManager(token=token) as queuemanager:\n      queuemanager.MultiDestroyFlowStates(child_flows)\n\n    aff4.FACTORY.MultiDelete(child_flows, token=token)\n    return len(child_flows)", "response": "Deletes flows initiated by the job that are older than the specified timestamp."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nrun a working thread and returns immediately.", "response": "def RunAsync(self):\n    \"\"\"Runs a working thread and returns immediately.\"\"\"\n    self.running_thread = threading.Thread(\n        name=self.thread_name, target=self._RunLoop)\n    self.running_thread.daemon = True\n    self.running_thread.start()\n    return self.running_thread"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns True if there s a currently running iteration of this job.", "response": "def IsRunning(self):\n    \"\"\"Returns True if there's a currently running iteration of this job.\"\"\"\n    current_urn = self.Get(self.Schema.CURRENT_FLOW_URN)\n    if not current_urn:\n      return False\n\n    try:\n      current_flow = aff4.FACTORY.Open(\n          urn=current_urn, aff4_type=flow.GRRFlow, token=self.token, mode=\"r\")\n    except aff4.InstantiationError:\n      # This isn't a flow, something went really wrong, clear it out.\n      logging.error(\"Unable to open cron job run: %s\", current_urn)\n      self.DeleteAttribute(self.Schema.CURRENT_FLOW_URN)\n      self.Flush()\n      return False\n\n    return current_flow.GetRunner().IsRunning()"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn True if the current time is due to run based on the specified frequency.", "response": "def DueToRun(self):\n    \"\"\"Called periodically by the cron daemon, if True Run() will be called.\n\n    Returns:\n        True if it is time to run based on the specified frequency.\n    \"\"\"\n    if self.Get(self.Schema.DISABLED):\n      return False\n\n    cron_args = self.Get(self.Schema.CRON_ARGS)\n    last_run_time = self.Get(self.Schema.LAST_RUN_TIME)\n    now = rdfvalue.RDFDatetime.Now()\n\n    # Its time to run.\n    if (last_run_time is None or\n        now > cron_args.periodicity.Expiry(last_run_time)):\n\n      # Not due to start yet.\n      if now < cron_args.start_time:\n        return False\n\n      # Do we allow overruns?\n      if cron_args.allow_overruns:\n        return True\n\n      # No currently executing job - lets go.\n      if self.Get(self.Schema.CURRENT_FLOW_URN) is None:\n        return True\n\n    return False"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef KillOldFlows(self):\n    if not self.IsRunning():\n      return False\n\n    start_time = self.Get(self.Schema.LAST_RUN_TIME)\n    lifetime = self.Get(self.Schema.CRON_ARGS).lifetime\n\n    elapsed = rdfvalue.RDFDatetime.Now() - start_time\n\n    if lifetime and elapsed > lifetime:\n      self.StopCurrentRun()\n      stats_collector_instance.Get().IncrementCounter(\n          \"cron_job_timeout\", fields=[self.urn.Basename()])\n      stats_collector_instance.Get().RecordEvent(\n          \"cron_job_latency\", elapsed.seconds, fields=[self.urn.Basename()])\n      return True\n\n    return False", "response": "Disable old flows if the flow is still running."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nruns the cron job.", "response": "def Run(self, force=False):\n    \"\"\"Do the actual work of the Cron.\n\n    Will first check if DueToRun is True.\n\n    CronJob object must be locked (i.e. opened via OpenWithLock) for Run() to be\n    called.\n\n    Args:\n      force: If True, the job will run no matter what (i.e. even if DueToRun()\n        returns False).\n\n    Raises:\n      LockError: if the object is not locked.\n    \"\"\"\n    if not self.locked:\n      raise aff4.LockError(\"CronJob must be locked for Run() to be called.\")\n\n    self.KillOldFlows()\n\n    # If currently running flow has finished, update our state.\n    current_flow_urn = self.Get(self.Schema.CURRENT_FLOW_URN)\n    if current_flow_urn:\n      current_flow = aff4.FACTORY.Open(current_flow_urn, token=self.token)\n      runner = current_flow.GetRunner()\n      if not runner.IsRunning():\n        if runner.context.state == rdf_flow_runner.FlowContext.State.ERROR:\n          self.Set(\n              self.Schema.LAST_RUN_STATUS,\n              rdf_cronjobs.CronJobRunStatus(\n                  status=rdf_cronjobs.CronJobRunStatus.Status.ERROR))\n          stats_collector_instance.Get().IncrementCounter(\n              \"cron_job_failure\", fields=[self.urn.Basename()])\n        else:\n          self.Set(\n              self.Schema.LAST_RUN_STATUS,\n              rdf_cronjobs.CronJobRunStatus(\n                  status=rdf_cronjobs.CronJobRunStatus.Status.OK))\n\n          start_time = self.Get(self.Schema.LAST_RUN_TIME)\n          elapsed = time.time() - start_time.AsSecondsSinceEpoch()\n          stats_collector_instance.Get().RecordEvent(\n              \"cron_job_latency\", elapsed, fields=[self.urn.Basename()])\n\n        self.DeleteAttribute(self.Schema.CURRENT_FLOW_URN)\n        self.Flush()\n\n    if not force and not self.DueToRun():\n      return\n\n    # Make sure the flow is created with cron job as a parent folder.\n    cron_args = self.Get(self.Schema.CRON_ARGS)\n    cron_args.flow_runner_args.base_session_id = self.urn\n\n    flow_urn = flow.StartAFF4Flow(\n        runner_args=cron_args.flow_runner_args,\n        args=cron_args.flow_args,\n        token=self.token,\n        sync=False)\n\n    self.Set(self.Schema.CURRENT_FLOW_URN, flow_urn)\n    self.Set(self.Schema.LAST_RUN_TIME, rdfvalue.RDFDatetime.Now())\n    self.Flush()"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nlaunch a fingerprint client action.", "response": "def FingerprintFile(self, pathspec, max_filesize=None, request_data=None):\n    \"\"\"Launch a fingerprint client action.\"\"\"\n    request = rdf_client_action.FingerprintRequest(pathspec=pathspec)\n    if max_filesize is not None:\n      request.max_filesize = max_filesize\n\n    # Generic hash.\n    request.AddRequest(\n        fp_type=rdf_client_action.FingerprintTuple.Type.FPT_GENERIC,\n        hashers=[\n            rdf_client_action.FingerprintTuple.HashType.MD5,\n            rdf_client_action.FingerprintTuple.HashType.SHA1,\n            rdf_client_action.FingerprintTuple.HashType.SHA256\n        ])\n\n    # Authenticode hash.\n    request.AddRequest(\n        fp_type=rdf_client_action.FingerprintTuple.Type.FPT_PE_COFF,\n        hashers=[\n            rdf_client_action.FingerprintTuple.HashType.MD5,\n            rdf_client_action.FingerprintTuple.HashType.SHA1,\n            rdf_client_action.FingerprintTuple.HashType.SHA256\n        ])\n\n    self.CallClient(\n        self.fingerprint_file_mixin_client_action,\n        request,\n        next_state=\"ProcessFingerprint\",\n        request_data=request_data)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef ProcessFingerprint(self, responses):\n    if not responses.success:\n      # Its better to raise rather than merely logging since it will make it to\n      # the flow's protobuf and users can inspect the reason this flow failed.\n      raise flow.FlowError(\"Could not fingerprint file: %s\" % responses.status)\n\n    response = responses.First()\n    if response.pathspec.path:\n      pathspec = response.pathspec\n    else:\n      pathspec = self.args.pathspec\n\n    self.state.urn = pathspec.AFF4Path(self.client_urn)\n\n    hash_obj = response.hash\n\n    if data_store.AFF4Enabled():\n      with aff4.FACTORY.Create(\n          self.state.urn, aff4_grr.VFSFile, mode=\"w\", token=self.token) as fd:\n        fd.Set(fd.Schema.HASH, hash_obj)\n\n    if data_store.RelationalDBEnabled():\n      path_info = rdf_objects.PathInfo.FromPathSpec(pathspec)\n      path_info.hash_entry = response.hash\n\n      data_store.REL_DB.WritePathInfos(self.client_id, [path_info])\n\n    self.ReceiveFileFingerprint(\n        self.state.urn, hash_obj, request_data=responses.request_data)", "response": "Store the fingerprint response."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nclean out compiled protos.", "response": "def Clean():\n  \"\"\"Clean out compiled protos.\"\"\"\n  # Find all the compiled proto files and unlink them.\n  for (root, _, files) in os.walk(ROOT):\n    for filename in files:\n      full_filename = os.path.join(root, filename)\n      if full_filename.endswith(\"_pb2.py\") or full_filename.endswith(\n          \"_pb2.pyc\"):\n        os.unlink(full_filename)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nmake sure our protos have been compiled to python libraries.", "response": "def MakeProto():\n  \"\"\"Make sure our protos have been compiled to python libraries.\"\"\"\n  # Start running from one directory above the grr directory which is found by\n  # this scripts's location as __file__.\n  cwd = os.path.dirname(os.path.abspath(__file__))\n\n  # Find all the .proto files.\n  protos_to_compile = []\n  for (root, _, files) in os.walk(cwd):\n    for filename in files:\n      full_filename = os.path.join(root, filename)\n      if full_filename.endswith(\".proto\"):\n        proto_stat = os.stat(full_filename)\n\n        pb2_path = full_filename.rsplit(\".\", 1)[0] + \"_pb2.py\"\n        try:\n          pb2_stat = os.stat(pb2_path)\n          if pb2_stat.st_mtime >= proto_stat.st_mtime:\n            continue\n\n        except (OSError, IOError):\n          pass\n\n        protos_to_compile.append(full_filename)\n\n  if protos_to_compile:\n    # Find the protoc compiler.\n    protoc = os.environ.get(\"PROTOC\", \"protoc\")\n    try:\n      output = subprocess.check_output([protoc, \"--version\"])\n    except (IOError, OSError):\n      raise RuntimeError(\"Unable to launch %s protoc compiler. Please \"\n                         \"set the PROTOC environment variable.\", protoc)\n\n    if b\"3.6.1\" not in output:\n      raise RuntimeError(\"Incompatible protoc compiler detected. \"\n                         \"We need 3.6.1 not %s\" % output)\n\n    for proto in protos_to_compile:\n      command = [\n          protoc,\n          # Write the python files next to the .proto files.\n          \"--python_out\",\n          ROOT,\n          \"--proto_path=%s\" % ROOT,\n          proto\n      ]\n      print(\n          \"Compiling %s with (cwd: %s): %s\" % (proto, ROOT, \" \".join(command)))\n      # The protoc compiler is too dumb to deal with full paths - it expects a\n      # relative path from the current working directory.\n      subprocess.check_call(command, cwd=ROOT)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreset the lexer to process a new data feed.", "response": "def Reset(self):\n    \"\"\"Reset the lexer to process a new data feed.\"\"\"\n    # The first state\n    self.state = \"INITIAL\"\n    self.state_stack = []\n\n    # The buffer we are parsing now\n    self.buffer = \"\"\n    self.error = 0\n    self.verbose = 0\n\n    # The index into the buffer where we are currently pointing\n    self.processed = 0\n    self.processed_buffer = \"\""}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef NextToken(self):\n    # Nothing in the input stream - no token can match.\n    if not self.buffer:\n      return\n\n    current_state = self.state\n    for token in self._tokens:\n      # Does the rule apply to us?\n      if token.state_regex and not token.state_regex.match(current_state):\n        continue\n\n      if self.verbose:\n        logging.debug(\"%s: Trying to match %r with %r\", self.state,\n                      self.buffer[:10], token.re_str)\n\n      # Try to match the rule\n      m = token.regex.match(self.buffer)\n      if not m:\n        continue\n\n      if self.verbose:\n        logging.debug(\"%s matched %s\", token.re_str, m.group(0))\n\n      # A token matched the empty string. We can not consume the token from the\n      # input stream.\n      if m.end() == 0:\n        raise RuntimeError(\"Lexer bug! Token can not match the empty string.\")\n\n      # The match consumes the data off the buffer (the handler can put it back\n      # if it likes)\n      self.processed_buffer += self.buffer[:m.end()]\n      self.buffer = self.buffer[m.end():]\n      self.processed += m.end()\n\n      next_state = token.next_state\n      for action in token.actions:\n        if self.verbose:\n          logging.debug(\"Calling %s with %s\", action, m.group(0))\n\n        # Is there a callback to handle this action?\n        cb = getattr(self, action, self.Default)\n\n        # Allow a callback to skip other callbacks.\n        try:\n          possible_next_state = cb(string=m.group(0), match=m)\n          if possible_next_state == \"CONTINUE\":\n            continue\n          # Override the state from the Token\n          elif possible_next_state:\n            next_state = possible_next_state\n        except ParseError as e:\n          self.Error(e)\n\n      # Update the next state\n      if next_state:\n        self.state = next_state\n\n      return token\n\n    # Check that we are making progress - if we are too full, we assume we are\n    # stuck.\n    self.Error(\"Lexer stuck at state %s\" % (self.state))\n    self.processed_buffer += self.buffer[:1]\n    self.buffer = self.buffer[1:]\n    return \"Error\"", "response": "Fetch the next token by trying to match any of the regexes in order."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\npushes the current state on the state stack.", "response": "def PushState(self, **_):\n    \"\"\"Push the current state on the state stack.\"\"\"\n    if self.verbose:\n      logging.debug(\"Storing state %r\", self.state)\n    self.state_stack.append(self.state)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\npopping the previous state from the stack.", "response": "def PopState(self, **_):\n    \"\"\"Pop the previous state from the stack.\"\"\"\n    try:\n      self.state = self.state_stack.pop()\n      if self.verbose:\n        logging.debug(\"Returned state to %s\", self.state)\n\n      return self.state\n    except IndexError:\n      self.Error(\"Tried to pop the state but failed - possible recursion error\")"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\npushing the match back on the stream.", "response": "def PushBack(self, string=\"\", **_):\n    \"\"\"Push the match back on the stream.\"\"\"\n    precondition.AssertType(string, Text)\n    self.buffer = string + self.buffer\n    self.processed_buffer = self.processed_buffer[:-len(string)]"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nadds a new argument to the current expression.", "response": "def AddArg(self, arg):\n    \"\"\"Adds a new arg to this expression.\n\n    Args:\n       arg: The argument to add (string).\n\n    Returns:\n      True if this arg is the last arg, False otherwise.\n\n    Raises:\n      ParseError: If there are too many args.\n    \"\"\"\n    self.args.append(arg)\n    if len(self.args) > self.number_of_args:\n      raise ParseError(\"Too many args for this expression.\")\n\n    elif len(self.args) == self.number_of_args:\n      return True\n\n    return False"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef StringEscape(self, string, match, **_):\n    precondition.AssertType(string, Text)\n    if match.group(1) in \"'\\\"rnbt\":\n      self.string += compatibility.UnescapeString(string)\n    else:\n      self.string += string", "response": "Escape backslashes found inside a string quote."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ninsert an argument into the current expression.", "response": "def InsertArg(self, string=\"\", **_):\n    \"\"\"Insert an arg to the current expression.\"\"\"\n    if self.verbose:\n      logging.debug(\"Storing Argument %s\", utils.SmartUnicode(string))\n\n    # This expression is complete\n    if self.current_expression.AddArg(string):\n      self.stack.append(self.current_expression)\n      self.current_expression = self.expression_cls()\n      return self.PopState()"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncreating a new hunt.", "response": "def CreateHunt(flow_name=None,\n               flow_args=None,\n               hunt_runner_args=None,\n               context=None):\n  \"\"\"Creates a new hunt.\n\n  Args:\n    flow_name: String with a name of a flow that will run on all the clients\n        in the hunt.\n    flow_args: Flow arguments to be used. A proto, that depends on a flow.\n    hunt_runner_args: flows_pb2.HuntRunnerArgs instance. Used to specify\n        description, client_rule_set, output_plugins and other useful\n        hunt attributes.\n    context: API context.\n\n  Raises:\n    ValueError: if flow_name is empty.\n\n  Returns:\n    Hunt object corresponding to the created hunt.\n  \"\"\"\n  if not flow_name:\n    raise ValueError(\"flow_name can't be empty\")\n\n  request = hunt_pb2.ApiCreateHuntArgs(flow_name=flow_name)\n  if flow_args:\n    request.flow_args.value = flow_args.SerializeToString()\n    request.flow_args.type_url = utils.GetTypeUrl(flow_args)\n\n  if hunt_runner_args:\n    request.hunt_runner_args.CopyFrom(hunt_runner_args)\n\n  data = context.SendRequest(\"CreateHunt\", request)\n  return Hunt(data=data, context=context)"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nlists all GRR hunts.", "response": "def ListHunts(context=None):\n  \"\"\"List all GRR hunts.\"\"\"\n\n  items = context.SendIteratorRequest(\"ListHunts\", hunt_pb2.ApiListHuntsArgs())\n  return utils.MapItemsIterator(lambda data: Hunt(data=data, context=context),\n                                items)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef ListHuntApprovals(context=None):\n  items = context.SendIteratorRequest(\"ListHuntApprovals\",\n                                      hunt_pb2.ApiListHuntApprovalsArgs())\n\n  def MapHuntApproval(data):\n    return HuntApproval(data=data, username=context.username, context=context)\n\n  return utils.MapItemsIterator(MapHuntApproval, items)", "response": "List all hunt approvals belonging to requesting user."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nfetch and return a proper HuntApproval object.", "response": "def Get(self):\n    \"\"\"Fetch and return a proper HuntApproval object.\"\"\"\n\n    args = user_pb2.ApiGetHuntApprovalArgs(\n        hunt_id=self.hunt_id,\n        approval_id=self.approval_id,\n        username=self.username)\n    result = self._context.SendRequest(\"GetHuntApproval\", args)\n    return HuntApproval(\n        data=result, username=self._context.username, context=self._context)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef WaitUntilValid(self, timeout=None):\n\n    return utils.Poll(\n        generator=self.Get,\n        condition=lambda f: f.data.is_valid,\n        timeout=timeout)", "response": "Wait until the approval is valid."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning a reference to an approval.", "response": "def Approval(self, username, approval_id):\n    \"\"\"Returns a reference to an approval.\"\"\"\n\n    return HuntApprovalRef(\n        hunt_id=self.hunt_id,\n        username=username,\n        approval_id=approval_id,\n        context=self._context)"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ncreates a new approval for the current user to access this hunt.", "response": "def CreateApproval(self,\n                     reason=None,\n                     notified_users=None,\n                     email_cc_addresses=None):\n    \"\"\"Create a new approval for the current user to access this hunt.\"\"\"\n\n    if not reason:\n      raise ValueError(\"reason can't be empty\")\n\n    if not notified_users:\n      raise ValueError(\"notified_users list can't be empty.\")\n\n    approval = user_pb2.ApiHuntApproval(\n        reason=reason,\n        notified_users=notified_users,\n        email_cc_addresses=email_cc_addresses or [])\n    args = user_pb2.ApiCreateHuntApprovalArgs(\n        hunt_id=self.hunt_id, approval=approval)\n\n    data = self._context.SendRequest(\"CreateHuntApproval\", args)\n    return HuntApproval(\n        data=data, username=self._context.username, context=self._context)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef Modify(self, client_limit=None, client_rate=None, duration=None):\n    args = hunt_pb2.ApiModifyHuntArgs(hunt_id=self.hunt_id)\n\n    if client_limit is not None:\n      args.client_limit = client_limit\n\n    if client_rate is not None:\n      args.client_rate = client_rate\n\n    if duration is not None:\n      args.duration = duration\n\n    data = self._context.SendRequest(\"ModifyHunt\", args)\n    return Hunt(data=data, context=self._context)", "response": "Modifies a number of hunt arguments."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef Get(self):\n\n    args = hunt_pb2.ApiGetHuntArgs(hunt_id=self.hunt_id)\n    data = self._context.SendRequest(\"GetHunt\", args)\n    return Hunt(data=data, context=self._context)", "response": "Fetch hunt s data and return proper Hunt object."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _MakeConnection(self, database=\"\"):\n    first_attempt_time = time.time()\n    wait_time = config.CONFIG[\"Mysql.max_connect_wait\"]\n    while wait_time == 0 or time.time() - first_attempt_time < wait_time:\n      try:\n        connection_args = dict(\n            user=config.CONFIG[\"Mysql.database_username\"],\n            db=database,\n            charset=\"utf8\",\n            passwd=config.CONFIG[\"Mysql.database_password\"],\n            autocommit=True,\n            cursorclass=cursors.DictCursor,\n            host=config.CONFIG[\"Mysql.host\"],\n            port=config.CONFIG[\"Mysql.port\"])\n\n        key_path = config.CONFIG[\"Mysql.client_key_path\"]\n        if key_path:\n          cert_path = config.CONFIG[\"Mysql.client_cert_path\"]\n          ca_cert_path = config.CONFIG[\"Mysql.ca_cert_path\"]\n          logging.debug(\"Client key file configured, trying to use SSL.\")\n\n          connection_args[\"ssl\"] = {\n              \"key\": key_path,\n              \"cert\": cert_path,\n              \"ca\": ca_cert_path,\n          }\n\n        return MySQLdb.connect(**connection_args)\n\n      except MySQLdb.OperationalError as e:\n        # This is a fatal error, we just raise the top level exception here.\n        if \"Access denied\" in str(e):\n          raise Error(str(e))\n\n        if \"Can't connect\" in str(e):\n          logging.warning(\"Datastore connection retrying after failed with %s.\",\n                          str(e))\n          time.sleep(.5)\n          continue\n        raise\n\n    raise IOError(\"Unable to connect to Mysql database.\")", "response": "Make a connection to the MySQL server."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef DropConnection(self, connection):\n    try:\n      connection.cursor.close()\n    except MySQLdb.Error:\n      pass\n\n    try:\n      connection.dbh.close()\n    except MySQLdb.Error:\n      pass", "response": "Attempt to cleanly drop the connection."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ndrops all existing tables.", "response": "def DropTables(self):\n    \"\"\"Drop all existing tables.\"\"\"\n\n    rows, _ = self.ExecuteQuery(\n        \"SELECT table_name FROM information_schema.tables \"\n        \"WHERE table_schema='%s'\" % self.database_name)\n    for row in rows:\n      self.ExecuteQuery(\"DROP TABLE `%s`\" % row[\"table_name\"])"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef DeleteAttributes(self,\n                       subject,\n                       attributes,\n                       start=None,\n                       end=None,\n                       sync=True):\n    \"\"\"Remove some attributes from a subject.\"\"\"\n    _ = sync  # Unused\n    if not attributes:\n      return\n\n    if isinstance(attributes, string_types):\n      raise ValueError(\n          \"String passed to DeleteAttributes (non string iterable expected).\")\n\n    for attribute in attributes:\n      timestamp = self._MakeTimestamp(start, end)\n      attribute = utils.SmartUnicode(attribute)\n      queries = self._BuildDelete(subject, attribute, timestamp)\n      self._ExecuteQueries(queries)", "response": "Removes some attributes from a subject."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef ResolveMulti(self, subject, attributes, timestamp=None, limit=None):\n    for attribute in attributes:\n      query, args = self._BuildQuery(subject, attribute, timestamp, limit)\n      result, _ = self.ExecuteQuery(query, args)\n\n      for row in result:\n        value = self._Decode(attribute, row[\"value\"])\n\n        yield (attribute, value, row[\"timestamp\"])\n\n      if limit:\n        limit -= len(result)\n\n      if limit is not None and limit <= 0:\n        break", "response": "Resolves multiple attributes at once for one subject."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef MultiResolvePrefix(self,\n                         subjects,\n                         attribute_prefix,\n                         timestamp=None,\n                         limit=None):\n    \"\"\"Result multiple subjects using one or more attribute regexps.\"\"\"\n    result = {}\n\n    for subject in subjects:\n      values = self.ResolvePrefix(\n          subject, attribute_prefix, timestamp=timestamp, limit=limit)\n\n      if values:\n        result[subject] = values\n        if limit:\n          limit -= len(values)\n\n      if limit is not None and limit <= 0:\n        break\n\n    return iteritems(result)", "response": "Resolve multiple subjects using one or more attribute regexps."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef ResolvePrefix(self, subject, attribute_prefix, timestamp=None,\n                    limit=None):\n    \"\"\"ResolvePrefix.\"\"\"\n    if isinstance(attribute_prefix, string_types):\n      attribute_prefix = [attribute_prefix]\n\n    results = []\n\n    for prefix in attribute_prefix:\n      query, args = self._BuildQuery(\n          subject, prefix, timestamp, limit, is_prefix=True)\n      rows, _ = self.ExecuteQuery(query, args)\n      for row in sorted(rows, key=lambda x: x[\"attribute\"]):\n        attribute = row[\"attribute\"]\n        value = self._Decode(attribute, row[\"value\"])\n        results.append((attribute, value, row[\"timestamp\"]))\n\n    return results", "response": "Resolves the given attribute_prefix with the given subject."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nset multiple attributes values for this subject in one operation.", "response": "def MultiSet(self,\n               subject,\n               values,\n               timestamp=None,\n               replace=True,\n               sync=True,\n               to_delete=None):\n    \"\"\"Set multiple attributes' values for this subject in one operation.\"\"\"\n    to_delete = set(to_delete or [])\n\n    # Prepare a bulk insert operation.\n    subject = utils.SmartUnicode(subject)\n    to_insert = []\n    to_replace = []\n    transaction = []\n\n    # Build a document for each unique timestamp.\n    for attribute, sequence in iteritems(values):\n      for value in sequence:\n        if isinstance(value, tuple):\n          value, entry_timestamp = value\n        else:\n          entry_timestamp = timestamp\n\n        if entry_timestamp is None:\n          entry_timestamp = timestamp\n\n        if entry_timestamp is not None:\n          entry_timestamp = int(entry_timestamp)\n        else:\n          entry_timestamp = time.time() * 1e6\n\n        attribute = utils.SmartUnicode(attribute)\n        data = self._Encode(value)\n\n        # Replacing means to delete all versions of the attribute first.\n        if replace or attribute in to_delete:\n          existing = self._CountExistingRows(subject, attribute)\n          if existing:\n            to_replace.append([subject, attribute, data, entry_timestamp])\n          else:\n            to_insert.append([subject, attribute, data, entry_timestamp])\n          if attribute in to_delete:\n            to_delete.remove(attribute)\n\n        else:\n          to_insert.append([subject, attribute, data, entry_timestamp])\n\n    if to_delete:\n      self.DeleteAttributes(subject, to_delete)\n\n    if sync:\n      if to_replace:\n        transaction.extend(self._BuildReplaces(to_replace))\n      if to_insert:\n        transaction.extend(self._BuildInserts(to_insert))\n      if transaction:\n        self._ExecuteTransaction(transaction)\n    else:\n      if to_replace:\n        with self.buffer_lock:\n          self.to_replace.extend(to_replace)\n      if to_insert:\n        with self.buffer_lock:\n          self.to_insert.extend(to_insert)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef ExecuteQuery(self, query, args=None):\n\n    def Action(connection):\n      connection.cursor.execute(query, args)\n      rowcount = connection.cursor.rowcount\n      results = connection.cursor.fetchall()\n      return results, rowcount\n\n    return self._RetryWrapper(Action)", "response": "Get connection from pool and execute query."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _ExecuteTransaction(self, transaction):\n\n    def Action(connection):\n      connection.cursor.execute(\"START TRANSACTION\")\n      for query in transaction:\n        connection.cursor.execute(query[\"query\"], query[\"args\"])\n      connection.cursor.execute(\"COMMIT\")\n      return connection.cursor.fetchall()\n\n    return self._RetryWrapper(Action)", "response": "Get connection from pool and execute transaction."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nbuilds a mapping between column names and types.", "response": "def _CalculateAttributeStorageTypes(self):\n    \"\"\"Build a mapping between column names and types.\"\"\"\n    self.attribute_types = {}\n\n    for attribute in itervalues(aff4.Attribute.PREDICATES):\n      self.attribute_types[attribute.predicate] = (\n          attribute.attribute_type.data_store_type)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _Encode(self, value):\n    try:\n      return value.SerializeToString().encode(\"hex\")\n    except AttributeError:\n      if isinstance(value, (int, long)):\n        return str(value).encode(\"hex\")\n      else:\n        # Types \"string\" and \"bytes\" are stored as strings here.\n        return utils.SmartStr(value).encode(\"hex\")", "response": "Encode the value for the attribute."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _BuildQuery(self,\n                  subject,\n                  attribute=None,\n                  timestamp=None,\n                  limit=None,\n                  is_prefix=False):\n    \"\"\"Build the SELECT query to be executed.\"\"\"\n    args = []\n    subject = utils.SmartUnicode(subject)\n    criteria = \"WHERE aff4.subject_hash=unhex(md5(%s))\"\n    args.append(subject)\n    sorting = \"\"\n    tables = \"FROM aff4\"\n\n    # Set fields, tables, and criteria and append args\n    if attribute is not None:\n      if is_prefix:\n        tables += \" JOIN attributes ON aff4.attribute_hash=attributes.hash\"\n        prefix = attribute + \"%\"\n        criteria += \" AND attributes.attribute like %s\"\n        args.append(prefix)\n      else:\n        criteria += \" AND aff4.attribute_hash=unhex(md5(%s))\"\n        args.append(attribute)\n\n    # Limit to time range if specified\n    if isinstance(timestamp, (tuple, list)):\n      criteria += \" AND aff4.timestamp >= %s AND aff4.timestamp <= %s\"\n      args.append(int(timestamp[0]))\n      args.append(int(timestamp[1]))\n\n    fields = \"aff4.value, aff4.timestamp\"\n    if is_prefix:\n      fields += \", attributes.attribute\"\n\n    # Modify fields and sorting for timestamps.\n    if timestamp is None or timestamp == self.NEWEST_TIMESTAMP:\n      tables += (\" JOIN (SELECT attribute_hash, MAX(timestamp) timestamp \"\n                 \"%s %s GROUP BY attribute_hash) maxtime ON \"\n                 \"aff4.attribute_hash=maxtime.attribute_hash AND \"\n                 \"aff4.timestamp=maxtime.timestamp\") % (tables, criteria)\n      criteria = \"WHERE aff4.subject_hash=unhex(md5(%s))\"\n      args.append(subject)\n    else:\n      # Always order results.\n      sorting = \"ORDER BY aff4.timestamp DESC\"\n    # Add limit if set.\n    if limit:\n      sorting += \" LIMIT %s\" % int(limit)\n\n    query = \" \".join([\"SELECT\", fields, tables, criteria, sorting])\n\n    return (query, args)", "response": "Builds the SELECT query to be executed."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nbuild the DELETE query to be executed.", "response": "def _BuildDelete(self, subject, attribute=None, timestamp=None):\n    \"\"\"Build the DELETE query to be executed.\"\"\"\n    subjects_q = {\n        \"query\": \"DELETE subjects FROM subjects WHERE hash=unhex(md5(%s))\",\n        \"args\": [subject]\n    }\n\n    aff4_q = {\n        \"query\": \"DELETE aff4 FROM aff4 WHERE subject_hash=unhex(md5(%s))\",\n        \"args\": [subject]\n    }\n\n    locks_q = {\n        \"query\": \"DELETE locks FROM locks WHERE subject_hash=unhex(md5(%s))\",\n        \"args\": [subject]\n    }\n\n    if attribute:\n      aff4_q[\"query\"] += \" AND attribute_hash=unhex(md5(%s))\"\n      aff4_q[\"args\"].append(attribute)\n\n      if isinstance(timestamp, (tuple, list)):\n        aff4_q[\"query\"] += \" AND aff4.timestamp >= %s AND aff4.timestamp <= %s\"\n        aff4_q[\"args\"].append(int(timestamp[0]))\n        aff4_q[\"args\"].append(int(timestamp[1]))\n\n      attributes_q = {\n          \"query\": \"DELETE attributes FROM attributes LEFT JOIN aff4 ON \"\n                   \"aff4.attribute_hash=attributes.hash \"\n                   \"WHERE attributes.hash=unhex(md5(%s)) \"\n                   \"AND aff4.attribute_hash IS NULL\",\n          \"args\": [attribute]\n      }\n      # If only attribute is being deleted we will not check to clean up\n      # subject and lock tables.\n      return [aff4_q, attributes_q]\n    # If a subject is being deleted we clean up the locks and subjects table\n    # but assume it has attributes that are common to other subjects\n\n    return [aff4_q, locks_q, subjects_q]"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncreating a timestamp using a start and end time.", "response": "def _MakeTimestamp(self, start=None, end=None):\n    \"\"\"Create a timestamp using a start and end time.\n\n    Args:\n      start: Start timestamp.\n      end: End timestamp.\n\n    Returns:\n      A tuple (start, end) of converted timestamps or None for all time.\n    \"\"\"\n    mysql_unsigned_bigint_max = 18446744073709551615\n    ts_start = int(start or 0)\n    if end is None:\n      ts_end = mysql_unsigned_bigint_max\n    else:\n      ts_end = int(end)\n    if ts_start == 0 and ts_end == mysql_unsigned_bigint_max:\n      return None\n    else:\n      return (ts_start, ts_end)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nremoving the lock. Note that this only resets the lock if we actually hold it since lock_expiration == self.expires and lock_owner = self.lock_token.", "response": "def Release(self):\n    \"\"\"Remove the lock.\n\n    Note that this only resets the lock if we actually hold it since\n    lock_expiration == self.expires and lock_owner = self.lock_token.\n    \"\"\"\n    if self.locked:\n      query = (\"UPDATE locks SET lock_expiration=0, lock_owner=0 \"\n               \"WHERE lock_expiration=%s \"\n               \"AND lock_owner=%s \"\n               \"AND subject_hash=unhex(md5(%s))\")\n      args = [self.expires, self.lock_token, self.subject]\n      self.store.ExecuteQuery(query, args)\n      self.locked = False"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nfind objects associated with keywords.", "response": "def Lookup(self,\n             keywords,\n             start_time=FIRST_TIMESTAMP,\n             end_time=LAST_TIMESTAMP,\n             last_seen_map=None):\n    \"\"\"Finds objects associated with keywords.\n\n    Find the names related to all keywords.\n\n    Args:\n      keywords: A collection of keywords that we are interested in.\n      start_time: Only considers keywords added at or after this point in time.\n      end_time: Only considers keywords at or before this point in time.\n      last_seen_map: If present, is treated as a dict and populated to map pairs\n        (keyword, name) to the timestamp of the latest connection found.\n    Returns:\n      A set of potentially relevant names.\n\n    \"\"\"\n    posting_lists = self.ReadPostingLists(\n        keywords,\n        start_time=start_time,\n        end_time=end_time,\n        last_seen_map=last_seen_map)\n\n    results = list(itervalues(posting_lists))\n    relevant_set = results[0]\n\n    for hits in results:\n      relevant_set &= hits\n\n      if not relevant_set:\n        return relevant_set\n\n    return relevant_set"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef ReadPostingLists(self,\n                       keywords,\n                       start_time=FIRST_TIMESTAMP,\n                       end_time=LAST_TIMESTAMP,\n                       last_seen_map=None):\n    \"\"\"Finds all objects associated with any of the keywords.\n\n    Args:\n      keywords: A collection of keywords that we are interested in.\n      start_time: Only considers keywords added at or after this point in time.\n      end_time: Only considers keywords at or before this point in time.\n      last_seen_map: If present, is treated as a dict and populated to map pairs\n        (keyword, name) to the timestamp of the latest connection found.\n    Returns:\n      A dict mapping each keyword to a set of relevant names.\n\n    \"\"\"\n    return data_store.DB.IndexReadPostingLists(\n        self.urn, keywords, start_time, end_time, last_seen_map=last_seen_map)", "response": "Reads all postings for the given keywords."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nassociates keywords with name.", "response": "def AddKeywordsForName(self, name, keywords):\n    \"\"\"Associates keywords with name.\n\n    Records that keywords are associated with name.\n\n    Args:\n      name: A name which should be associated with some keywords.\n      keywords: A collection of keywords to associate with name.\n    \"\"\"\n    data_store.DB.IndexAddKeywordsForName(self.urn, name, keywords)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nremoving keywords for a name.", "response": "def RemoveKeywordsForName(self, name, keywords):\n    \"\"\"Removes keywords for a name.\n\n    Args:\n      name: A name which should not be associated with some keywords anymore.\n      keywords: A collection of keywords.\n    \"\"\"\n    data_store.DB.IndexRemoveKeywordsForName(self.urn, name, keywords)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreading an old config file and imports keys and user accounts.", "response": "def ImportConfig(filename, config):\n  \"\"\"Reads an old config file and imports keys and user accounts.\"\"\"\n  sections_to_import = [\"PrivateKeys\"]\n  entries_to_import = [\n      \"Client.executable_signing_public_key\", \"CA.certificate\",\n      \"Frontend.certificate\"\n  ]\n  options_imported = 0\n  old_config = grr_config.CONFIG.MakeNewConfig()\n  old_config.Initialize(filename)\n\n  for entry in old_config.raw_data:\n    try:\n      section = entry.split(\".\")[0]\n      if section in sections_to_import or entry in entries_to_import:\n        config.Set(entry, old_config.Get(entry))\n        print(\"Imported %s.\" % entry)\n        options_imported += 1\n\n    except Exception as e:  # pylint: disable=broad-except\n      print(\"Exception during import of %s: %s\" % (entry, e))\n  return options_imported"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef RetryQuestion(question_text, output_re=\"\", default_val=None):\n  while True:\n    if default_val is not None:\n      new_text = \"%s [%s]: \" % (question_text, default_val)\n    else:\n      new_text = \"%s: \" % question_text\n    # pytype: disable=wrong-arg-count\n    output = builtins.input(new_text) or str(default_val)\n    # pytype: enable=wrong-arg-count\n    output = output.strip()\n    if not output_re or re.match(output_re, output):\n      break\n    else:\n      print(\"Invalid input, must match %s\" % output_re)\n  return output", "response": "Continually ask a question until the output_re is matched."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef ConfigureHostnames(config, external_hostname = None):\n  if not external_hostname:\n    try:\n      external_hostname = socket.gethostname()\n    except (OSError, IOError):\n      print(\"Sorry, we couldn't guess your hostname.\\n\")\n\n    external_hostname = RetryQuestion(\n        \"Please enter your hostname e.g. \"\n        \"grr.example.com\", \"^[\\\\.A-Za-z0-9-]+$\", external_hostname)\n\n  print(\"\"\"\\n\\n-=Server URL=-\nThe Server URL specifies the URL that the clients will connect to\ncommunicate with the server. For best results this should be publicly\naccessible. By default this will be port 8080 with the URL ending in /control.\n\"\"\")\n  frontend_url = RetryQuestion(\"Frontend URL\", \"^http://.*/$\",\n                               \"http://%s:8080/\" % external_hostname)\n  config.Set(\"Client.server_urls\", [frontend_url])\n\n  frontend_port = urlparse.urlparse(frontend_url).port or grr_config.CONFIG.Get(\n      \"Frontend.bind_port\")\n  config.Set(\"Frontend.bind_port\", frontend_port)\n\n  print(\"\"\"\\n\\n-=AdminUI URL=-:\nThe UI URL specifies where the Administrative Web Interface can be found.\n\"\"\")\n  ui_url = RetryQuestion(\"AdminUI URL\", \"^http[s]*://.*$\",\n                         \"http://%s:8000\" % external_hostname)\n  config.Set(\"AdminUI.url\", ui_url)\n  ui_port = urlparse.urlparse(ui_url).port or grr_config.CONFIG.Get(\n      \"AdminUI.port\")\n  config.Set(\"AdminUI.port\", ui_port)", "response": "This configures the hostnames stored in the config."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nchecking whether a MySQL connection is established to a MySQL server.", "response": "def CheckMySQLConnection(db_options):\n  \"\"\"Checks whether a connection can be established to MySQL.\n\n  Args:\n    db_options: A dict mapping GRR MySQL config options to their values.\n\n  Returns:\n    A boolean indicating whether a connection could be made to a MySQL server\n    instance with the given options.\n  \"\"\"\n  for tries_left in range(_MYSQL_MAX_RETRIES, -1, -1):\n    try:\n      connection_options = dict(\n          host=db_options[\"Mysql.host\"],\n          port=db_options[\"Mysql.port\"],\n          db=db_options[\"Mysql.database_name\"],\n          user=db_options[\"Mysql.database_username\"],\n          passwd=db_options[\"Mysql.database_password\"],\n          charset=\"utf8\")\n\n      ssl_enabled = \"Mysql.client_key_path\" in db_options\n      if ssl_enabled:\n        connection_options[\"ssl\"] = {\n            \"key\": db_options[\"Mysql.client_key_path\"],\n            \"cert\": db_options[\"Mysql.client_cert_path\"],\n            \"ca\": db_options[\"Mysql.ca_cert_path\"],\n        }\n\n      connection = MySQLdb.connect(**connection_options)\n\n      if ssl_enabled:\n        cursor = connection.cursor()\n        cursor.execute(\"SHOW VARIABLES LIKE 'have_ssl'\")\n        res = cursor.fetchone()\n        if res[0] == \"have_ssl\" and res[1] == \"YES\":\n          print(\"SSL enabled successfully.\")\n        else:\n          print(\"Unable to establish SSL connection to MySQL.\")\n          return False\n\n      return True\n    except MySQLdb.OperationalError as mysql_op_error:\n      if len(mysql_op_error.args) < 2:\n        # We expect the exception's arguments to be an error-code and\n        # an error message.\n        print(\"Unexpected exception type received from MySQL. %d attempts \"\n              \"left: %s\" % (tries_left, mysql_op_error))\n        time.sleep(_MYSQL_RETRY_WAIT_SECS)\n        continue\n      if mysql_op_error.args[0] == mysql_conn_errors.CONNECTION_ERROR:\n        print(\"Failed to connect to MySQL. Is it running? %d attempts left.\" %\n              tries_left)\n      elif mysql_op_error.args[0] == mysql_conn_errors.UNKNOWN_HOST:\n        print(\"Unknown-hostname error encountered while trying to connect to \"\n              \"MySQL.\")\n        return False  # No need for retry.\n      elif mysql_op_error.args[0] == general_mysql_errors.BAD_DB_ERROR:\n        # GRR db doesn't exist yet. That's expected if this is the initial\n        # setup.\n        return True\n      elif mysql_op_error.args[0] in (\n          general_mysql_errors.ACCESS_DENIED_ERROR,\n          general_mysql_errors.DBACCESS_DENIED_ERROR):\n        print(\"Permission error encountered while trying to connect to \"\n              \"MySQL: %s\" % mysql_op_error)\n        return False  # No need for retry.\n      else:\n        print(\"Unexpected operational error encountered while trying to \"\n              \"connect to MySQL. %d attempts left: %s\" %\n              (tries_left, mysql_op_error))\n    except MySQLdb.Error as mysql_error:\n      print(\"Unexpected error encountered while trying to connect to MySQL. \"\n            \"%d attempts left: %s\" % (tries_left, mysql_error))\n    time.sleep(_MYSQL_RETRY_WAIT_SECS)\n  return False"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef ConfigureMySQLDatastore(config):\n  print(\"GRR will use MySQL as its database backend. Enter connection details:\")\n  datastore_init_complete = False\n  db_options = {}\n  while not datastore_init_complete:\n    db_options[\"Datastore.implementation\"] = \"MySQLAdvancedDataStore\"\n    db_options[\"Mysql.host\"] = RetryQuestion(\"MySQL Host\", \"^[\\\\.A-Za-z0-9-]+$\",\n                                             config[\"Mysql.host\"])\n    db_options[\"Mysql.port\"] = int(\n        RetryQuestion(\"MySQL Port (0 for local socket)\", \"^[0-9]+$\",\n                      config[\"Mysql.port\"]))\n    db_options[\"Mysql.database_name\"] = RetryQuestion(\n        \"MySQL Database\", \"^[A-Za-z0-9-]+$\", config[\"Mysql.database_name\"])\n    db_options[\"Mysql.database_username\"] = RetryQuestion(\n        \"MySQL Username\", \"[A-Za-z0-9-@]+$\", config[\"Mysql.database_username\"])\n    # TODO(hanuszczak): Incorrect type specification for `getpass`.\n    # pytype: disable=wrong-arg-types\n    db_options[\"Mysql.database_password\"] = getpass.getpass(\n        prompt=\"Please enter password for database user %s: \" %\n        db_options[\"Mysql.database_username\"])\n    # pytype: enable=wrong-arg-types\n\n    use_ssl = RetryBoolQuestion(\"Configure SSL connections for MySQL?\", False)\n    if use_ssl:\n      db_options[\"Mysql.client_key_path\"] = RetryQuestion(\n          \"Path to the client private key file\",\n          default_val=config[\"Mysql.client_key_path\"])\n      db_options[\"Mysql.client_cert_path\"] = RetryQuestion(\n          \"Path to the client certificate file\",\n          default_val=config[\"Mysql.client_cert_path\"])\n      db_options[\"Mysql.ca_cert_path\"] = RetryQuestion(\n          \"Path to the CA certificate file\",\n          default_val=config[\"Mysql.ca_cert_path\"])\n\n    if CheckMySQLConnection(db_options):\n      print(\"Successfully connected to MySQL with the provided details.\")\n      datastore_init_complete = True\n    else:\n      print(\"Error: Could not connect to MySQL with the provided details.\")\n      should_retry = RetryBoolQuestion(\n          \"Re-enter MySQL details? Answering 'no' will abort config \"\n          \"initialization: \", True)\n      if should_retry:\n        db_options.clear()\n      else:\n        raise ConfigInitError()\n\n  for option, value in iteritems(db_options):\n    config.Set(option, value)", "response": "Configure a MySQL datastore."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nguide the user through configuration of the datastore.", "response": "def ConfigureDatastore(config):\n  \"\"\"Guides the user through configuration of the datastore.\"\"\"\n  print(\"\\n\\n-=GRR Datastore=-\\n\"\n        \"For GRR to work each GRR server has to be able to communicate with\\n\"\n        \"the datastore. To do this we need to configure a datastore.\\n\")\n\n  existing_datastore = grr_config.CONFIG.Get(\"Datastore.implementation\")\n\n  if not existing_datastore or existing_datastore == \"FakeDataStore\":\n    ConfigureMySQLDatastore(config)\n    return\n\n  print(\"Found existing settings:\\n  Datastore: %s\" % existing_datastore)\n  if existing_datastore == \"SqliteDataStore\":\n    set_up_mysql = RetryBoolQuestion(\n        \"The SQLite datastore is no longer supported. Would you like to\\n\"\n        \"set up a MySQL datastore? Answering 'no' will abort config \"\n        \"initialization.\", True)\n    if set_up_mysql:\n      print(\"\\nPlease note that no data will be migrated from SQLite to \"\n            \"MySQL.\\n\")\n      ConfigureMySQLDatastore(config)\n    else:\n      raise ConfigInitError()\n  elif existing_datastore == \"MySQLAdvancedDataStore\":\n    print(\"  MySQL Host: %s\\n  MySQL Port: %s\\n  MySQL Database: %s\\n\"\n          \"  MySQL Username: %s\\n\" %\n          (grr_config.CONFIG.Get(\"Mysql.host\"),\n           grr_config.CONFIG.Get(\"Mysql.port\"),\n           grr_config.CONFIG.Get(\"Mysql.database_name\"),\n           grr_config.CONFIG.Get(\"Mysql.database_username\")))\n    if grr_config.CONFIG.Get(\"Mysql.client_key_path\"):\n      print(\"  MySQL client key file: %s\\n\"\n            \"  MySQL client cert file: %s\\n\"\n            \"  MySQL ca cert file: %s\\n\" %\n            (grr_config.CONFIG.Get(\"Mysql.client_key_path\"),\n             grr_config.CONFIG.Get(\"Mysql.client_cert_path\"),\n             grr_config.CONFIG.Get(\"Mysql.ca_cert_path\")))\n\n    if not RetryBoolQuestion(\"Do you want to keep this configuration?\", True):\n      ConfigureMySQLDatastore(config)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nguiding the user through configuration of various URLs used by GRR.", "response": "def ConfigureUrls(config, external_hostname = None):\n  \"\"\"Guides the user through configuration of various URLs used by GRR.\"\"\"\n  print(\"\\n\\n-=GRR URLs=-\\n\"\n        \"For GRR to work each client has to be able to communicate with the\\n\"\n        \"server. To do this we normally need a public dns name or IP address\\n\"\n        \"to communicate with. In the standard configuration this will be used\\n\"\n        \"to host both the client facing server and the admin user interface.\\n\")\n\n  existing_ui_urn = grr_config.CONFIG.Get(\"AdminUI.url\", default=None)\n  existing_frontend_urns = grr_config.CONFIG.Get(\"Client.server_urls\")\n  if not existing_frontend_urns:\n    # Port from older deprecated setting Client.control_urls.\n    existing_control_urns = grr_config.CONFIG.Get(\n        \"Client.control_urls\", default=None)\n    if existing_control_urns is not None:\n      existing_frontend_urns = []\n      for existing_control_urn in existing_control_urns:\n        if not existing_control_urn.endswith(\"control\"):\n          raise RuntimeError(\"Invalid existing control URL: %s\" %\n                             existing_control_urn)\n\n        existing_frontend_urns.append(\n            existing_control_urn.rsplit(\"/\", 1)[0] + \"/\")\n\n      config.Set(\"Client.server_urls\", existing_frontend_urns)\n      config.Set(\"Client.control_urls\", [\"deprecated use Client.server_urls\"])\n\n  if not existing_frontend_urns or not existing_ui_urn:\n    ConfigureHostnames(config, external_hostname=external_hostname)\n  else:\n    print(\"Found existing settings:\\n  AdminUI URL: %s\\n  \"\n          \"Frontend URL(s): %s\\n\" % (existing_ui_urn, existing_frontend_urns))\n    if not RetryBoolQuestion(\"Do you want to keep this configuration?\", True):\n      ConfigureHostnames(config, external_hostname=external_hostname)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef ConfigureEmails(config):\n  print(\"\\n\\n-=GRR Emails=-\\n\"\n        \"GRR needs to be able to send emails for various logging and\\n\"\n        \"alerting functions. The email domain will be appended to GRR\\n\"\n        \"usernames when sending emails to users.\\n\")\n\n  existing_log_domain = grr_config.CONFIG.Get(\"Logging.domain\", default=None)\n  existing_al_email = grr_config.CONFIG.Get(\n      \"Monitoring.alert_email\", default=None)\n  existing_em_email = grr_config.CONFIG.Get(\n      \"Monitoring.emergency_access_email\", default=None)\n  if existing_log_domain and existing_al_email and existing_em_email:\n    print(\"Found existing settings:\\n\"\n          \"  Email Domain: %s\\n  Alert Email Address: %s\\n\"\n          \"  Emergency Access Email Address: %s\\n\" %\n          (existing_log_domain, existing_al_email, existing_em_email))\n    if RetryBoolQuestion(\"Do you want to keep this configuration?\", True):\n      return\n\n  print(\"\\n\\n-=Monitoring/Email Domain=-\\n\"\n        \"Emails concerning alerts or updates must be sent to this domain.\\n\")\n  domain = RetryQuestion(\"Email Domain e.g example.com\",\n                         \"^([\\\\.A-Za-z0-9-]+)*$\",\n                         grr_config.CONFIG.Get(\"Logging.domain\"))\n  config.Set(\"Logging.domain\", domain)\n\n  print(\"\\n\\n-=Alert Email Address=-\\n\"\n        \"Address where monitoring events get sent, e.g. crashed clients, \\n\"\n        \"broken server, etc.\\n\")\n  email = RetryQuestion(\"Alert Email Address\", \"\", \"grr-monitoring@%s\" % domain)\n  config.Set(\"Monitoring.alert_email\", email)\n\n  print(\"\\n\\n-=Emergency Email Address=-\\n\"\n        \"Address where high priority events such as an emergency ACL bypass \"\n        \"are sent.\\n\")\n  emergency_email = RetryQuestion(\"Emergency Access Email Address\", \"\",\n                                  \"grr-emergency@%s\" % domain)\n  config.Set(\"Monitoring.emergency_access_email\", emergency_email)", "response": "Configure emails for the GRR."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef InstallTemplatePackage():\n  virtualenv_bin = os.path.dirname(sys.executable)\n  extension = os.path.splitext(sys.executable)[1]\n  pip = \"%s/pip%s\" % (virtualenv_bin, extension)\n\n  # Install the GRR server component to satisfy the dependency below.\n  major_minor_version = \".\".join(\n      pkg_resources.get_distribution(\"grr-response-core\").version.split(\".\")\n      [0:2])\n  # Note that this version spec requires a recent version of pip\n  subprocess.check_call([\n      sys.executable, pip, \"install\", \"--upgrade\", \"-f\",\n      \"https://storage.googleapis.com/releases.grr-response.com/index.html\",\n      \"grr-response-templates==%s.*\" % major_minor_version\n  ])", "response": "Call pip to install the templates."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nfinalizing the configuration initialization.", "response": "def FinalizeConfigInit(config,\n                       token,\n                       admin_password = None,\n                       redownload_templates = False,\n                       repack_templates = True,\n                       prompt = True):\n  \"\"\"Performs the final steps of config initialization.\"\"\"\n  config.Set(\"Server.initialized\", True)\n  print(\"\\nWriting configuration to %s.\" % config[\"Config.writeback\"])\n  config.Write()\n  print(\"Initializing the datastore.\")\n  # Reload the config and initialize the GRR database.\n  server_startup.Init()\n\n  print(\"\\nStep 3: Adding GRR Admin User\")\n  try:\n    CreateUser(\"admin\", password=admin_password, is_admin=True)\n  except UserAlreadyExistsError:\n    if prompt:\n      # pytype: disable=wrong-arg-count\n      if ((builtins.input(\"User 'admin' already exists, do you want to \"\n                          \"reset the password? [yN]: \").upper() or \"N\") == \"Y\"):\n        UpdateUser(\"admin\", password=admin_password, is_admin=True)\n      # pytype: enable=wrong-arg-count\n    else:\n      UpdateUser(\"admin\", password=admin_password, is_admin=True)\n\n  print(\"\\nStep 4: Repackaging clients with new configuration.\")\n  if prompt:\n    redownload_templates = RetryBoolQuestion(\n        \"Server debs include client templates. Re-download templates?\", False)\n    repack_templates = RetryBoolQuestion(\"Repack client templates?\", True)\n  if redownload_templates:\n    InstallTemplatePackage()\n  # Build debug binaries, then build release binaries.\n  if repack_templates:\n    repacking.TemplateRepacker().RepackAllTemplates(upload=True, token=token)\n  print(\"\\nGRR Initialization complete! You can edit the new configuration \"\n        \"in %s.\\n\" % config[\"Config.writeback\"])\n  print(\"Please restart the service for the new configuration to take \"\n        \"effect.\\n\")"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ninitializes or update a GRR configuration.", "response": "def Initialize(config=None,\n               external_hostname = None,\n               admin_password = None,\n               redownload_templates = False,\n               repack_templates = True,\n               token = None):\n  \"\"\"Initialize or update a GRR configuration.\"\"\"\n\n  print(\"Checking write access on config %s\" % config[\"Config.writeback\"])\n  if not os.access(config.parser.filename, os.W_OK):\n    raise IOError(\"Config not writeable (need sudo?)\")\n\n  print(\"\\nStep 0: Importing Configuration from previous installation.\")\n  options_imported = 0\n  prev_config_file = config.Get(\"ConfigUpdater.old_config\", default=None)\n  if prev_config_file and os.access(prev_config_file, os.R_OK):\n    print(\"Found config file %s.\" % prev_config_file)\n    # pytype: disable=wrong-arg-count\n    if builtins.input(\"Do you want to import this configuration? \"\n                      \"[yN]: \").upper() == \"Y\":\n      options_imported = ImportConfig(prev_config_file, config)\n    # pytype: enable=wrong-arg-count\n  else:\n    print(\"No old config file found.\")\n\n  print(\"\\nStep 1: Setting Basic Configuration Parameters\")\n  print(\"We are now going to configure the server using a bunch of questions.\")\n  ConfigureDatastore(config)\n  ConfigureUrls(config, external_hostname=external_hostname)\n  ConfigureEmails(config)\n\n  print(\"\\nStep 2: Key Generation\")\n  if config.Get(\"PrivateKeys.server_key\", default=None):\n    if options_imported > 0:\n      print(\"Since you have imported keys from another installation in the \"\n            \"last step,\\nyou probably do not want to generate new keys now.\")\n    # pytype: disable=wrong-arg-count\n    if (builtins.input(\"You already have keys in your config, do you want to\"\n                       \" overwrite them? [yN]: \").upper() or \"N\") == \"Y\":\n      config_updater_keys_util.GenerateKeys(config, overwrite_keys=True)\n    # pytype: enable=wrong-arg-count\n  else:\n    config_updater_keys_util.GenerateKeys(config)\n\n  FinalizeConfigInit(\n      config,\n      token,\n      admin_password=admin_password,\n      redownload_templates=redownload_templates,\n      repack_templates=repack_templates,\n      prompt=True)"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ninitialize GRR with no prompts.", "response": "def InitializeNoPrompt(config=None,\n                       external_hostname = None,\n                       admin_password = None,\n                       mysql_hostname = None,\n                       mysql_port = None,\n                       mysql_username = None,\n                       mysql_password = None,\n                       mysql_db = None,\n                       mysql_client_key_path = None,\n                       mysql_client_cert_path = None,\n                       mysql_ca_cert_path = None,\n                       redownload_templates = False,\n                       repack_templates = True,\n                       token = None):\n  \"\"\"Initialize GRR with no prompts.\n\n  Args:\n    config: config object\n    external_hostname: A hostname.\n    admin_password: A password used for the admin user.\n    mysql_hostname: A hostname used for establishing connection to MySQL.\n    mysql_port: A port used for establishing connection to MySQL.\n    mysql_username: A username used for establishing connection to MySQL.\n    mysql_password: A password used for establishing connection to MySQL.\n    mysql_db: Name of the MySQL database to use.\n    mysql_client_key_path: The path name of the client private key file.\n    mysql_client_cert_path: The path name of the client public key certificate.\n    mysql_ca_cert_path: The path name of the CA certificate file.\n    redownload_templates: Indicates whether templates should be re-downloaded.\n    repack_templates: Indicates whether templates should be re-packed.\n    token: auth token\n\n  Raises:\n    ValueError: if required flags are not provided, or if the config has\n      already been initialized.\n    IOError: if config is not writeable\n    ConfigInitError: if GRR is unable to connect to a running MySQL instance.\n\n  This method does the minimum work necessary to configure GRR without any user\n  prompting, relying heavily on config default values. User must supply the\n  external hostname, admin password, and MySQL password; everything else is set\n  automatically.\n  \"\"\"\n  if config[\"Server.initialized\"]:\n    raise ValueError(\"Config has already been initialized.\")\n  if not external_hostname:\n    raise ValueError(\n        \"--noprompt set, but --external_hostname was not provided.\")\n  if not admin_password:\n    raise ValueError(\"--noprompt set, but --admin_password was not provided.\")\n  if mysql_password is None:\n    raise ValueError(\"--noprompt set, but --mysql_password was not provided.\")\n\n  print(\"Checking write access on config %s\" % config.parser)\n  if not os.access(config.parser.filename, os.W_OK):\n    raise IOError(\"Config not writeable (need sudo?)\")\n\n  config_dict = {}\n  config_dict[\"Datastore.implementation\"] = \"MySQLAdvancedDataStore\"\n  config_dict[\"Mysql.host\"] = mysql_hostname or config[\"Mysql.host\"]\n  config_dict[\"Mysql.port\"] = mysql_port or config[\"Mysql.port\"]\n  config_dict[\"Mysql.database_name\"] = mysql_db or config[\"Mysql.database_name\"]\n  config_dict[\"Mysql.database_username\"] = (\n      mysql_username or config[\"Mysql.database_username\"])\n  config_dict[\"Client.server_urls\"] = [\n      \"http://%s:%s/\" % (external_hostname, config[\"Frontend.bind_port\"])\n  ]\n  config_dict[\"AdminUI.url\"] = \"http://%s:%s\" % (external_hostname,\n                                                 config[\"AdminUI.port\"])\n  config_dict[\"Logging.domain\"] = external_hostname\n  config_dict[\"Monitoring.alert_email\"] = (\"grr-monitoring@%s\" %\n                                           external_hostname)\n  config_dict[\"Monitoring.emergency_access_email\"] = (\"grr-emergency@%s\" %\n                                                      external_hostname)\n  # Print all configuration options, except for the MySQL password.\n  print(\"Setting configuration as:\\n\\n%s\" % config_dict)\n  config_dict[\"Mysql.database_password\"] = mysql_password\n\n  if mysql_client_key_path is not None:\n    config_dict[\"Mysql.client_key_path\"] = mysql_client_key_path\n    config_dict[\"Mysql.client_cert_path\"] = mysql_client_cert_path\n    config_dict[\"Mysql.ca_cert_path\"] = mysql_ca_cert_path\n\n  if CheckMySQLConnection(config_dict):\n    print(\"Successfully connected to MySQL with the given configuration.\")\n  else:\n    print(\"Error: Could not connect to MySQL with the given configuration.\")\n    raise ConfigInitError()\n  for key, value in iteritems(config_dict):\n    config.Set(key, value)\n  config_updater_keys_util.GenerateKeys(config)\n  FinalizeConfigInit(\n      config,\n      token,\n      admin_password=admin_password,\n      redownload_templates=redownload_templates,\n      repack_templates=repack_templates,\n      prompt=False)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef UploadSignedBinary(source_path,\n                       binary_type,\n                       platform,\n                       upload_subdirectory=\"\"):\n  \"\"\"Signs a binary and uploads it to the datastore.\n\n  Args:\n    source_path: Path to the binary to upload.\n    binary_type: Type of the binary, e.g python-hack or executable.\n    platform: Client platform where the binary is intended to be run.\n    upload_subdirectory: Path of a subdirectory to upload the binary to,\n      relative to the canonical path for binaries of the given type and\n      platform.\n\n  Raises:\n    BinaryTooLargeError: If the binary to upload is too large.\n  \"\"\"\n  file_size = os.path.getsize(source_path)\n  if file_size > _MAX_SIGNED_BINARY_BYTES:\n    raise BinaryTooLargeError(\n        \"File [%s] is of size %d (bytes), which exceeds the allowed maximum \"\n        \"of %d bytes.\" % (source_path, file_size, _MAX_SIGNED_BINARY_BYTES))\n\n  context = [\"Platform:%s\" % platform.title(), \"Client Context\"]\n  signing_key = grr_config.CONFIG.Get(\n      \"PrivateKeys.executable_signing_private_key\", context=context)\n\n  root_api = maintenance_utils.InitGRRRootAPI()\n  binary_path = \"/\".join([\n      platform.lower(),\n      upload_subdirectory,\n      os.path.basename(source_path),\n  ])\n  binary = root_api.GrrBinary(binary_type, binary_path)\n\n  with open(source_path, \"rb\") as fd:\n    binary.Upload(\n        fd,\n        sign_fn=binary.DefaultUploadSigner(\n            private_key=signing_key.GetRawPrivateKey()))\n\n  print(\"Uploaded %s to %s\" % (binary_type, binary_path))", "response": "Uploads a binary to the datastore."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef CreateUser(username, password=None, is_admin=False):\n  grr_api = maintenance_utils.InitGRRRootAPI()\n  try:\n    user_exists = grr_api.GrrUser(username).Get() is not None\n  except api_errors.ResourceNotFoundError:\n    user_exists = False\n  if user_exists:\n    raise UserAlreadyExistsError(\"User '%s' already exists.\" % username)\n  user_type, password = _GetUserTypeAndPassword(\n      username, password=password, is_admin=is_admin)\n  grr_api.CreateGrrUser(\n      username=username, user_type=user_type, password=password)", "response": "Creates a new GRR user."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nupdate the password or privilege - level for a user.", "response": "def UpdateUser(username, password=None, is_admin=False):\n  \"\"\"Updates the password or privilege-level for a user.\"\"\"\n  user_type, password = _GetUserTypeAndPassword(\n      username, password=password, is_admin=is_admin)\n  grr_api = maintenance_utils.InitGRRRootAPI()\n  grr_user = grr_api.GrrUser(username).Get()\n  grr_user.Modify(user_type=user_type, password=password)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn a string with summary info for a user.", "response": "def GetUserSummary(username):\n  \"\"\"Returns a string with summary info for a user.\"\"\"\n  grr_api = maintenance_utils.InitGRRRootAPI()\n  try:\n    return _Summarize(grr_api.GrrUser(username).Get().data)\n  except api_errors.ResourceNotFoundError:\n    raise UserNotFoundError(username)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a string containing summary info for all GRR users.", "response": "def GetAllUserSummaries():\n  \"\"\"Returns a string containing summary info for all GRR users.\"\"\"\n  grr_api = maintenance_utils.InitGRRRootAPI()\n  user_wrappers = sorted(grr_api.ListGrrUsers(), key=lambda x: x.username)\n  summaries = [_Summarize(w.data) for w in user_wrappers]\n  return \"\\n\\n\".join(summaries)"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns a string with summary info for a user.", "response": "def _Summarize(user_info):\n  \"\"\"Returns a string with summary info for a user.\"\"\"\n  return \"Username: %s\\nIs Admin: %s\" % (\n      user_info.username,\n      user_info.user_type == api_user.ApiGrrUser.UserType.USER_TYPE_ADMIN)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef DeleteUser(username):\n  grr_api = maintenance_utils.InitGRRRootAPI()\n  try:\n    grr_api.GrrUser(username).Get().Delete()\n  except api_errors.ResourceNotFoundError:\n    raise UserNotFoundError(username)", "response": "Deletes a GRR user from the datastore."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _GetUserTypeAndPassword(username, password=None, is_admin=False):\n  if is_admin:\n    user_type = api_user.ApiGrrUser.UserType.USER_TYPE_ADMIN\n  else:\n    user_type = api_user.ApiGrrUser.UserType.USER_TYPE_STANDARD\n  if password is None:\n    # TODO\n    # pytype: disable=wrong-arg-types\n    password = getpass.getpass(prompt=\"Please enter password for user '%s':\" %\n                               username)\n    # pytype: enable=wrong-arg-types\n  return user_type, password", "response": "Returns the user - type and password for a user."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef FromResponses(cls, request=None, responses=None):\n    res = cls()\n    res.request = request\n    if request:\n      res.request_data = request.request_data\n\n    for r in responses or []:\n      if isinstance(r, rdf_flow_objects.FlowResponse):\n        res.responses.append(r.payload)\n      elif isinstance(r, rdf_flow_objects.FlowStatus):\n        res.status = r\n        res.success = r.status == \"OK\"\n      elif isinstance(r, rdf_flow_objects.FlowIterator):\n        pass\n      else:\n        raise TypeError(\"Got unexpected response type: %s\" % type(r))\n    return res", "response": "Creates a Responses object from new style flow request and responses."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncreate a Responses object from a legacy flow request and responses.", "response": "def FromLegacyResponses(cls, request=None, responses=None):\n    \"\"\"Creates a Responses object from old style flow request and responses.\"\"\"\n    res = cls()\n    res.request = request\n    if request:\n      res.request_data = rdf_protodict.Dict(request.data)\n    dropped_responses = []\n    # The iterator that was returned as part of these responses. This should\n    # be passed back to actions that expect an iterator.\n    res.iterator = None\n\n    if not responses:\n      return res\n\n    # This may not be needed if we can assume that responses are\n    # returned in lexical order from the data_store.\n    responses.sort(key=operator.attrgetter(\"response_id\"))\n\n    if request.HasField(\"request\"):\n      client_action_name = request.request.name\n      action_registry = server_stubs.ClientActionStub.classes\n      if client_action_name not in action_registry:\n        raise RuntimeError(\n            \"Got unknown client action: %s.\" % client_action_name)\n      expected_response_classes = action_registry[\n          client_action_name].out_rdfvalues\n\n    old_response_id = None\n\n    # Filter the responses by authorized states\n    for msg in responses:\n      # Check if the message is authenticated correctly.\n      if msg.auth_state != msg.AuthorizationState.AUTHENTICATED:\n        logging.warning(\"%s: Messages must be authenticated (Auth state %s)\",\n                        msg.session_id, msg.auth_state)\n        dropped_responses.append(msg)\n        # Skip this message - it is invalid\n        continue\n\n      # Handle retransmissions\n      if msg.response_id == old_response_id:\n        continue\n\n      old_response_id = msg.response_id\n\n      # Check for iterators\n      if msg.type == msg.Type.ITERATOR:\n        if res.iterator:\n          raise ValueError(\"Received multiple iterator messages at once.\")\n        res.iterator = rdf_client_action.Iterator(msg.payload)\n        continue\n\n      # Look for a status message\n      if msg.type == msg.Type.STATUS:\n        # Our status is set to the first status message that we see in\n        # the responses. We ignore all other messages after that.\n        res.status = rdf_flows.GrrStatus(msg.payload)\n\n        # Check this to see if the call succeeded\n        res.success = res.status.status == res.status.ReturnedStatus.OK\n\n        # Ignore all other messages\n        break\n\n      if msg.type == msg.Type.MESSAGE:\n        if request.HasField(\"request\"):\n          # Let's do some verification for requests that came from clients.\n          if not expected_response_classes:\n            raise RuntimeError(\"Client action %s does not specify out_rdfvalue.\"\n                               % client_action_name)\n          else:\n            args_rdf_name = msg.args_rdf_name\n            if not args_rdf_name:\n              raise RuntimeError(\"Deprecated message format received: \"\n                                 \"args_rdf_name is None.\")\n            elif args_rdf_name not in [\n                x.__name__ for x in expected_response_classes\n            ]:\n              raise RuntimeError(\"Response type was %s but expected %s for %s.\"\n                                 % (args_rdf_name, expected_response_classes,\n                                    client_action_name))\n      # Use this message\n      res.responses.append(msg.payload)\n\n    if res.status is None:\n      # This is a special case of de-synchronized messages.\n      if dropped_responses:\n        logging.error(\n            \"De-synchronized messages detected:\\n %s\",\n            \"\\n\".join([utils.SmartUnicode(x) for x in dropped_responses]))\n\n      res.LogFlowState(responses)\n\n      raise ValueError(\"No valid Status message.\")\n\n    return res"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nretrieves the configuration except for the blocked parameters.", "response": "def Run(self, unused_arg):\n    \"\"\"Retrieve the configuration except for the blocked parameters.\"\"\"\n\n    out = self.out_rdfvalues[0]()\n    for descriptor in config.CONFIG.type_infos:\n      if descriptor.name in self.BLOCKED_PARAMETERS:\n        value = \"[Redacted]\"\n      else:\n        try:\n          value = config.CONFIG.Get(descriptor.name, default=None)\n        except (config_lib.Error, KeyError, AttributeError, ValueError) as e:\n          logging.info(\"Config reading error: %s\", e)\n          continue\n\n      if value is not None:\n        out[descriptor.name] = value\n\n    self.SendReply(out)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef Run(self, arg):\n    try:\n      if self.grr_worker.client.FleetspeakEnabled():\n        raise ValueError(\"Not supported on Fleetspeak enabled clients.\")\n    except AttributeError:\n      pass\n\n    smart_arg = {str(field): value for field, value in iteritems(arg)}\n\n    disallowed_fields = [\n        field for field in smart_arg\n        if field not in UpdateConfiguration.UPDATABLE_FIELDS\n    ]\n\n    if disallowed_fields:\n      raise ValueError(\"Received an update request for restricted field(s) %s.\"\n                       % \",\".join(disallowed_fields))\n\n    if platform.system() != \"Windows\":\n      # Check config validity before really applying the changes. This isn't\n      # implemented for our Windows clients though, whose configs are stored in\n      # the registry, as opposed to in the filesystem.\n\n      canary_config = config.CONFIG.CopyConfig()\n\n      # Prepare a temporary file we'll write changes to.\n      with tempfiles.CreateGRRTempFile(mode=\"w+\") as temp_fd:\n        temp_filename = temp_fd.name\n\n      # Write canary_config changes to temp_filename.\n      canary_config.SetWriteBack(temp_filename)\n      self._UpdateConfig(smart_arg, canary_config)\n\n      try:\n        # Assert temp_filename is usable by loading it.\n        canary_config.SetWriteBack(temp_filename)\n      # Wide exception handling passed here from config_lib.py...\n      except Exception:  # pylint: disable=broad-except\n        logging.warning(\"Updated config file %s is not usable.\", temp_filename)\n        raise\n\n      # If temp_filename works, remove it (if not, it's useful for debugging).\n      os.unlink(temp_filename)\n\n    # The changes seem to work, so push them to the real config.\n    self._UpdateConfig(smart_arg, config.CONFIG)", "response": "Does the actual work."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef Run(self, arg):\n    if arg is None:\n      arg = rdf_client_action.GetClientStatsRequest()\n\n    proc = psutil.Process(os.getpid())\n    meminfo = proc.memory_info()\n    boot_time = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(psutil.boot_time())\n    create_time = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(proc.create_time())\n    stats_collector = stats_collector_instance.Get()\n    response = rdf_client_stats.ClientStats(\n        RSS_size=meminfo.rss,\n        VMS_size=meminfo.vms,\n        memory_percent=proc.memory_percent(),\n        bytes_received=stats_collector.GetMetricValue(\n            \"grr_client_received_bytes\"),\n        bytes_sent=stats_collector.GetMetricValue(\"grr_client_sent_bytes\"),\n        create_time=create_time,\n        boot_time=boot_time)\n\n    response.cpu_samples = self.grr_worker.stats_collector.CpuSamplesBetween(\n        start_time=arg.start_time, end_time=arg.end_time)\n    response.io_samples = self.grr_worker.stats_collector.IOSamplesBetween(\n        start_time=arg.start_time, end_time=arg.end_time)\n\n    self.Send(response)", "response": "Returns the client stats."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef Run(self, unused_arg, ttl=None):\n    logging.debug(\"Sending startup information.\")\n    boot_time = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(psutil.boot_time())\n    response = rdf_client.StartupInfo(\n        boot_time=boot_time, client_info=GetClientInformation())\n\n    self.grr_worker.SendReply(\n        response,\n        session_id=self.well_known_session_id,\n        response_id=0,\n        request_id=0,\n        message_type=rdf_flows.GrrMessage.Type.MESSAGE,\n        require_fastpoll=False,\n        ttl=ttl)", "response": "Sends the startup information."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef ConvertStringToFilename(name):\n  return re.sub(\n      r\"\\W\", lambda x: \"%%%02X\" % ord(x.group(0)), name,\n      flags=re.UNICODE).rstrip(\"/\")", "response": "Converts an unicode string to a filesystem safe filename."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nresolves the destination of a subject.", "response": "def ResolveSubjectDestination(subject, regexes):\n  \"\"\"Returns the directory/filename where the subject will be stored.\n\n  Args:\n   subject: The subject.\n   regexes: The list of regular expressions by priority.\n\n  Returns:\n   File name and directory.\n  \"\"\"\n  components = Components(subject)\n  if not components:\n    # No components to work with.\n    return \"aff4\", \"\"\n  # Make all the components safe to use.\n  path = utils.JoinPath(*[ConvertStringToFilename(x) for x in components])\n  for route in regexes:\n    m = route.match(path)\n    if m:\n      value = m.group(\"path\")\n      if value:\n        base = os.path.basename(value)\n        dirname = os.path.dirname(value)\n        return base, dirname\n  # Default value if nothing else matches.\n  return \"aff4\", \"\""}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncreating a name that identifies a database file.", "response": "def MakeDestinationKey(directory, filename):\n  \"\"\"Creates a name that identifies a database file.\"\"\"\n  return utils.SmartStr(utils.JoinPath(directory, filename)).lstrip(\"/\")"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef DatabaseDirectorySize(root_path, extension):\n  directories = collections.deque([root_path])\n  total_size = 0\n  total_files = 0\n  while directories:\n    directory = directories.popleft()\n    try:\n      items = os.listdir(directory)\n    except OSError:\n      continue\n    for comp in items:\n      path = os.path.join(directory, comp)\n      try:\n        statinfo = os.lstat(path)\n        if stat.S_ISLNK(statinfo.st_mode):\n          continue\n        if stat.S_ISDIR(statinfo.st_mode):\n          directories.append(path)\n        elif stat.S_ISREG(statinfo.st_mode):\n          if comp.endswith(extension):\n            total_size += statinfo.st_size\n            total_files += 1\n      except OSError:\n        continue\n  return total_size, total_files", "response": "Compute size ( in bytes ) and number of files of a file - based data store."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef GetChildrenByPriority(self, allow_external=True):\n    for child in sorted(self.OpenChildren(), key=lambda x: x.PRIORITY):\n      if not allow_external and child.EXTERNAL:\n        continue\n      if child.Get(child.Schema.ACTIVE):\n        yield child", "response": "Generator that yields active filestore children in priority order."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nchecks a list of hashes for presence in the store.", "response": "def CheckHashes(self, hashes, external=True):\n    \"\"\"Checks a list of hashes for presence in the store.\n\n    Sub stores need to pass back the original HashDigest objects since they\n    carry state about the original file source.\n\n    Only unique hashes are checked, if there is duplication in the hashes input\n    it is the caller's responsibility to maintain any necessary mappings.\n\n    Args:\n      hashes: A list of Hash objects to check.\n      external: If true, attempt to check stores defined as EXTERNAL.\n\n    Yields:\n      Tuples of (RDFURN, hash object) that exist in the store.\n    \"\"\"\n    hashes = set(hashes)\n    for child in self.GetChildrenByPriority(allow_external=external):\n      for urn, hash_obj in child.CheckHashes(hashes):\n        yield urn, hash_obj\n\n        hashes.discard(hash_obj)\n\n      # Nothing to search for, we are done.\n      if not hashes:\n        break"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef AddFile(self, fd, external=True):\n    files_for_write = []\n\n    for sub_store in self.GetChildrenByPriority(allow_external=external):\n      new_file = sub_store.AddFile(fd)\n      if new_file:\n        files_for_write.append(new_file)\n\n    fd.Seek(0)\n    while files_for_write:\n      # If we got filehandles back, send them the data.\n      data = fd.Read(self.CHUNK_SIZE)\n      if not data:\n        break\n\n      for child in files_for_write:\n        child.Write(data)\n\n    for child in files_for_write:\n      child.Close()", "response": "Add a file to the file store."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nquerying the index for matches starting with target_prefix.", "response": "def Query(cls, index_urn, target_prefix=\"\", limit=100, token=None):\n    \"\"\"Search the index for matches starting with target_prefix.\n\n    Args:\n       index_urn: The index to use. Should be a urn that points to the sha256\n         namespace.\n       target_prefix: The prefix to match against the index.\n       limit: Either a tuple of (start, limit) or a maximum number of results to\n         return.\n       token: A DB token.\n\n    Returns:\n      URNs of files which have the same data as this file - as read from the\n      index.\n    \"\"\"\n    return data_store.DB.FileHashIndexQuery(\n        index_urn, target_prefix, limit=limit)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef CheckHashes(self, hashes):\n    hash_map = {}\n    for hsh in hashes:\n      if hsh.HasField(\"sha256\"):\n        # The canonical name of the file is where we store the file hash.\n        hash_map[aff4.ROOT_URN.Add(\"files/hash/generic/sha256\").Add(\n            str(hsh.sha256))] = hsh\n\n    for metadata in aff4.FACTORY.Stat(list(hash_map)):\n      yield metadata[\"urn\"], hash_map[metadata[\"urn\"]]", "response": "Checks the hashes against the filestore."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _HashFile(self, fd):\n    hashes = data_store_utils.GetFileHashEntry(fd)\n    if hashes:\n      found_all = True\n      for fingerprint_type, hash_types in iteritems(self.HASH_TYPES):\n        for hash_type in hash_types:\n          if fingerprint_type == \"pecoff\":\n            hash_type = \"pecoff_%s\" % hash_type\n          if not hashes.HasField(hash_type):\n            found_all = False\n            break\n        if not found_all:\n          break\n      if found_all:\n        return hashes\n\n    fingerprinter = fingerprint.Fingerprinter(fd)\n    if \"generic\" in self.HASH_TYPES:\n      hashers = self._GetHashers(self.HASH_TYPES[\"generic\"])\n      fingerprinter.EvalGeneric(hashers=hashers)\n    if \"pecoff\" in self.HASH_TYPES:\n      hashers = self._GetHashers(self.HASH_TYPES[\"pecoff\"])\n      if hashers:\n        fingerprinter.EvalPecoff(hashers=hashers)\n\n    if not hashes:\n      hashes = fd.Schema.HASH()\n\n    for result in fingerprinter.HashIt():\n      fingerprint_type = result[\"name\"]\n      for hash_type in self.HASH_TYPES[fingerprint_type]:\n        if hash_type not in result:\n          continue\n\n        if hash_type == \"SignedData\":\n          # There can be several certs in the same file.\n          for signed_data in result[hash_type]:\n            hashes.signed_data.Append(\n                revision=signed_data[0],\n                cert_type=signed_data[1],\n                certificate=signed_data[2])\n          continue\n\n        # Set the hashes in the original object\n        if fingerprint_type == \"generic\":\n          hashes.Set(hash_type, result[hash_type])\n\n        elif fingerprint_type == \"pecoff\":\n          hashes.Set(\"pecoff_%s\" % hash_type, result[hash_type])\n\n        else:\n          logging.error(\"Unknown fingerprint_type %s.\", fingerprint_type)\n\n    return hashes", "response": "Get the required hashes in the file."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nadd a file to the hash file store.", "response": "def AddFile(self, fd):\n    \"\"\"Adds a file to the hash file store.\n\n    We take a file in the client space:\n      aff4:/C.123123123/fs/os/usr/local/blah\n\n    Hash it, update the hash in the original file if its different to the\n    one calculated on the client, and copy the original AFF4 object to\n\n      aff4:/files/hash/generic/sha256/123123123 (canonical reference)\n\n    We then create symlinks for all other hash types:\n\n      aff4:/files/hash/generic/sha1/345345345\n      aff4:/files/hash/generic/md5/456456456\n      aff4:/files/hash/pecoff/md5/aaaaaaaa (only for PEs)\n      aff4:/files/hash/pecoff/sha1/bbbbbbbb (only for PEs)\n\n    When present in PE files, the signing data (revision, cert_type,\n    certificate) is added to the original object.\n\n    This can't be done simply in the FileStore.Write() method with fixed hash\n    buffer sizes because the authenticode hashes need to track hashing of\n    different-sized regions based on the signature information.\n\n    Args:\n      fd: File open for reading.\n\n    Raises:\n      IOError: If there was an error writing the file.\n    \"\"\"\n    hashes = self._HashFile(fd)\n\n    # The empty file is very common, we don't keep the back references for it\n    # in the DB since it just takes up too much space.\n    empty_hash = (\"e3b0c44298fc1c149afbf4c8996fb924\"\n                  \"27ae41e4649b934ca495991b7852b855\")\n    if hashes.sha256 == empty_hash:\n      return\n\n    # Update the hashes field now that we have calculated them all.\n    fd.Set(fd.Schema.HASH, hashes)\n    fd.Flush()\n\n    # sha256 is the canonical location.\n    canonical_urn = self.PATH.Add(\"generic/sha256\").Add(str(hashes.sha256))\n    if not list(aff4.FACTORY.Stat([canonical_urn])):\n      aff4.FACTORY.Copy(fd.urn, canonical_urn)\n      # Remove the STAT entry, it makes no sense to copy it between clients.\n      with aff4.FACTORY.Open(\n          canonical_urn, mode=\"rw\", token=self.token) as new_fd:\n        new_fd.Set(new_fd.Schema.STAT(None))\n\n    self._AddToIndex(canonical_urn, fd.urn)\n\n    for hash_type, hash_digest in hashes.ListSetFields():\n      # Determine fingerprint type.\n      hash_type = hash_type.name\n      # No need to create a symlink for sha256, it's the canonical location.\n      if hash_type == \"sha256\":\n        continue\n      hash_digest = str(hash_digest)\n      fingerprint_type = \"generic\"\n      if hash_type.startswith(\"pecoff_\"):\n        fingerprint_type = \"pecoff\"\n        hash_type = hash_type[len(\"pecoff_\"):]\n      if hash_type not in self.HASH_TYPES[fingerprint_type]:\n        continue\n\n      file_store_urn = self.PATH.Add(fingerprint_type).Add(hash_type).Add(\n          hash_digest)\n\n      with aff4.FACTORY.Create(\n          file_store_urn, aff4.AFF4Symlink, token=self.token) as symlink:\n        symlink.Set(symlink.Schema.SYMLINK_TARGET, canonical_urn)\n\n    # We do not want to be externally written here.\n    return None"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns all the hashes in the file store.", "response": "def ListHashes(age=aff4.NEWEST_TIME):\n    \"\"\"Yields all the hashes in the file store.\n\n    Args:\n      age: AFF4 age specification. Only get hits corresponding to the given age\n        spec. Should be aff4.NEWEST_TIME or a time range given as a tuple\n        (start, end) in microseconds since Jan 1st, 1970. If just a microseconds\n        value is given it's treated as the higher end of the range, i.e. (0,\n        age). See aff4.FACTORY.ParseAgeSpecification for details.\n\n    Yields:\n      FileStoreHash instances corresponding to all the hashes in the file store.\n\n    Raises:\n      ValueError: if age was set to aff4.ALL_TIMES.\n    \"\"\"\n    if age == aff4.ALL_TIMES:\n      raise ValueError(\"age==aff4.ALL_TIMES is not allowed.\")\n\n    urns = []\n    for fingerprint_type, hash_types in iteritems(HashFileStore.HASH_TYPES):\n      for hash_type in hash_types:\n        urns.append(HashFileStore.PATH.Add(fingerprint_type).Add(hash_type))\n\n    for _, values in aff4.FACTORY.MultiListChildren(urns, age=age):\n      for value in values:\n        yield FileStoreHash(value)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef GetClientsForHash(cls, hash_obj, token=None, age=aff4.NEWEST_TIME):\n\n    if age == aff4.ALL_TIMES:\n      raise ValueError(\"age==aff4.ALL_TIMES is not supported.\")\n\n    results = cls.GetClientsForHashes([hash_obj], token=token, age=age)\n    for _, client_files in results:\n      for client_file in client_files:\n        yield client_file", "response": "Retrieves all clients for the given hash."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nretrieving all the clients for the given hashes.", "response": "def GetClientsForHashes(cls, hashes, token=None, age=aff4.NEWEST_TIME):\n    \"\"\"Yields (hash, client_files) pairs for all the specified hashes.\n\n    Args:\n      hashes: List of RDFURN's.\n      token: Security token.\n      age: AFF4 age specification. Only get hits corresponding to the given age\n        spec. Should be aff4.NEWEST_TIME or a time range given as a tuple\n        (start, end) in microseconds since Jan 1st, 1970. If just a microseconds\n        value is given it's treated as the higher end of the range, i.e. (0,\n        age). See aff4.FACTORY.ParseAgeSpecification for details.\n\n    Yields:\n      (hash, client_files) tuples, where hash is a FileStoreHash instance and\n      client_files is a list of RDFURN's corresponding to client files that\n      have the hash.\n\n    Raises:\n      ValueError: if age was set to aff4.ALL_TIMES.\n    \"\"\"\n    if age == aff4.ALL_TIMES:\n      raise ValueError(\"age==aff4.ALL_TIMES is not supported.\")\n    timestamp = aff4.FACTORY.ParseAgeSpecification(age)\n\n    index_objects = list(aff4.FACTORY.MultiOpen(hashes, token=token))\n    index_locations = {}\n    for o in index_objects:\n      index_locations.setdefault(o.urn, []).append(o.symlink_urn)\n    for hash_obj, client_files in data_store.DB.FileHashIndexQueryMultiple(\n        index_locations, timestamp=timestamp):\n      symlinks = index_locations[hash_obj]\n      for original_hash in symlinks:\n        hash_obj = original_hash or hash_obj\n        yield (FileStoreHash(hash_obj), client_files)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef CheckHashes(self, hashes, unused_external=True):\n    hash_map = {}\n    for hsh in hashes:\n      if hsh.HasField(\"sha1\"):\n        hash_urn = self.PATH.Add(str(hsh.sha1))\n        logging.debug(\"Checking URN %s\", str(hash_urn))\n        hash_map[hash_urn] = hsh\n\n    for metadata in aff4.FACTORY.Stat(list(hash_map)):\n      yield metadata[\"urn\"], hash_map[metadata[\"urn\"]]", "response": "Checks a list of hashes for presence in the store."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef AddHash(self, sha1, md5, crc, file_name, file_size, product_code_list,\n              op_system_code_list, special_code):\n    \"\"\"Adds a new file from the NSRL hash database.\n\n    We create a new subject in:\n      aff4:/files/nsrl/<sha1>\n    with all the other arguments as attributes.\n\n    Args:\n      sha1: SHA1 digest as a hex encoded string.\n      md5: MD5 digest as a hex encoded string.\n      crc: File CRC as an integer.\n      file_name: Filename.\n      file_size: Size of file.\n      product_code_list: List of products this file is part of.\n      op_system_code_list: List of operating systems this file is part of.\n      special_code: Special code (malicious/special/normal file).\n    \"\"\"\n    file_store_urn = self.PATH.Add(sha1)\n\n    special_code = self.FILE_TYPES.get(special_code, self.FILE_TYPES[\"\"])\n\n    with aff4.FACTORY.Create(\n        file_store_urn, NSRLFile, mode=\"w\", token=self.token) as fd:\n      fd.Set(\n          fd.Schema.NSRL(\n              sha1=sha1.decode(\"hex\"),\n              md5=md5.decode(\"hex\"),\n              crc32=crc,\n              file_name=file_name,\n              file_size=file_size,\n              product_code=product_code_list,\n              op_system_code=op_system_code_list,\n              file_type=special_code))", "response": "Adds a new file from the NSRL hash database."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef Run(self):\n    if not data_store.AFF4Enabled():\n      return\n\n    try:\n      filestore = aff4.FACTORY.Create(\n          FileStore.PATH, FileStore, mode=\"rw\", token=aff4.FACTORY.root_token)\n      filestore.Close()\n      hash_filestore = aff4.FACTORY.Create(\n          HashFileStore.PATH,\n          HashFileStore,\n          mode=\"rw\",\n          token=aff4.FACTORY.root_token)\n      hash_filestore.Close()\n      nsrl_filestore = aff4.FACTORY.Create(\n          NSRLFileStore.PATH,\n          NSRLFileStore,\n          mode=\"rw\",\n          token=aff4.FACTORY.root_token)\n      nsrl_filestore.Close()\n    except access_control.UnauthorizedAccess:\n      # The aff4:/files area is ACL protected, this might not work on components\n      # that have ACL enforcement.\n      pass", "response": "Create FileStore and HashFileStore namespaces."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef WriteArtifact(self, artifact, cursor=None):\n    name = Text(artifact.name)\n\n    try:\n      cursor.execute(\"INSERT INTO artifacts (name, definition) VALUES (%s, %s)\",\n                     [name, artifact.SerializeToString()])\n    except MySQLdb.IntegrityError as error:\n      if error.args[0] == mysql_error_constants.DUP_ENTRY:\n        raise db.DuplicatedArtifactError(name, cause=error)\n      else:\n        raise", "response": "Writes a new artifact to the database."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef ReadArtifact(self, name, cursor=None):\n    cursor.execute(\"SELECT definition FROM artifacts WHERE name = %s\", [name])\n\n    row = cursor.fetchone()\n    if row is None:\n      raise db.UnknownArtifactError(name)\n    else:\n      return _RowToArtifact(row)", "response": "Reads an artifact from the database."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef ReadAllArtifacts(self, cursor=None):\n    cursor.execute(\"SELECT definition FROM artifacts\")\n    return [_RowToArtifact(row) for row in cursor.fetchall()]", "response": "Reads all artifacts from the database."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef DeleteArtifact(self, name, cursor=None):\n\n    cursor.execute(\"DELETE FROM artifacts WHERE name = %s\", [name])\n\n    if cursor.rowcount == 0:\n      raise db.UnknownArtifactError(name)", "response": "Deletes an artifact with given name from the database."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nprocessing a certificate message.", "response": "def ProcessMessage(self, message):\n    \"\"\"Begins an enrollment flow for this client.\n\n    Args:\n        message: The Certificate sent by the client. Note that this message is\n          not authenticated.\n    \"\"\"\n    cert = rdf_crypto.Certificate(message.payload)\n\n    queue = self.well_known_session_id.Queue()\n\n    client_id = message.source\n\n    # It makes no sense to enrol the same client multiple times, so we\n    # eliminate duplicates. Note, that we can still enroll clients multiple\n    # times due to cache expiration.\n    try:\n      enrolment_cache.Get(client_id)\n      return\n    except KeyError:\n      enrolment_cache.Put(client_id, 1)\n\n    # Create a new client object for this client.\n    if data_store.AFF4Enabled():\n      client = aff4.FACTORY.Create(\n          client_id, aff4_grr.VFSGRRClient, mode=\"rw\", token=self.token)\n      client_cert = client.Get(client.Schema.CERT)\n\n    if data_store.RelationalDBEnabled():\n      try:\n        md = data_store.REL_DB.ReadClientMetadata(client_id.Basename())\n        client_cert = md.certificate\n      except db.UnknownClientError:\n        client_cert = None\n\n    if data_store.RelationalDBEnabled():\n      data_store.REL_DB.WriteClientMetadata(\n          client_id.Basename(), fleetspeak_enabled=False)\n\n    # Only enroll this client if it has no certificate yet.\n    if not client_cert:\n      # Start the enrollment flow for this client.\n\n      # Note, that the actual CAEnroler class is autogenerated from the\n      # CAEnrolerMixin by the DualDBFlow decorator confusing the linter - hence\n      # the disable directive.\n      flow.StartAFF4Flow(\n          client_id=client_id,\n          flow_name=CAEnroler.__name__,  # pylint: disable=undefined-variable\n          csr=cert,\n          queue=queue,\n          token=self.token)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _ForemanOp(self):\n    period = config.CONFIG[\"Client.foreman_check_frequency\"]\n    self._threads[\"Worker\"].SendReply(\n        rdf_protodict.DataBlob(),\n        session_id=rdfvalue.FlowSessionID(flow_name=\"Foreman\"),\n        require_fastpoll=False)\n    time.sleep(period)", "response": "Sends Foreman checks periodically."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nsends a block of messages through Fleetspeak.", "response": "def _SendMessages(self, grr_msgs, background=False):\n    \"\"\"Sends a block of messages through Fleetspeak.\"\"\"\n    message_list = rdf_flows.PackedMessageList()\n    communicator.Communicator.EncodeMessageList(\n        rdf_flows.MessageList(job=grr_msgs), message_list)\n    fs_msg = fs_common_pb2.Message(\n        message_type=\"MessageList\",\n        destination=fs_common_pb2.Address(service_name=\"GRR\"),\n        background=background)\n    fs_msg.data.Pack(message_list.AsPrimitiveProto())\n\n    try:\n      sent_bytes = self._fs.Send(fs_msg)\n    except (IOError, struct.error):\n      logging.critical(\"Broken local Fleetspeak connection (write end).\")\n      raise\n\n    stats_collector_instance.Get().IncrementCounter(\"grr_client_sent_bytes\",\n                                                    sent_bytes)"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nsends messages through Fleetspeak.", "response": "def _SendOp(self):\n    \"\"\"Sends messages through Fleetspeak.\"\"\"\n    msg = self._sender_queue.get()\n    msgs = []\n    background_msgs = []\n    if not msg.require_fastpoll:\n      background_msgs.append(msg)\n    else:\n      msgs.append(msg)\n\n    count = 1\n    size = len(msg.SerializeToString())\n\n    while count < 100 and size < 1024 * 1024:\n      try:\n        msg = self._sender_queue.get(timeout=1)\n        if not msg.require_fastpoll:\n          background_msgs.append(msg)\n        else:\n          msgs.append(msg)\n        count += 1\n        size += len(msg.SerializeToString())\n      except queue.Empty:\n        break\n\n    if msgs:\n      self._SendMessages(msgs)\n    if background_msgs:\n      self._SendMessages(background_msgs, background=True)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreceive a single message through Fleetspeak.", "response": "def _ReceiveOp(self):\n    \"\"\"Receives a single message through Fleetspeak.\"\"\"\n    try:\n      fs_msg, received_bytes = self._fs.Recv()\n    except (IOError, struct.error):\n      logging.critical(\"Broken local Fleetspeak connection (read end).\")\n      raise\n\n    received_type = fs_msg.data.TypeName()\n    if not received_type.endswith(\"grr.GrrMessage\"):\n      raise ValueError(\n          \"Unexpected proto type received through Fleetspeak: %r; expected \"\n          \"grr.GrrMessage.\" % received_type)\n\n    stats_collector_instance.Get().IncrementCounter(\"grr_client_received_bytes\",\n                                                    received_bytes)\n\n    grr_msg = rdf_flows.GrrMessage.FromSerializedString(fs_msg.data.value)\n    # Authentication is ensured by Fleetspeak.\n    grr_msg.auth_state = jobs_pb2.GrrMessage.AUTHENTICATED\n\n    self._threads[\"Worker\"].QueueMessages([grr_msg])"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef DeleteArtifactsFromDatastore(artifact_names, reload_artifacts=True):\n  artifacts_list = sorted(\n      REGISTRY.GetArtifacts(reload_datastore_artifacts=reload_artifacts))\n\n  to_delete = set(artifact_names)\n  deps = set()\n  for artifact_obj in artifacts_list:\n    if artifact_obj.name in to_delete:\n      continue\n\n    if GetArtifactDependencies(artifact_obj) & to_delete:\n      deps.add(str(artifact_obj.name))\n\n  if deps:\n    raise ValueError(\n        \"Artifact(s) %s depend(s) on one of the artifacts to delete.\" %\n        (\",\".join(deps)))\n\n  store = ArtifactCollection(rdfvalue.RDFURN(\"aff4:/artifact_store\"))\n  all_artifacts = list(store)\n\n  filtered_artifacts, found_artifact_names = set(), set()\n  for artifact_value in all_artifacts:\n    if artifact_value.name in to_delete:\n      found_artifact_names.add(artifact_value.name)\n    else:\n      filtered_artifacts.add(artifact_value)\n\n  if len(found_artifact_names) != len(to_delete):\n    not_found = to_delete - found_artifact_names\n    raise ValueError(\"Artifact(s) to delete (%s) not found.\" %\n                     \",\".join(not_found))\n\n  # TODO(user): this is ugly and error- and race-condition- prone.\n  # We need to store artifacts not in a *Collection, which is an\n  # append-only object, but in some different way that allows easy\n  # deletion. Possible option - just store each artifact in a separate object\n  # in the same folder.\n  store.Delete()\n\n  with data_store.DB.GetMutationPool() as pool:\n    for artifact_value in filtered_artifacts:\n      store.Add(artifact_value, mutation_pool=pool)\n\n  if data_store.RelationalDBEnabled():\n    for artifact_name in to_delete:\n      data_store.REL_DB.DeleteArtifact(str(artifact_name))\n\n  for artifact_value in to_delete:\n    REGISTRY.UnregisterArtifact(artifact_value)", "response": "Deletes a list of artifacts from the data store."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef ValidateSyntax(rdf_artifact):\n  if not rdf_artifact.doc:\n    raise rdf_artifacts.ArtifactSyntaxError(rdf_artifact, \"missing doc\")\n\n  for supp_os in rdf_artifact.supported_os:\n    valid_os = rdf_artifact.SUPPORTED_OS_LIST\n    if supp_os not in valid_os:\n      detail = \"invalid `supported_os` ('%s' not in %s)\" % (supp_os, valid_os)\n      raise rdf_artifacts.ArtifactSyntaxError(rdf_artifact, detail)\n\n  for condition in rdf_artifact.conditions:\n    # FIXME(hanuszczak): It does not look like the code below can throw\n    # `ConditionException`. Do we really need it then?\n    try:\n      of = objectfilter.Parser(condition).Parse()\n      of.Compile(objectfilter.BaseFilterImplementation)\n    except rdf_artifacts.ConditionError as e:\n      detail = \"invalid condition '%s'\" % condition\n      raise rdf_artifacts.ArtifactSyntaxError(rdf_artifact, detail, e)\n\n  for label in rdf_artifact.labels:\n    if label not in rdf_artifact.ARTIFACT_LABELS:\n      raise rdf_artifacts.ArtifactSyntaxError(rdf_artifact,\n                                              \"invalid label '%s'\" % label)\n\n  # Anything listed in provides must be defined in the KnowledgeBase\n  valid_provides = rdf_client.KnowledgeBase().GetKbFieldNames()\n  for kb_var in rdf_artifact.provides:\n    if kb_var not in valid_provides:\n      detail = \"broken `provides` ('%s' not in %s)\" % (kb_var, valid_provides)\n      raise rdf_artifacts.ArtifactSyntaxError(rdf_artifact, detail)\n\n  # Any %%blah%% path dependencies must be defined in the KnowledgeBase\n  for dep in GetArtifactPathDependencies(rdf_artifact):\n    if dep not in valid_provides:\n      detail = \"broken path dependencies ('%s' not in %s)\" % (dep,\n                                                              valid_provides)\n      raise rdf_artifacts.ArtifactSyntaxError(rdf_artifact, detail)\n\n  for source in rdf_artifact.sources:\n    try:\n      source.Validate()\n    except rdf_artifacts.ArtifactSourceSyntaxError as e:\n      raise rdf_artifacts.ArtifactSyntaxError(rdf_artifact, \"bad source\", e)", "response": "Validates artifact syntax.\n\n  This method can be used to validate individual artifacts as they are loaded,\n  without needing all artifacts to be loaded first, as for Validate().\n\n  Args:\n    rdf_artifact: RDF object artifact.\n\n  Raises:\n    ArtifactSyntaxError: If artifact syntax is invalid."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef ValidateDependencies(rdf_artifact):\n  for dependency in GetArtifactDependencies(rdf_artifact):\n    try:\n      dependency_obj = REGISTRY.GetArtifact(dependency)\n    except rdf_artifacts.ArtifactNotRegisteredError as e:\n      raise rdf_artifacts.ArtifactDependencyError(\n          rdf_artifact, \"missing dependency\", cause=e)\n\n    message = dependency_obj.error_message\n    if message:\n      raise rdf_artifacts.ArtifactDependencyError(\n          rdf_artifact, \"dependency error\", cause=message)", "response": "Validates artifact dependencies.\n\n  This method checks whether all dependencies of the artifact are present\n  and contain no errors.\n\n  This method can be called only after all other artifacts have been loaded.\n\n  Args:\n    rdf_artifact: RDF object artifact.\n\n  Raises:\n    ArtifactDependencyError: If a dependency is missing or contains errors."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a set of artifact names that depend on the given artifact.", "response": "def GetArtifactDependencies(rdf_artifact, recursive=False, depth=1):\n  \"\"\"Return a set of artifact dependencies.\n\n  Args:\n    rdf_artifact: RDF object artifact.\n    recursive: If True recurse into dependencies to find their dependencies.\n    depth: Used for limiting recursion depth.\n\n  Returns:\n    A set of strings containing the dependent artifact names.\n\n  Raises:\n    RuntimeError: If maximum recursion depth reached.\n  \"\"\"\n  deps = set()\n  for source in rdf_artifact.sources:\n    # ARTIFACT is the legacy name for ARTIFACT_GROUP\n    # per: https://github.com/ForensicArtifacts/artifacts/pull/143\n    # TODO(user): remove legacy support after migration.\n    if source.type in (rdf_artifacts.ArtifactSource.SourceType.ARTIFACT,\n                       rdf_artifacts.ArtifactSource.SourceType.ARTIFACT_GROUP):\n      if source.attributes.GetItem(\"names\"):\n        deps.update(source.attributes.GetItem(\"names\"))\n\n  if depth > 10:\n    raise RuntimeError(\"Max artifact recursion depth reached.\")\n\n  deps_set = set(deps)\n  if recursive:\n    for dep in deps:\n      artifact_obj = REGISTRY.GetArtifact(dep)\n      new_dep = GetArtifactDependencies(artifact_obj, True, depth=depth + 1)\n      if new_dep:\n        deps_set.update(new_dep)\n\n  return deps_set"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn all the artifacts in the list returns them and their dependencies.", "response": "def GetArtifactsDependenciesClosure(name_list, os_name=None):\n  \"\"\"For all the artifacts in the list returns them and their dependencies.\"\"\"\n\n  artifacts = set(REGISTRY.GetArtifacts(os_name=os_name, name_list=name_list))\n\n  dependencies = set()\n  for art in artifacts:\n    dependencies.update(GetArtifactDependencies(art, recursive=True))\n  if dependencies:\n    artifacts.update(\n        set(\n            REGISTRY.GetArtifacts(\n                os_name=os_name, name_list=list(dependencies))))\n\n  return artifacts"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef GetArtifactPathDependencies(rdf_artifact):\n  deps = set()\n  for source in rdf_artifact.sources:\n    for arg, value in iteritems(source.attributes):\n      paths = []\n      if arg in [\"path\", \"query\"]:\n        paths.append(value)\n      if arg == \"key_value_pairs\":\n        # This is a REGISTRY_VALUE {key:blah, value:blah} dict.\n        paths.extend([x[\"key\"] for x in value])\n      if arg in [\"keys\", \"paths\", \"path_list\", \"content_regex_list\"]:\n        paths.extend(value)\n      for path in paths:\n        for match in artifact_utils.INTERPOLATED_REGEX.finditer(path):\n          deps.add(match.group()[2:-2])  # Strip off %%.\n  deps.update(GetArtifactParserDependencies(rdf_artifact))\n  return deps", "response": "Returns a set of knowledgebase path dependencies for the given artifact."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning the set of knowledgebase path dependencies required by the parser.", "response": "def GetArtifactParserDependencies(rdf_artifact):\n  \"\"\"Return the set of knowledgebase path dependencies required by the parser.\n\n  Args:\n    rdf_artifact: RDF artifact object.\n\n  Returns:\n    A set of strings for the required kb objects e.g.\n    [\"users.appdata\", \"systemroot\"]\n  \"\"\"\n  deps = set()\n  processors = parser.Parser.GetClassesByArtifact(rdf_artifact.name)\n  for p in processors:\n    deps.update(p.knowledgebase_dependencies)\n  return deps"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef AddDir(self, dirpath):\n    if dirpath not in self._dirs:\n      self._dirs.add(dirpath)\n      return True\n    return False", "response": "Adds a directory path as a source."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nadd a file path as a source.", "response": "def AddFile(self, filepath):\n    \"\"\"Adds a file path as a source.\n\n    Args:\n      filepath: a string representing a path to the file.\n\n    Returns:\n      True if the file is not an already existing source.\n    \"\"\"\n    if filepath not in self._files:\n      self._files.add(filepath)\n      return True\n    return False"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef AddDatastore(self, urn):\n    if urn not in self._datastores:\n      self._datastores.add(urn)\n      return True\n    return False", "response": "Adds a datastore URN as a source."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nyielding all defined source file paths.", "response": "def GetAllFiles(self):\n    \"\"\"Yields all defined source file paths.\n\n    This includes file paths defined directly and those defined implicitly by\n    defining a directory.\n    \"\"\"\n    for filepath in self._files:\n      yield filepath\n\n    for dirpath in self._dirs:\n      for filepath in ArtifactRegistrySources._GetDirYamlFiles(dirpath):\n        if filepath in self._files:\n          continue\n        yield filepath"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nloading artifacts from the data store.", "response": "def _LoadArtifactsFromDatastore(self):\n    \"\"\"Load artifacts from the data store.\"\"\"\n    loaded_artifacts = []\n\n    # TODO(hanuszczak): Why do we have to remove anything? If some artifact\n    # tries to shadow system artifact shouldn't we just ignore them and perhaps\n    # issue some warning instead? The datastore being loaded should be read-only\n    # during upload.\n\n    # A collection of artifacts that shadow system artifacts and need\n    # to be deleted from the data store.\n    to_delete = []\n\n    for artifact_coll_urn in self._sources.GetDatastores():\n      artifact_coll = ArtifactCollection(artifact_coll_urn)\n\n      if data_store.RelationalDBEnabled():\n        artifact_list = data_store.REL_DB.ReadAllArtifacts()\n      else:\n        artifact_list = list(artifact_coll)\n\n      for artifact_value in artifact_list:\n        try:\n          self.RegisterArtifact(\n              artifact_value,\n              source=\"datastore:%s\" % artifact_coll_urn,\n              overwrite_if_exists=True)\n          loaded_artifacts.append(artifact_value)\n          logging.debug(\"Loaded artifact %s from %s\", artifact_value.name,\n                        artifact_coll_urn)\n        except rdf_artifacts.ArtifactDefinitionError as e:\n          # TODO(hanuszczak): String matching on exception message is rarely\n          # a good idea. Instead this should be refectored to some exception\n          # class and then handled separately.\n          if \"system artifact\" in str(e):\n            to_delete.append(artifact_value.name)\n          else:\n            raise\n\n    if to_delete:\n      DeleteArtifactsFromDatastore(to_delete, reload_artifacts=False)\n      self._dirty = True\n\n      # TODO(hanuszczak): This is connected to the previous TODO comment. Why\n      # do we throw exception at this point? Why do we delete something and then\n      # abort the whole upload procedure by throwing an exception?\n      detail = \"system artifacts were shadowed and had to be deleted\"\n      raise rdf_artifacts.ArtifactDefinitionError(to_delete, detail)\n\n    # Once all artifacts are loaded we can validate.\n    revalidate = True\n    while revalidate:\n      revalidate = False\n      for artifact_obj in loaded_artifacts[:]:\n        try:\n          Validate(artifact_obj)\n        except rdf_artifacts.ArtifactDefinitionError as e:\n          logging.error(\"Artifact %s did not validate: %s\", artifact_obj.name,\n                        e)\n          artifact_obj.error_message = utils.SmartStr(e)\n          loaded_artifacts.remove(artifact_obj)\n          revalidate = True"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef ArtifactsFromYaml(self, yaml_content):\n    raw_list = yaml.ParseMany(yaml_content)\n\n    # TODO(hanuszczak): I am very sceptical about that \"doing the right thing\"\n    # below. What are the real use cases?\n\n    # Try to do the right thing with json/yaml formatted as a list.\n    if (isinstance(raw_list, list) and len(raw_list) == 1 and\n        isinstance(raw_list[0], list)):\n      raw_list = raw_list[0]\n\n    # Convert json into artifact and validate.\n    valid_artifacts = []\n    for artifact_dict in raw_list:\n      # In this case we are feeding parameters directly from potentially\n      # untrusted yaml/json to our RDFValue class. However, safe_load ensures\n      # these are all primitive types as long as there is no other\n      # deserialization involved, and we are passing these into protobuf\n      # primitive types.\n      try:\n        artifact_value = rdf_artifacts.Artifact(**artifact_dict)\n        valid_artifacts.append(artifact_value)\n      except (TypeError, AttributeError, type_info.TypeValueError) as e:\n        name = artifact_dict.get(\"name\")\n        raise rdf_artifacts.ArtifactDefinitionError(\n            name, \"invalid definition\", cause=e)\n\n    return valid_artifacts", "response": "Get a list of Artifacts from a yaml file."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nloading artifacts from a list of file paths as json or yaml.", "response": "def _LoadArtifactsFromFiles(self, file_paths, overwrite_if_exists=True):\n    \"\"\"Load artifacts from file paths as json or yaml.\"\"\"\n    loaded_files = []\n    loaded_artifacts = []\n    for file_path in file_paths:\n      try:\n        with io.open(file_path, mode=\"r\", encoding=\"utf-8\") as fh:\n          logging.debug(\"Loading artifacts from %s\", file_path)\n          for artifact_val in self.ArtifactsFromYaml(fh.read()):\n            self.RegisterArtifact(\n                artifact_val,\n                source=\"file:%s\" % file_path,\n                overwrite_if_exists=overwrite_if_exists)\n            loaded_artifacts.append(artifact_val)\n            logging.debug(\"Loaded artifact %s from %s\", artifact_val.name,\n                          file_path)\n\n        loaded_files.append(file_path)\n      except (IOError, OSError) as e:\n        logging.error(\"Failed to open artifact file %s. %s\", file_path, e)\n      except rdf_artifacts.ArtifactDefinitionError as e:\n        logging.error(\"Invalid artifact found in file %s with error: %s\",\n                      file_path, e)\n        raise\n\n    # Once all artifacts are loaded we can validate.\n    for artifact_value in loaded_artifacts:\n      Validate(artifact_value)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef RegisterArtifact(self,\n                       artifact_rdfvalue,\n                       source=\"datastore\",\n                       overwrite_if_exists=False,\n                       overwrite_system_artifacts=False):\n    \"\"\"Registers a new artifact.\"\"\"\n    artifact_name = artifact_rdfvalue.name\n    if artifact_name in self._artifacts:\n      if not overwrite_if_exists:\n        details = \"artifact already exists and `overwrite_if_exists` is unset\"\n        raise rdf_artifacts.ArtifactDefinitionError(artifact_name, details)\n      elif not overwrite_system_artifacts:\n        artifact_obj = self._artifacts[artifact_name]\n        if not artifact_obj.loaded_from.startswith(\"datastore:\"):\n          # This artifact was not uploaded to the datastore but came from a\n          # file, refuse to overwrite.\n          details = \"system artifact cannot be overwritten\"\n          raise rdf_artifacts.ArtifactDefinitionError(artifact_name, details)\n\n    # Preserve where the artifact was loaded from to help debugging.\n    artifact_rdfvalue.loaded_from = source\n    # Clear any stale errors.\n    artifact_rdfvalue.error_message = None\n    self._artifacts[artifact_rdfvalue.name] = artifact_rdfvalue", "response": "Registers a new artifact."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreloads artifacts from all sources.", "response": "def _ReloadArtifacts(self):\n    \"\"\"Load artifacts from all sources.\"\"\"\n    self._artifacts = {}\n    self._LoadArtifactsFromFiles(self._sources.GetAllFiles())\n    self.ReloadDatastoreArtifacts()"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nremove artifacts that came from the datastore.", "response": "def _UnregisterDatastoreArtifacts(self):\n    \"\"\"Remove artifacts that came from the datastore.\"\"\"\n    to_remove = []\n    for name, artifact in iteritems(self._artifacts):\n      if artifact.loaded_from.startswith(\"datastore\"):\n        to_remove.append(name)\n    for key in to_remove:\n      self._artifacts.pop(key)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef GetArtifacts(self,\n                   os_name=None,\n                   name_list=None,\n                   source_type=None,\n                   exclude_dependents=False,\n                   provides=None,\n                   reload_datastore_artifacts=False):\n    \"\"\"Retrieve artifact classes with optional filtering.\n\n    All filters must match for the artifact to be returned.\n\n    Args:\n      os_name: string to match against supported_os\n      name_list: list of strings to match against artifact names\n      source_type: rdf_artifacts.ArtifactSource.SourceType to match against\n        source_type\n      exclude_dependents: if true only artifacts with no dependencies will be\n        returned\n      provides: return the artifacts that provide these dependencies\n      reload_datastore_artifacts: If true, the data store sources are queried\n        for new artifacts.\n\n    Returns:\n      set of artifacts matching filter criteria\n    \"\"\"\n    self._CheckDirty(reload_datastore_artifacts=reload_datastore_artifacts)\n    results = set()\n    for artifact in itervalues(self._artifacts):\n\n      # artifact.supported_os = [] matches all OSes\n      if os_name and artifact.supported_os and (\n          os_name not in artifact.supported_os):\n        continue\n      if name_list and artifact.name not in name_list:\n        continue\n      if source_type:\n        source_types = [c.type for c in artifact.sources]\n        if source_type not in source_types:\n          continue\n      if exclude_dependents and GetArtifactPathDependencies(artifact):\n        continue\n\n      if not provides:\n        results.add(artifact)\n      else:\n        # This needs to remain the last test, if it matches the result is added\n        for provide_string in artifact.provides:\n          if provide_string in provides:\n            results.add(artifact)\n            break\n\n    return results", "response": "Returns a set of artifacts that match the given filters."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef GetArtifact(self, name):\n    self._CheckDirty()\n    result = self._artifacts.get(name)\n    if not result:\n      raise rdf_artifacts.ArtifactNotRegisteredError(\n          \"Artifact %s missing from registry. You may need to sync the \"\n          \"artifact repo by running make in the artifact directory.\" % name)\n    return result", "response": "Get an artifact by name."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef SearchDependencies(self,\n                         os_name,\n                         artifact_name_list,\n                         existing_artifact_deps=None,\n                         existing_expansion_deps=None):\n    \"\"\"Return a set of artifact names needed to fulfill dependencies.\n\n    Search the path dependency tree for all artifacts that can fulfill\n    dependencies of artifact_name_list.  If multiple artifacts provide a\n    dependency, they are all included.\n\n    Args:\n      os_name: operating system string\n      artifact_name_list: list of artifact names to find dependencies for.\n      existing_artifact_deps: existing dependencies to add to, for recursion,\n        e.g. set([\"WindowsRegistryProfiles\", \"WindowsEnvironmentVariablePath\"])\n      existing_expansion_deps: existing expansion dependencies to add to, for\n        recursion, e.g. set([\"users.userprofile\", \"users.homedir\"])\n\n    Returns:\n      (artifact_names, expansion_names): a tuple of sets, one with artifact\n          names, the other expansion names\n    \"\"\"\n    artifact_deps = existing_artifact_deps or set()\n    expansion_deps = existing_expansion_deps or set()\n\n    artifact_objs = self.GetArtifacts(\n        os_name=os_name, name_list=artifact_name_list)\n    artifact_deps = artifact_deps.union([a.name for a in artifact_objs])\n\n    for artifact in artifact_objs:\n      expansions = GetArtifactPathDependencies(artifact)\n      if expansions:\n        expansion_deps = expansion_deps.union(set(expansions))\n        # Get the names of the artifacts that provide those expansions\n        new_artifact_names = self.GetArtifactNames(\n            os_name=os_name, provides=expansions)\n        missing_artifacts = new_artifact_names - artifact_deps\n\n        if missing_artifacts:\n          # Add those artifacts and any child dependencies\n          new_artifacts, new_expansions = self.SearchDependencies(\n              os_name,\n              new_artifact_names,\n              existing_artifact_deps=artifact_deps,\n              existing_expansion_deps=expansion_deps)\n          artifact_deps = artifact_deps.union(new_artifacts)\n          expansion_deps = expansion_deps.union(new_expansions)\n\n    return artifact_deps, expansion_deps", "response": "Search the path dependency tree for all artifacts that satisfy the given artifact names."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef DumpArtifactsToYaml(self, sort_by_os=True):\n    artifact_list = self.GetArtifacts()\n    if sort_by_os:\n      # Sort so its easier to split these if necessary.\n      yaml_list = []\n      done_set = set()\n      for os_name in rdf_artifacts.Artifact.SUPPORTED_OS_LIST:\n        done_set = set(a for a in artifact_list if a.supported_os == [os_name])\n        # Separate into knowledge_base and non-kb for easier sorting.\n        done_set = sorted(done_set, key=lambda x: x.name)\n        yaml_list.extend(x.ToYaml() for x in done_set if x.provides)\n        yaml_list.extend(x.ToYaml() for x in done_set if not x.provides)\n        artifact_list = artifact_list.difference(done_set)\n      yaml_list.extend(x.ToYaml() for x in artifact_list)  # The rest.\n    else:\n      yaml_list = [x.ToYaml() for x in artifact_list]\n\n    return \"---\\n\\n\".join(yaml_list)", "response": "Dump a list of artifacts into a yaml string."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef IsFleetspeakEnabledClient(grr_id, token=None):\n  if grr_id is None:\n    return False\n\n  if data_store.RelationalDBEnabled():\n    md = data_store.REL_DB.ReadClientMetadata(grr_id)\n    if not md:\n      return False\n    return md.fleetspeak_enabled\n\n  else:\n    with aff4.FACTORY.Create(\n        rdf_client.ClientURN(grr_id),\n        aff4.AFF4Object.classes[\"VFSGRRClient\"],\n        mode=\"r\",\n        token=token) as client:\n      return bool(client.Get(client.Schema.FLEETSPEAK_ENABLED))", "response": "Returns True if the GRR id is a Fleetspeak client."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef SendGrrMessageThroughFleetspeak(grr_id, msg):\n  fs_msg = fs_common_pb2.Message(\n      message_type=\"GrrMessage\",\n      destination=fs_common_pb2.Address(\n          client_id=GRRIDToFleetspeakID(grr_id), service_name=\"GRR\"))\n  fs_msg.data.Pack(msg.AsPrimitiveProto())\n  fleetspeak_connector.CONN.outgoing.InsertMessage(fs_msg)", "response": "Sends the given GrrMessage through FS."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning the GRR label to use for a fleetspeak client.", "response": "def GetLabelFromFleetspeak(client_id):\n  \"\"\"Returns the primary GRR label to use for a fleetspeak client.\"\"\"\n  res = fleetspeak_connector.CONN.outgoing.ListClients(\n      admin_pb2.ListClientsRequest(client_ids=[GRRIDToFleetspeakID(client_id)]))\n  if not res.clients or not res.clients[0].labels:\n    return fleetspeak_connector.unknown_label\n\n  for label in res.clients[0].labels:\n    if label.service_name != \"client\":\n      continue\n    if label.label in fleetspeak_connector.label_map:\n      return fleetspeak_connector.label_map[label.label]\n\n  return fleetspeak_connector.unknown_label"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef StartFileFetch(self, pathspec, request_data=None):\n    # Create an index so we can find this pathspec later.\n    self.state.indexed_pathspecs.append(pathspec)\n    self.state.request_data_list.append(request_data)\n    self._TryToStartNextPathspec()", "response": "Starts a new file fetch."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _TryToStartNextPathspec(self):\n\n    # Nothing to do here.\n    if self.state.maximum_pending_files <= len(self.state.pending_files):\n      return\n\n    if self.state.maximum_pending_files <= len(self.state.pending_hashes):\n      return\n\n    try:\n      index = self.state.next_pathspec_to_start\n      pathspec = self.state.indexed_pathspecs[index]\n      self.state.next_pathspec_to_start = index + 1\n    except IndexError:\n      # We did all the pathspecs, nothing left to do here.\n      return\n\n    # Add the file tracker to the pending hashes list where it waits until the\n    # hash comes back.\n    self.state.pending_hashes[index] = {\"index\": index}\n\n    # First state the file, then hash the file.\n\n    # TODO(hanuszczak): Support for old clients ends on 2021-01-01.\n    # This conditional should be removed after that date.\n    if self.client_version >= 3221:\n      stub = server_stubs.GetFileStat\n      request = rdf_client_action.GetFileStatRequest(pathspec=pathspec)\n      request_name = \"GetFileStat\"\n    else:\n      stub = server_stubs.StatFile\n      request = rdf_client_action.ListDirRequest(pathspec=pathspec)\n      request_name = \"StatFile\"\n\n    self.CallClient(\n        stub,\n        request,\n        next_state=\"StoreStat\",\n        request_data=dict(index=index, request_name=request_name))\n\n    request = rdf_client_action.FingerprintRequest(\n        pathspec=pathspec, max_filesize=self.state.file_size)\n    request.AddRequest(\n        fp_type=rdf_client_action.FingerprintTuple.Type.FPT_GENERIC,\n        hashers=[\n            rdf_client_action.FingerprintTuple.HashType.MD5,\n            rdf_client_action.FingerprintTuple.HashType.SHA1,\n            rdf_client_action.FingerprintTuple.HashType.SHA256\n        ])\n\n    self.CallClient(\n        server_stubs.HashFile,\n        request,\n        next_state=\"ReceiveFileHash\",\n        request_data=dict(index=index))", "response": "Try to start the next pathspec."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _RemoveCompletedPathspec(self, index):\n    pathspec = self.state.indexed_pathspecs[index]\n    request_data = self.state.request_data_list[index]\n\n    self.state.indexed_pathspecs[index] = None\n    self.state.request_data_list[index] = None\n    self.state.pending_hashes.pop(index, None)\n    self.state.pending_files.pop(index, None)\n\n    # We have a bit more room in the pending_hashes so we try to schedule\n    # another pathspec.\n    self._TryToStartNextPathspec()\n    return pathspec, request_data", "response": "Removes a pathspec from the list of pathspecs."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _ReceiveFetchedFile(self, tracker):\n    index = tracker[\"index\"]\n\n    _, request_data = self._RemoveCompletedPathspec(index)\n\n    # Report the request_data for this flow's caller.\n    self.ReceiveFetchedFile(\n        tracker[\"stat_entry\"], tracker[\"hash_obj\"], request_data=request_data)", "response": "Called by the FileFlow when a file is fetched."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _FileFetchFailed(self, index, request_name):\n\n    pathspec, request_data = self._RemoveCompletedPathspec(index)\n\n    # Report the request_data for this flow's caller.\n    self.FileFetchFailed(pathspec, request_name, request_data=request_data)", "response": "Remove the pathspec for this index and call the FileFetchFailed method."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef StoreStat(self, responses):\n    index = responses.request_data[\"index\"]\n    if not responses.success:\n      self.Log(\"Failed to stat file: %s\", responses.status)\n      # Report failure.\n      self._FileFetchFailed(index, responses.request_data[\"request_name\"])\n      return\n\n    tracker = self.state.pending_hashes[index]\n    tracker[\"stat_entry\"] = responses.First()", "response": "Stores the stat entry in the flow s state."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nadd hash digest to the tracker and check with filestore.", "response": "def ReceiveFileHash(self, responses):\n    \"\"\"Add hash digest to tracker and check with filestore.\"\"\"\n    index = responses.request_data[\"index\"]\n    if not responses.success:\n      self.Log(\"Failed to hash file: %s\", responses.status)\n      self.state.pending_hashes.pop(index, None)\n      # Report the error.\n      self._FileFetchFailed(index, \"HashFile\")\n      return\n\n    self.state.files_hashed += 1\n    response = responses.First()\n    if response.HasField(\"hash\"):\n      hash_obj = response.hash\n    else:\n      # Deprecate this method of returning hashes.\n      hash_obj = rdf_crypto.Hash()\n\n      if len(response.results) < 1 or response.results[0][\"name\"] != \"generic\":\n        self.Log(\"Failed to hash file: %s\", self.state.indexed_pathspecs[index])\n        self.state.pending_hashes.pop(index, None)\n        return\n\n      result = response.results[0]\n\n      try:\n        for hash_type in [\"md5\", \"sha1\", \"sha256\"]:\n          value = result.GetItem(hash_type)\n          setattr(hash_obj, hash_type, value)\n      except AttributeError:\n        self.Log(\"Failed to hash file: %s\", self.state.indexed_pathspecs[index])\n        self.state.pending_hashes.pop(index, None)\n        return\n\n    try:\n      tracker = self.state.pending_hashes[index]\n    except KeyError:\n      # Hashing the file failed, but we did stat it.\n      self._FileFetchFailed(index, responses.request.request.name)\n      return\n\n    tracker[\"hash_obj\"] = hash_obj\n    tracker[\"bytes_read\"] = response.bytes_read\n\n    self.state.files_hashed_since_check += 1\n    if self.state.files_hashed_since_check >= self.MIN_CALL_TO_FILE_STORE:\n      self._CheckHashesWithFileStore()"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _LegacyCheckHashesWithFileStore(self):\n    if not self.state.pending_hashes:\n      return\n\n    # This map represents all the hashes in the pending urns.\n    file_hashes = {}\n\n    # Store a mapping of hash to tracker. Keys are hashdigest objects,\n    # values are arrays of tracker dicts.\n    hash_to_tracker = {}\n    for index, tracker in iteritems(self.state.pending_hashes):\n\n      # We might not have gotten this hash yet\n      if tracker.get(\"hash_obj\") is None:\n        continue\n\n      hash_obj = tracker[\"hash_obj\"]\n      digest = hash_obj.sha256\n      file_hashes[index] = hash_obj\n      hash_to_tracker.setdefault(digest, []).append(tracker)\n\n    # First we get all the files which are present in the file store.\n    files_in_filestore = {}\n\n    # TODO(amoser): This object never changes, could this be a class attribute?\n    filestore_obj = aff4.FACTORY.Open(\n        legacy_filestore.FileStore.PATH,\n        legacy_filestore.FileStore,\n        mode=\"r\",\n        token=self.token)\n\n    for file_store_urn, hash_obj in filestore_obj.CheckHashes(\n        itervalues(file_hashes), external=self.state.use_external_stores):\n\n      # Since checkhashes only returns one digest per unique hash we need to\n      # find any other files pending download with the same hash.\n      for tracker in hash_to_tracker[hash_obj.sha256]:\n        self.state.files_skipped += 1\n        file_hashes.pop(tracker[\"index\"])\n        files_in_filestore[file_store_urn] = hash_obj\n        # Remove this tracker from the pending_hashes store since we no longer\n        # need to process it.\n        self.state.pending_hashes.pop(tracker[\"index\"])\n\n    # Now that the check is done, reset our counter\n    self.state.files_hashed_since_check = 0\n    # Now copy all existing files to the client aff4 space.\n    for filestore_file_urn, hash_obj in iteritems(files_in_filestore):\n\n      for file_tracker in hash_to_tracker.get(hash_obj.sha256, []):\n        stat_entry = file_tracker[\"stat_entry\"]\n        # Copy the existing file from the filestore to the client namespace.\n        target_urn = stat_entry.pathspec.AFF4Path(self.client_urn)\n\n        aff4.FACTORY.Copy(\n            filestore_file_urn, target_urn, update_timestamps=True)\n\n        with aff4.FACTORY.Open(\n            target_urn, mode=\"rw\", token=self.token) as new_fd:\n          new_fd.Set(new_fd.Schema.STAT, stat_entry)\n          # Due to potential filestore corruption, the existing files\n          # can have 0 size.\n          if new_fd.size == 0:\n            new_fd.size = (file_tracker[\"bytes_read\"] or stat_entry.st_size)\n\n        if data_store.RelationalDBEnabled():\n          path_info = rdf_objects.PathInfo.FromStatEntry(stat_entry)\n          path_info.hash_entry = hash_obj\n          data_store.REL_DB.WritePathInfos(self.client_id, [path_info])\n\n        # Add this file to the filestore index.\n        filestore_obj.AddURNToIndex(str(hash_obj.sha256), target_urn)\n\n        # Report this hit to the flow's caller.\n        self._ReceiveFetchedFile(file_tracker)\n\n    # Now we iterate over all the files which are not in the store and arrange\n    # for them to be copied.\n    for index in file_hashes:\n\n      # Move the tracker from the pending hashes store to the pending files\n      # store - it will now be downloaded.\n      file_tracker = self.state.pending_hashes.pop(index)\n      self.state.pending_files[index] = file_tracker\n\n      # If we already know how big the file is we use that, otherwise fall back\n      # to the size reported by stat.\n      if file_tracker[\"bytes_read\"] > 0:\n        file_tracker[\"size_to_download\"] = file_tracker[\"bytes_read\"]\n      else:\n        file_tracker[\"size_to_download\"] = file_tracker[\"stat_entry\"].st_size\n\n      # We do not have the file here yet - we need to retrieve it.\n      expected_number_of_hashes = (\n          file_tracker[\"size_to_download\"] // self.CHUNK_SIZE + 1)\n\n      # We just hash ALL the chunks in the file now. NOTE: This maximizes client\n      # VFS cache hit rate and is far more efficient than launching multiple\n      # GetFile flows.\n      self.state.files_to_fetch += 1\n\n      for i in range(expected_number_of_hashes):\n        if i == expected_number_of_hashes - 1:\n          # The last chunk is short.\n          length = file_tracker[\"size_to_download\"] % self.CHUNK_SIZE\n        else:\n          length = self.CHUNK_SIZE\n        self.CallClient(\n            server_stubs.HashBuffer,\n            pathspec=file_tracker[\"stat_entry\"].pathspec,\n            offset=i * self.CHUNK_SIZE,\n            length=length,\n            next_state=\"CheckHash\",\n            request_data=dict(index=index))\n\n    if self.state.files_hashed % 100 == 0:\n      self.Log(\"Hashed %d files, skipped %s already stored.\",\n               self.state.files_hashed, self.state.files_skipped)", "response": "Checks all queued up hashes for existence in file store."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nchecking all queued up hashes for existence in file store.", "response": "def _CheckHashesWithFileStore(self):\n    \"\"\"Check all queued up hashes for existence in file store.\n\n    Hashes which do not exist in the file store will be downloaded. This\n    function flushes the entire queue (self.state.pending_hashes) in order to\n    minimize the round trips to the file store.\n\n    If a file was found in the file store it is not scheduled for collection\n    and its PathInfo is written to the datastore pointing to the file store's\n    hash. Otherwise, we request the client to hash every block in the file,\n    and add it to the file tracking queue (self.state.pending_files).\n    \"\"\"\n    if not data_store.RelationalDBEnabled():\n      return self._LegacyCheckHashesWithFileStore()\n\n    if not self.state.pending_hashes:\n      return\n\n    # This map represents all the hashes in the pending urns.\n    file_hashes = {}\n\n    # Store a mapping of hash to tracker. Keys are hashdigest objects,\n    # values are arrays of tracker dicts.\n    hash_to_tracker = {}\n    for index, tracker in iteritems(self.state.pending_hashes):\n\n      # We might not have gotten this hash yet\n      if tracker.get(\"hash_obj\") is None:\n        continue\n\n      hash_obj = tracker[\"hash_obj\"]\n      digest = hash_obj.sha256\n      file_hashes[index] = hash_obj\n      hash_to_tracker.setdefault(rdf_objects.SHA256HashID(digest),\n                                 []).append(tracker)\n\n    # First we get all the files which are present in the file store.\n    files_in_filestore = set()\n\n    statuses = file_store.CheckHashes([\n        rdf_objects.SHA256HashID.FromBytes(ho.sha256.AsBytes())\n        for ho in itervalues(file_hashes)\n    ])\n    for hash_id, status in iteritems(statuses):\n      self.HeartBeat()\n\n      if not status:\n        continue\n\n      # Since checkhashes only returns one digest per unique hash we need to\n      # find any other files pending download with the same hash.\n      for tracker in hash_to_tracker[hash_id]:\n        self.state.files_skipped += 1\n        file_hashes.pop(tracker[\"index\"])\n        files_in_filestore.add(hash_id)\n        # Remove this tracker from the pending_hashes store since we no longer\n        # need to process it.\n        self.state.pending_hashes.pop(tracker[\"index\"])\n\n    # Now that the check is done, reset our counter\n    self.state.files_hashed_since_check = 0\n    # Now copy all existing files to the client aff4 space.\n    for hash_id in files_in_filestore:\n\n      for file_tracker in hash_to_tracker.get(hash_id, []):\n        stat_entry = file_tracker[\"stat_entry\"]\n        path_info = rdf_objects.PathInfo.FromStatEntry(stat_entry)\n        path_info.hash_entry = file_tracker[\"hash_obj\"]\n        data_store.REL_DB.WritePathInfos(self.client_id, [path_info])\n\n        # Report this hit to the flow's caller.\n        self._ReceiveFetchedFile(file_tracker)\n\n    # Now we iterate over all the files which are not in the store and arrange\n    # for them to be copied.\n    for index in file_hashes:\n\n      # Move the tracker from the pending hashes store to the pending files\n      # store - it will now be downloaded.\n      file_tracker = self.state.pending_hashes.pop(index)\n      self.state.pending_files[index] = file_tracker\n\n      # If we already know how big the file is we use that, otherwise fall back\n      # to the size reported by stat.\n      if file_tracker[\"bytes_read\"] > 0:\n        file_tracker[\"size_to_download\"] = file_tracker[\"bytes_read\"]\n      else:\n        file_tracker[\"size_to_download\"] = file_tracker[\"stat_entry\"].st_size\n\n      # We do not have the file here yet - we need to retrieve it.\n      expected_number_of_hashes = (\n          file_tracker[\"size_to_download\"] // self.CHUNK_SIZE + 1)\n\n      # We just hash ALL the chunks in the file now. NOTE: This maximizes client\n      # VFS cache hit rate and is far more efficient than launching multiple\n      # GetFile flows.\n      self.state.files_to_fetch += 1\n\n      for i in range(expected_number_of_hashes):\n        if i == expected_number_of_hashes - 1:\n          # The last chunk is short.\n          length = file_tracker[\"size_to_download\"] % self.CHUNK_SIZE\n        else:\n          length = self.CHUNK_SIZE\n\n        self.CallClient(\n            server_stubs.HashBuffer,\n            pathspec=file_tracker[\"stat_entry\"].pathspec,\n            offset=i * self.CHUNK_SIZE,\n            length=length,\n            next_state=\"CheckHash\",\n            request_data=dict(index=index))\n\n    if self.state.files_hashed % 100 == 0:\n      self.Log(\"Hashed %d files, skipped %s already stored.\",\n               self.state.files_hashed, self.state.files_skipped)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef CheckHash(self, responses):\n    index = responses.request_data[\"index\"]\n\n    if index not in self.state.pending_files:\n      # This is a blobhash for a file we already failed to read and logged as\n      # below, check here to avoid logging dups.\n      return\n\n    file_tracker = self.state.pending_files[index]\n\n    hash_response = responses.First()\n    if not responses.success or not hash_response:\n      urn = file_tracker[\"stat_entry\"].pathspec.AFF4Path(self.client_urn)\n      self.Log(\"Failed to read %s: %s\", urn, responses.status)\n      self._FileFetchFailed(index, responses.request.request.name)\n      return\n\n    file_tracker.setdefault(\"hash_list\", []).append(hash_response)\n\n    self.state.blob_hashes_pending += 1\n\n    if self.state.blob_hashes_pending > self.MIN_CALL_TO_FILE_STORE:\n      self.FetchFileContent()", "response": "Checks the block hash for a file."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef FetchFileContent(self):\n    if not self.state.pending_files:\n      return\n\n    # Check what blobs we already have in the blob store.\n    blob_hashes = []\n    for file_tracker in itervalues(self.state.pending_files):\n      for hash_response in file_tracker.get(\"hash_list\", []):\n        blob_hashes.append(rdf_objects.BlobID.FromBytes(hash_response.data))\n\n    # This is effectively a BlobStore call.\n    existing_blobs = data_store.BLOBS.CheckBlobsExist(blob_hashes)\n\n    self.state.blob_hashes_pending = 0\n\n    # If we encounter hashes that we already have, we will update\n    # self.state.pending_files right away so we can't use an iterator here.\n    for index, file_tracker in list(self.state.pending_files.items()):\n      for i, hash_response in enumerate(file_tracker.get(\"hash_list\", [])):\n        # Make sure we read the correct pathspec on the client.\n        hash_response.pathspec = file_tracker[\"stat_entry\"].pathspec\n\n        if existing_blobs[rdf_objects.BlobID.FromBytes(hash_response.data)]:\n          # If we have the data we may call our state directly.\n          self.CallStateInline(\n              messages=[hash_response],\n              next_state=\"WriteBuffer\",\n              request_data=dict(index=index, blob_index=i))\n        else:\n          # We dont have this blob - ask the client to transmit it.\n          self.CallClient(\n              server_stubs.TransferBuffer,\n              hash_response,\n              next_state=\"WriteBuffer\",\n              request_data=dict(index=index, blob_index=i))", "response": "Fetch the file s content as possible."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef WriteBuffer(self, responses):\n\n    index = responses.request_data[\"index\"]\n    if index not in self.state.pending_files:\n      return\n\n    # Failed to read the file - ignore it.\n    if not responses.success:\n      self._FileFetchFailed(index, responses.request.request.name)\n      return\n\n    response = responses.First()\n    file_tracker = self.state.pending_files.get(index)\n    if file_tracker:\n      blob_dict = file_tracker.setdefault(\"blobs\", {})\n      blob_index = responses.request_data[\"blob_index\"]\n      blob_dict[blob_index] = (response.data, response.length)\n\n      if len(blob_dict) == len(file_tracker[\"hash_list\"]):\n        # Write the file to the data store.\n        stat_entry = file_tracker[\"stat_entry\"]\n        urn = stat_entry.pathspec.AFF4Path(self.client_urn)\n\n        if data_store.AFF4Enabled():\n          with aff4.FACTORY.Create(\n              urn, aff4_grr.VFSBlobImage, mode=\"w\", token=self.token) as fd:\n\n            fd.SetChunksize(self.CHUNK_SIZE)\n            fd.Set(fd.Schema.STAT(stat_entry))\n            fd.Set(fd.Schema.PATHSPEC(stat_entry.pathspec))\n            fd.Set(fd.Schema.CONTENT_LAST(rdfvalue.RDFDatetime().Now()))\n\n            for index in sorted(blob_dict):\n              digest, length = blob_dict[index]\n              fd.AddBlob(rdf_objects.BlobID.FromBytes(digest), length)\n\n        if data_store.RelationalDBEnabled():\n          path_info = rdf_objects.PathInfo.FromStatEntry(stat_entry)\n\n          # Adding files to filestore requires reading data from RELDB,\n          # thus protecting this code with a filestore-read-enabled check.\n          if data_store.RelationalDBEnabled():\n            blob_refs = []\n            offset = 0\n            for index in sorted(blob_dict):\n              digest, size = blob_dict[index]\n              blob_refs.append(\n                  rdf_objects.BlobReference(\n                      offset=offset,\n                      size=size,\n                      blob_id=rdf_objects.BlobID.FromBytes(digest)))\n              offset += size\n\n            hash_obj = file_tracker[\"hash_obj\"]\n\n            client_path = db.ClientPath.FromPathInfo(self.client_id, path_info)\n            hash_id = file_store.AddFileWithUnknownHash(\n                client_path,\n                blob_refs,\n                use_external_stores=self.state.use_external_stores)\n            # If the hash that we've calculated matches what we got from the\n            # client, then simply store the full hash entry.\n            # Otherwise store just the hash that we've calculated.\n            if hash_id.AsBytes() == hash_obj.sha256:\n              path_info.hash_entry = hash_obj\n            else:\n              path_info.hash_entry.sha256 = hash_id.AsBytes()\n\n          data_store.REL_DB.WritePathInfos(self.client_id, [path_info])\n\n        if (not data_store.RelationalDBEnabled() and\n            self.state.use_external_stores):\n          # Publish the new file event to cause the file to be added to the\n          # filestore.\n          events.Events.PublishEvent(\n              \"LegacyFileStore.AddFileToStore\", urn, token=self.token)\n\n        # Save some space.\n        del file_tracker[\"blobs\"]\n        del file_tracker[\"hash_list\"]\n\n        # File done, remove from the store and close it.\n        self._ReceiveFetchedFile(file_tracker)\n\n        self.state.files_fetched += 1\n\n        if not self.state.files_fetched % 100:\n          self.Log(\"Fetched %d of %d files.\", self.state.files_fetched,\n                   self.state.files_to_fetch)", "response": "Write the hash received to the blob image."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nprocess the new file and add it to the file store.", "response": "def ProcessMessages(self, msgs=None, token=None):\n    \"\"\"Process the new file and add to the file store.\"\"\"\n    if not data_store.AFF4Enabled():\n      return\n\n    filestore_fd = aff4.FACTORY.Create(\n        legacy_filestore.FileStore.PATH,\n        legacy_filestore.FileStore,\n        mode=\"w\",\n        token=token)\n\n    for vfs_urn in msgs:\n      with aff4.FACTORY.Open(vfs_urn, mode=\"rw\", token=token) as vfs_fd:\n        try:\n          filestore_fd.AddFile(vfs_fd)\n        except Exception as e:  # pylint: disable=broad-except\n          logging.exception(\"Exception while adding file to filestore: %s\", e)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef Start(self):\n    # Anyone can create the hunt but it will be created in the paused\n    # state. Permissions are required to actually start it.\n    with implementation.StartHunt(\n        runner_args=self.args.hunt_runner_args,\n        args=self.args.hunt_args,\n        token=self.token) as hunt_obj:\n\n      # Nothing really to do here - hunts are always created in the paused\n      # state.\n      self.Log(\"User %s created a new %s hunt (%s)\", self.token.username,\n               hunt_obj.args.flow_runner_args.flow_name, hunt_obj.urn)", "response": "Create the hunt in the paused state."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef GetLaunchedFlows(self, flow_type=\"outstanding\"):\n    result = None\n    all_clients = set(self.ListAllClients())\n    finished_clients = set(self.ListFinishedClients())\n    outstanding_clients = all_clients - finished_clients\n\n    if flow_type == \"all\":\n      result = all_clients\n    elif flow_type == \"finished\":\n      result = finished_clients\n    elif flow_type == \"outstanding\":\n      result = outstanding_clients\n\n    # Now get the flows for all these clients.\n    flows = aff4.FACTORY.MultiListChildren(\n        [self.urn.Add(x.Basename()) for x in result])\n\n    return [x[0] for _, x in flows]", "response": "Retrieves the session IDs of all the flows we launched."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nmarking a client as done.", "response": "def MarkDone(self, responses):\n    \"\"\"Mark a client as done.\"\"\"\n    client_id = responses.request.client_id\n    self.AddResultsToCollection(responses, client_id)\n    self.MarkClientDone(client_id)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef ManuallyScheduleClients(self, token=None):\n\n    client_ids = set()\n    for flow_request in self.args.flows:\n      for client_id in flow_request.client_ids:\n        client_ids.add(client_id)\n\n    self.StartClients(self.session_id, client_ids, token=token)", "response": "Schedules all flows without using the Foreman.\n   "}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ntests to see if we re on a cloud machine.", "response": "def IsCloud(self, request, bios_version, services):\n    \"\"\"Test to see if we're on a cloud machine.\"\"\"\n    if request.bios_version_regex and bios_version:\n      if re.match(request.bios_version_regex, bios_version):\n        return True\n    if request.service_name_regex and services:\n      if re.search(request.service_name_regex, services):\n        return True\n    return False"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ngets metadata from local metadata server.", "response": "def GetMetaData(self, request):\n    \"\"\"Get metadata from local metadata server.\n\n    Any failed URL check will fail the whole action since our bios/service\n    checks may not always correctly identify cloud machines. We don't want to\n    wait on multiple DNS timeouts.\n\n    Args:\n      request: CloudMetadataRequest object\n    Returns:\n      rdf_cloud.CloudMetadataResponse object\n    Raises:\n      ValueError: if request has a timeout of 0. This is a defensive\n      check (we pass 1.0) because the requests library just times out and it's\n      not obvious why.\n    \"\"\"\n    if request.timeout == 0:\n      raise ValueError(\"Requests library can't handle timeout of 0\")\n    result = requests.request(\n        \"GET\", request.url, headers=request.headers, timeout=request.timeout)\n    # By default requests doesn't raise on HTTP error codes.\n    result.raise_for_status()\n\n    # Requests does not always raise an exception when an incorrect response\n    # is received. This fixes that behaviour.\n    if not result.ok:\n      raise requests.RequestException(response=result)\n\n    return rdf_cloud.CloudMetadataResponse(\n        label=request.label or request.url, text=result.text)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef MakePartialStat(self, fd):\n\n    is_dir = \"Container\" in fd.behaviours\n\n    return {\n        \"pathspec\": fd.Get(fd.Schema.PATHSPEC, \"\"),\n        \"st_atime\": fd.Get(fd.Schema.LAST, 0),\n        \"st_blksize\": 0,\n        \"st_blocks\": 0,\n        \"st_ctime\": 0,\n        \"st_dev\": 0,\n        \"st_gid\": 0,\n        \"st_ino\": 0,\n        \"st_mode\": self.default_dir_mode if is_dir else self.default_file_mode,\n        \"st_mtime\": 0,\n        \"st_nlink\": 0,\n        \"st_rdev\": 0,\n        \"st_size\": fd.Get(fd.Schema.SIZE, 0),\n        \"st_uid\": 0\n    }", "response": "Try and give a stat for something not in the data store."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreading a directory and returns a generator of filenames.", "response": "def Readdir(self, path, fh=None):\n    \"\"\"Reads a directory given by path.\n\n    Args:\n      path: The path to list children of.\n      fh: A file handler. Not used.\n\n    Yields:\n      A generator of filenames.\n\n    Raises:\n      FuseOSError: If we try and list a file.\n\n    \"\"\"\n    del fh\n\n    # We can't read a path if it's a file.\n    if not self._IsDir(path):\n      raise fuse.FuseOSError(errno.ENOTDIR)\n\n    fd = aff4.FACTORY.Open(self.root.Add(path), token=self.token)\n\n    children = fd.ListChildren()\n\n    # Make these special directories unicode to be consistent with the rest of\n    # aff4.\n    for directory in [u\".\", u\"..\"]:\n      yield directory\n\n    # ListChildren returns a generator, so we do the same.\n    for child in children:\n      # Filter out any directories we've chosen to ignore.\n      if child.Path() not in self.ignored_dirs:\n        yield child.Basename()"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nperform a stat on a file or directory.", "response": "def Getattr(self, path, fh=None):\n    \"\"\"Performs a stat on a file or directory.\n\n    Args:\n      path: The path to stat.\n      fh: A file handler. Not used.\n\n    Returns:\n      A dictionary mapping st_ names to their values.\n\n    Raises:\n      FuseOSError: When a path is supplied that grr doesn't know about, ie an\n      invalid file path.\n      ValueError: If an empty path is passed. (The empty string, when passed to\n      self.root.Add, returns a path for aff4:/, the root directory, which is not\n      the behaviour we want.)\n    \"\"\"\n    del fh\n\n    if not path:\n      raise fuse.FuseOSError(errno.ENOENT)\n\n    if path != self.root:\n      full_path = self.root.Add(path)\n    else:\n      full_path = path\n\n    fd = aff4.FACTORY.Open(full_path, token=self.token)\n\n    # The root aff4 path technically doesn't exist in the data store, so\n    # it is a special case.\n    if full_path == \"/\":\n      return self.MakePartialStat(fd)\n\n    fd = aff4.FACTORY.Open(full_path, token=self.token)\n    # Grab the stat according to aff4.\n    aff4_stat = fd.Get(fd.Schema.STAT)\n\n    # If the Schema for the object has a STAT attribute, go ahead and return\n    # it as a dictionary.\n    if aff4_stat:\n      return aff4_stat.AsDict()\n\n    # If the object didn't have a stored stat, we figure out if it is a special\n    # grr object, or just doesn't exist.\n\n    # We now check if the aff4 object actually has a row in the data store.\n    # This prevents us from being able to cd to directories that don't exist,\n    # since such directories have a newly-created empty AFF4Object,\n    # but no row in the data store. Anything that is a\n    # row in the data store will have a LAST attribute, so we check that.\n    elif fd.Get(fd.Schema.LAST) is None:\n      # We raise the \"no such file or directory\" error.\n      raise fuse.FuseOSError(errno.ENOENT)\n    else:\n      # This is an object that exists in the datastore, but has no STAT, so we\n      # don't know how to handle it.\n      pass\n\n    # If the object was in the data store, but didn't have a stat, we just\n    # try and guess some sensible values.\n    return self.MakePartialStat(fd)"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreads data from a file.", "response": "def Read(self, path, length=None, offset=0, fh=None):\n    \"\"\"Reads data from a file.\n\n    Args:\n      path: The path to the file to read.\n      length: How many bytes to read.\n      offset: Offset in bytes from which reading should start.\n      fh: A file handler. Not used.\n\n    Returns:\n      A string containing the file contents requested.\n\n    Raises:\n      FuseOSError: If we try and read a directory or if we try and read an\n      object that doesn't support reading.\n\n    \"\"\"\n    del fh\n\n    if self._IsDir(path):\n      raise fuse.FuseOSError(errno.EISDIR)\n\n    fd = aff4.FACTORY.Open(self.root.Add(path), token=self.token)\n\n    # If the object has Read() and Seek() methods, let's use them.\n    if all((hasattr(fd, \"Read\"), hasattr(fd, \"Seek\"), callable(fd.Read),\n            callable(fd.Seek))):\n      # By default, read the whole file.\n      if length is None:\n        length = fd.Get(fd.Schema.SIZE)\n\n      fd.Seek(offset)\n      return fd.Read(length)\n    else:\n      # If we don't have Read/Seek methods, we probably can't read this object.\n      raise fuse.FuseOSError(errno.EIO)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncheck if a given path has been updated in the cache.", "response": "def DataRefreshRequired(self, path=None, last=None):\n    \"\"\"True if we need to update this path from the client.\n\n    Args:\n      path: The path relative to the root to check freshness of.\n      last: An aff4:last attribute to check freshness of.\n\n      At least one of path or last must be supplied.\n\n    Returns:\n      True if the path hasn't been updated in the last\n      self.max_age_before_refresh seconds, else False.\n\n    Raises:\n      type_info.TypeValueError: If no arguments are supplied.\n    \"\"\"\n\n    # If we didn't get given a last attribute, use the path to get one from the\n    # object.\n    if last is None:\n      if path is None:\n        # If we didn't get a path either, we can't do anything.\n        raise type_info.TypeValueError(\"Either 'path' or 'last' must\"\n                                       \" be supplied as an argument.\")\n\n      fd = aff4.FACTORY.Open(self.root.Add(path), token=self.token)\n      # We really care about the last time the stat was updated, so we use\n      # this instead of the LAST attribute, which is the last time anything\n      # was updated about the object.\n      stat_obj = fd.Get(fd.Schema.STAT)\n      if stat_obj:\n        last = stat_obj.age\n      else:\n        last = rdfvalue.RDFDatetime(0)\n\n    # If the object doesn't even have a LAST attribute by this point,\n    # we say it hasn't been accessed within the cache expiry time.\n    if last is None:\n      return True\n    last = last.AsDatetime()\n\n    # Remember to use UTC time, since that's what the datastore uses.\n    return datetime.datetime.utcnow() - last > self.max_age_before_refresh"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nruns a flow on the client and waits for it to finish.", "response": "def _RunAndWaitForVFSFileUpdate(self, path):\n    \"\"\"Runs a flow on the client, and waits for it to finish.\"\"\"\n\n    client_id = rdf_client.GetClientURNFromPath(path)\n\n    # If we're not actually in a directory on a client, no need to run a flow.\n    if client_id is None:\n      return\n\n    flow_utils.UpdateVFSFileAndWait(\n        client_id,\n        token=self.token,\n        vfs_file_urn=self.root.Add(path),\n        timeout=self.timeout)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nupdating the directory listing from the client.", "response": "def Readdir(self, path, fh=None):\n    \"\"\"Updates the directory listing from the client.\n\n    Args:\n      path: The path to the directory to update. Client is inferred from this.\n      fh: A file handler. Not used.\n\n    Returns:\n      A list of filenames.\n\n    \"\"\"\n    if self.DataRefreshRequired(path):\n      self._RunAndWaitForVFSFileUpdate(path)\n\n    return super(GRRFuse, self).Readdir(path, fh=None)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn a list of the chunks that a file doesn t have.", "response": "def GetMissingChunks(self, fd, length, offset):\n    \"\"\"Return which chunks a file doesn't have.\n\n    Specifically, we return a list of the chunks specified by a\n    length-offset range which are not in the datastore.\n\n    Args:\n      fd: The database object to read chunks from.\n      length: Length to read.\n      offset: File offset to read from.\n\n    Returns:\n      A list of chunk numbers.\n    \"\"\"\n    start_chunk = offset // fd.chunksize\n    end_chunk = (offset + length - 1) // fd.chunksize\n\n    relevant_chunks = range(start_chunk, end_chunk + 1)\n\n    missing_chunks = set(relevant_chunks)\n    for idx, metadata in iteritems(fd.ChunksMetadata(relevant_chunks)):\n      if not self.DataRefreshRequired(last=metadata.get(\"last\", None)):\n        missing_chunks.remove(idx)\n\n    return sorted(missing_chunks)"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nlist all the filesystems mounted on the system.", "response": "def GetMountpoints():\n  \"\"\"List all the filesystems mounted on the system.\"\"\"\n  devices = {}\n\n  for filesys in GetFileSystems():\n    devices[filesys.f_mntonname] = (filesys.f_mntfromname, filesys.f_fstypename)\n\n  return devices"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef GetFileSystems():\n  version = OSXVersion()\n  major, minor = version.VersionAsMajorMinor()\n\n  libc = ctypes.cdll.LoadLibrary(ctypes.util.find_library(\"c\"))\n\n  if major <= 10 and minor <= 5:\n    use_64 = False\n    fs_struct = StatFSStruct\n  else:\n    use_64 = True\n    fs_struct = StatFS64Struct\n\n  # Get max 20 file systems.\n  struct_size = fs_struct.GetSize()\n  buf_size = struct_size * 20\n\n  cbuf = ctypes.create_string_buffer(buf_size)\n\n  if use_64:\n    # MNT_NOWAIT = 2 - don't ask the filesystems, just return cache.\n    ret = libc.getfsstat64(ctypes.byref(cbuf), buf_size, 2)\n  else:\n    ret = libc.getfsstat(ctypes.byref(cbuf), buf_size, 2)\n\n  if ret == 0:\n    logging.debug(\"getfsstat failed err: %s\", ret)\n    return []\n  return ParseFileSystemsStruct(fs_struct, ret, cbuf)", "response": "Make syscalls to get the mounted filesystems."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ntakes the struct type and parse it into a list of structs.", "response": "def ParseFileSystemsStruct(struct_class, fs_count, data):\n  \"\"\"Take the struct type and parse it into a list of structs.\"\"\"\n  results = []\n  cstr = lambda x: x.split(b\"\\x00\", 1)[0]\n  for count in range(0, fs_count):\n    struct_size = struct_class.GetSize()\n    s_data = data[count * struct_size:(count + 1) * struct_size]\n    s = struct_class(s_data)\n    s.f_fstypename = cstr(s.f_fstypename)\n    s.f_mntonname = cstr(s.f_mntonname)\n    s.f_mntfromname = cstr(s.f_mntfromname)\n    results.append(s)\n  return results"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nresolves the raw device that contains the path.", "response": "def GetRawDevice(path):\n  \"\"\"Resolve the raw device that contains the path.\"\"\"\n  device_map = GetMountpoints()\n\n  path = utils.SmartUnicode(path)\n  mount_point = path = utils.NormalizePath(path, \"/\")\n\n  result = rdf_paths.PathSpec(pathtype=rdf_paths.PathSpec.PathType.OS)\n\n  # Assign the most specific mount point to the result\n  while mount_point:\n    try:\n      result.path, fs_type = device_map[mount_point]\n      if fs_type in [\n          \"ext2\", \"ext3\", \"ext4\", \"vfat\", \"ntfs\", \"Apple_HFS\", \"hfs\", \"msdos\"\n      ]:\n        # These are read filesystems\n        result.pathtype = rdf_paths.PathSpec.PathType.OS\n      else:\n        result.pathtype = rdf_paths.PathSpec.PathType.UNSET\n\n      # Drop the mount point\n      path = utils.NormalizePath(path[len(mount_point):])\n\n      return result, path\n    except KeyError:\n      mount_point = os.path.dirname(mount_point)"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ninstalls the IOKit driver for the given file - system path.", "response": "def InstallDriver(kext_path):\n  \"\"\"Calls into the IOKit to load a kext by file-system path.\n\n  Apple kext API doco here:\n    http://developer.apple.com/library/mac/#documentation/IOKit/Reference/\n      KextManager_header_reference/Reference/reference.html\n\n  Args:\n    kext_path: Absolute or relative POSIX path to the kext.\n\n  Raises:\n    OSError: On failure to load the kext.\n  \"\"\"\n  km = objc.KextManager()\n\n  cf_kext_path = km.PyStringToCFString(kext_path)\n  kext_url = km.dll.CFURLCreateWithFileSystemPath(\n      objc.CF_DEFAULT_ALLOCATOR, cf_kext_path, objc.POSIX_PATH_STYLE, True)\n  status = km.iokit.KextManagerLoadKextWithURL(kext_url, None)\n\n  km.dll.CFRelease(kext_url)\n  km.dll.CFRelease(cf_kext_path)\n  if status is not objc.OS_SUCCESS:\n    raise OSError(\"Failed to load kext at {0}: {1}\".format(kext_path, status))"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef UninstallDriver(bundle_name):\n  km = objc.KextManager()\n\n  cf_bundle_name = km.PyStringToCFString(bundle_name)\n  status = km.iokit.KextManagerUnloadKextWithIdentifier(cf_bundle_name)\n  km.dll.CFRelease(cf_bundle_name)\n  return status", "response": "Unloads a kernel extension by its name."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nvalidate a sequence of path infos.", "response": "def _ValidatePathInfos(path_infos):\n  \"\"\"Validates a sequence of path infos.\"\"\"\n  precondition.AssertIterableType(path_infos, rdf_objects.PathInfo)\n\n  validated = set()\n  for path_info in path_infos:\n    _ValidatePathInfo(path_info)\n\n    path_key = (path_info.path_type, path_info.GetPathID())\n    if path_key in validated:\n      message = \"Conflicting writes for path: '{path}' ({path_type})\".format(\n          path=\"/\".join(path_info.components), path_type=path_info.path_type)\n      raise ValueError(message)\n\n    if path_info.HasField(\"hash_entry\"):\n      if path_info.hash_entry.sha256 is None:\n        message = \"Path with hash entry without SHA256: {}\".format(path_info)\n        raise ValueError(message)\n\n    validated.add(path_key)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nvalidate that a timerange argument is a valid time range.", "response": "def _ValidateTimeRange(timerange):\n  \"\"\"Parses a timerange argument and always returns non-None timerange.\"\"\"\n  if len(timerange) != 2:\n    raise ValueError(\"Timerange should be a sequence with 2 items.\")\n\n  (start, end) = timerange\n  precondition.AssertOptionalType(start, rdfvalue.RDFDatetime)\n  precondition.AssertOptionalType(end, rdfvalue.RDFDatetime)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _ValidateClosedTimeRange(time_range):\n  time_range_start, time_range_end = time_range\n  _ValidateTimestamp(time_range_start)\n  _ValidateTimestamp(time_range_end)\n  if time_range_start > time_range_end:\n    raise ValueError(\"Invalid time-range: %d > %d.\" %\n                     (time_range_start.AsMicrosecondsSinceEpoch(),\n                      time_range_end.AsMicrosecondsSinceEpoch()))", "response": "Checks that a time - range has both start and end timestamps set."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nwrite the client metadata for the given client_id.", "response": "def WriteClientMetadata(self,\n                          client_id,\n                          certificate=None,\n                          fleetspeak_enabled=None,\n                          first_seen=None,\n                          last_ping=None,\n                          last_clock=None,\n                          last_ip=None,\n                          last_foreman=None):\n    \"\"\"Write metadata about the client.\n\n    Updates one or more client metadata fields for the given client_id. Any of\n    the data fields can be left as None, and in this case are not changed.\n\n    Args:\n      client_id: A GRR client id string, e.g. \"C.ea3b2b71840d6fa7\".\n      certificate: If set, should be an rdfvalues.crypto.RDFX509 protocol\n        buffer. Normally only set during initial client record creation.\n      fleetspeak_enabled: A bool, indicating whether the client is connecting\n        through Fleetspeak.  Normally only set during initial client record\n        creation.\n      first_seen: An rdfvalue.Datetime, indicating the first time the client\n        contacted the server.\n      last_ping: An rdfvalue.Datetime, indicating the last time the client\n        contacted the server.\n      last_clock: An rdfvalue.Datetime, indicating the last client clock time\n        reported to the server.\n      last_ip: An rdfvalues.client.NetworkAddress, indicating the last observed\n        ip address for the client.\n      last_foreman: An rdfvalue.Datetime, indicating the last time that the\n        client sent a foreman message to the server.\n    \"\"\""}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef ReadClientMetadata(self, client_id):\n    result = self.MultiReadClientMetadata([client_id])\n    try:\n      return result[client_id]\n    except KeyError:\n      raise UnknownClientError(client_id)", "response": "Reads the ClientMetadata record for a single GRR client."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef ReadClientFullInfo(self, client_id):\n    result = self.MultiReadClientFullInfo([client_id])\n    try:\n      return result[client_id]\n    except KeyError:\n      raise UnknownClientError(client_id)", "response": "Reads full client information for a single client."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreading all available clients and yields all client full info protobufs.", "response": "def IterateAllClientsFullInfo(self, min_last_ping=None, batch_size=50000):\n    \"\"\"Iterates over all available clients and yields full info protobufs.\n\n    Args:\n      min_last_ping: If not None, only the clients with last-ping timestamps\n        newer than (or equal to) min_last_ping will be returned.\n      batch_size: Always reads <batch_size> client full infos at a time.\n\n    Yields:\n      An rdfvalues.objects.ClientFullInfo object for each client in the db.\n    \"\"\"\n    all_client_ids = self.ReadAllClientIDs(min_last_ping=min_last_ping)\n\n    for batch in collection.Batch(all_client_ids, batch_size):\n      res = self.MultiReadClientFullInfo(batch)\n      for full_info in itervalues(res):\n        yield full_info"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef IterateAllClientSnapshots(self, min_last_ping=None, batch_size=50000):\n    all_client_ids = self.ReadAllClientIDs(min_last_ping=min_last_ping)\n\n    for batch in collection.Batch(all_client_ids, batch_size):\n      res = self.MultiReadClientSnapshot(batch)\n      for snapshot in itervalues(res):\n        if snapshot:\n          yield snapshot", "response": "Reads all client snapshots from the database and yields them as ClientSnapshots objects."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef ListChildPathInfos(self, client_id, path_type, components,\n                         timestamp=None):\n    \"\"\"Lists path info records that correspond to children of given path.\n\n    Args:\n      client_id: An identifier string for a client.\n      path_type: A type of a path to retrieve path information for.\n      components: A tuple of path components of a path to retrieve child path\n        information for.\n      timestamp: If set, lists only descendants that existed only at that\n        timestamp.\n\n    Returns:\n      A list of `rdf_objects.PathInfo` instances sorted by path components.\n    \"\"\"\n    return self.ListDescendentPathInfos(\n        client_id, path_type, components, max_depth=1, timestamp=timestamp)", "response": "Lists path info records that correspond to children of given path."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef InitPathInfos(self, client_id, path_infos):\n    self.ClearPathHistory(client_id, path_infos)\n    self.WritePathInfos(client_id, path_infos)", "response": "Initializes a collection of path info records for a client."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef WritePathStatHistory(self, client_path, stat_entries):\n    client_path_history = ClientPathHistory()\n    for timestamp, stat_entry in iteritems(stat_entries):\n      client_path_history.AddStatEntry(timestamp, stat_entry)\n\n    self.MultiWritePathHistory({client_path: client_path_history})", "response": "Writes a collection of StatEntry observed for particular path."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nwrites a collection of Hash observed for particular path.", "response": "def WritePathHashHistory(self, client_path, hash_entries):\n    \"\"\"Writes a collection of `Hash` observed for particular path.\n\n    Args:\n      client_path: A `ClientPath` instance.\n      hash_entries: A dictionary with timestamps as keys and `Hash` instances as\n        values.\n    \"\"\"\n    client_path_history = ClientPathHistory()\n    for timestamp, hash_entry in iteritems(hash_entries):\n      client_path_history.AddHashEntry(timestamp, hash_entry)\n\n    self.MultiWritePathHistory({client_path: client_path_history})"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef ReadPathInfoHistory(self, client_id, path_type, components):\n    histories = self.ReadPathInfosHistories(client_id, path_type, [components])\n    return histories[components]", "response": "Reads a collection of hash and stat entries for given path."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef UpdateCronJob(self,\n                    cronjob_id,\n                    last_run_status=unchanged,\n                    last_run_time=unchanged,\n                    current_run_id=unchanged,\n                    state=unchanged,\n                    forced_run_requested=unchanged):\n    \"\"\"Updates run information for an existing cron job.\n\n    Args:\n      cronjob_id: The id of the cron job to update.\n      last_run_status: A CronJobRunStatus object.\n      last_run_time: The last time a run was started for this cron job.\n      current_run_id: The id of the currently active run.\n      state: The state dict for stateful cron jobs.\n      forced_run_requested: A boolean indicating if a forced run is pending for\n        this job.\n\n    Raises:\n      UnknownCronJobError: A cron job with the given id does not exist.\n    \"\"\"", "response": "Updates the run information for an existing cron job."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef UpdateFlow(self,\n                 client_id,\n                 flow_id,\n                 flow_obj=unchanged,\n                 flow_state=unchanged,\n                 client_crash_info=unchanged,\n                 pending_termination=unchanged,\n                 processing_on=unchanged,\n                 processing_since=unchanged,\n                 processing_deadline=unchanged):\n    \"\"\"Updates flow objects in the database.\n\n    Args:\n      client_id: The client id on which this flow is running.\n      flow_id: The id of the flow to update.\n      flow_obj: An updated rdf_flow_objects.Flow object.\n      flow_state: An update rdf_flow_objects.Flow.FlowState value.\n      client_crash_info: A rdf_client.ClientCrash object to store with the flow.\n      pending_termination: An rdf_flow_objects.PendingFlowTermination object.\n        Indicates that this flow is scheduled for termination.\n      processing_on: Worker this flow is currently processed on.\n      processing_since: Timstamp when the worker started processing this flow.\n      processing_deadline: Time after which this flow will be considered stuck\n        if processing hasn't finished.\n    \"\"\"", "response": "Updates the flow objects in the database."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef UpdateHuntObject(self,\n                       hunt_id,\n                       duration=None,\n                       client_rate=None,\n                       client_limit=None,\n                       hunt_state=None,\n                       hunt_state_comment=None,\n                       start_time=None,\n                       num_clients_at_start_time=None):\n    \"\"\"Updates the hunt object by applying the update function.\n\n    Each keyword argument when set to None, means that that corresponding value\n    shouldn't be updated.\n\n    Args:\n      hunt_id: Id of the hunt to be updated.\n      duration: A maximum allowed running time duration of the flow.\n      client_rate: Number correpsonding to hunt's client rate.\n      client_limit: Number corresponding hunt's client limit.\n      hunt_state: New Hunt.HuntState value.\n      hunt_state_comment: String correpsonding to a hunt state comment.\n      start_time: RDFDatetime corresponding to a start time of the hunt.\n      num_clients_at_start_time: Integer corresponding to a number of clients at\n        start time.\n    \"\"\"", "response": "Updates the hunt object by applying the update function."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nupdates the hunt object by applying the update function.", "response": "def UpdateHuntObject(self,\n                       hunt_id,\n                       duration=None,\n                       client_rate=None,\n                       client_limit=None,\n                       hunt_state=None,\n                       hunt_state_comment=None,\n                       start_time=None,\n                       num_clients_at_start_time=None):\n    \"\"\"Updates the hunt object by applying the update function.\"\"\"\n    _ValidateHuntId(hunt_id)\n    precondition.AssertOptionalType(duration, rdfvalue.Duration)\n    precondition.AssertOptionalType(client_rate, (float, int))\n    precondition.AssertOptionalType(client_limit, int)\n    if hunt_state is not None:\n      _ValidateEnumType(hunt_state, rdf_hunt_objects.Hunt.HuntState)\n    precondition.AssertOptionalType(hunt_state_comment, str)\n    precondition.AssertOptionalType(start_time, rdfvalue.RDFDatetime)\n    precondition.AssertOptionalType(num_clients_at_start_time, int)\n\n    return self.delegate.UpdateHuntObject(\n        hunt_id,\n        duration=duration,\n        client_rate=client_rate,\n        client_limit=client_limit,\n        hunt_state=hunt_state,\n        hunt_state_comment=hunt_state_comment,\n        start_time=start_time,\n        num_clients_at_start_time=num_clients_at_start_time)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _RemoveFromPool(self):\n    with self.pool.lock:\n\n      # Pool is shutting down, we can't interfere.\n      if not self.pool.started:\n        return False\n\n      # Keep a minimum number of threads in the pool.\n      if len(self.pool) <= self.pool.min_threads:\n        return False\n\n      # Remove us from our pool.\n      self.pool._RemoveWorker(self.name)  # pylint: disable=protected-access\n\n      return True", "response": "Remove ourself from the pool."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncreating a new thread pool with the given name.", "response": "def Factory(cls, name, min_threads, max_threads=None):\n    \"\"\"Creates a new thread pool with the given name.\n\n    If the thread pool of this name already exist, we just return the existing\n    one. This allows us to have different pools with different characteristics\n    used by different parts of the code, at the same time.\n\n    Args:\n      name: The name of the required pool.\n      min_threads: The number of threads in the pool.\n      max_threads: The maximum number of threads to grow the pool to. If not set\n        we do not grow the pool.\n\n    Returns:\n      A threadpool instance.\n    \"\"\"\n    with cls.factory_lock:\n      result = cls.POOLS.get(name)\n      if result is None:\n        cls.POOLS[name] = result = cls(\n            name, min_threads, max_threads=max_threads)\n\n      return result"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nadds a task to the queue.", "response": "def AddTask(self,\n              target,\n              args=(),\n              name=\"Unnamed task\",\n              blocking=True,\n              inline=True):\n    \"\"\"Adds a task to be processed later.\n\n    Args:\n      target: A callable which should be processed by one of the workers.\n      args: A tuple of arguments to target.\n      name: The name of this task. Used to identify tasks in the log.\n      blocking: If True we block until the task is finished, otherwise we raise\n        queue.Full\n      inline: If set, process the task inline when the queue is full. This\n        implies no blocking. Specifying inline helps if the worker tasks are\n        blocked because it still ensures some progress is made. However, this\n        can generally block the calling thread even after the threadpool is\n        available again and therefore decrease efficiency.\n\n    Raises:\n      ThreadPoolNotStartedError: if the pool was not started yet.\n      queue.Full: if the pool is full and can not accept new jobs.\n    \"\"\"\n    if not self.started:\n      raise ThreadPoolNotStartedError(self.name)\n\n    # This pool should have no worker threads - just run the task inline.\n    if self.max_threads == 0:\n      target(*args)\n      return\n\n    if inline:\n      blocking = False\n\n    with self.lock:\n      while True:\n        # This check makes sure that the threadpool will add new workers\n        # even if the queue is not full. This is needed for a scenario when\n        # a fresh threadpool is created (say, with min_threads=1 and\n        # max_threads=10) and 2 long-running tasks are added. The code below\n        # will spawn a new worker for a second long-running task.\n        if len(self) < self.max_threads:\n          try:\n            self._AddWorker()\n          except (RuntimeError, threading.ThreadError) as e:\n            logging.error(\n                \"Threadpool exception: \"\n                \"Could not spawn worker threads: %s\", e)\n\n        try:\n          # Push the task on the queue but raise if unsuccessful.\n          self._queue.put((target, args, name, time.time()), block=False)\n          return\n        except queue.Full:\n          # We increase the number of active threads if we do not exceed the\n          # maximum _and_ our process CPU utilization is not too high. This\n          # ensures that if the workers are waiting on IO we add more workers,\n          # but we do not waste workers when tasks are CPU bound.\n          if len(self) < self.max_threads:\n            try:\n              self._AddWorker()\n              continue\n\n            # If we fail to add a worker we should keep going anyway.\n            except (RuntimeError, threading.ThreadError) as e:\n              logging.error(\n                  \"Threadpool exception: \"\n                  \"Could not spawn worker threads: %s\", e)\n\n          # If we need to process the task inline just break out of the loop,\n          # therefore releasing the lock and run the task inline.\n          if inline:\n            break\n\n          # We should block and try again soon.\n          elif blocking:\n            try:\n              self._queue.put((target, args, name, time.time()),\n                              block=True,\n                              timeout=1)\n              return\n            except queue.Full:\n              continue\n\n          else:\n            raise Full()\n\n    # We don't want to hold the lock while running the task inline\n    if inline:\n      target(*args)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef Join(self):\n\n    for _ in range(self.JOIN_TIMEOUT_DECISECONDS):\n      if self._queue.empty() and not self.busy_threads:\n        return\n      time.sleep(0.1)\n\n    raise ValueError(\"Timeout during Join() for threadpool %s.\" % self.name)", "response": "Waits until all outstanding tasks are completed."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef Convert(self, values, start_index=0, end_index=None):\n    if not values:\n      return\n\n    try:\n      total_batch_count = len(values) // self.batch_size\n    except TypeError:\n      total_batch_count = -1\n\n    pool = ThreadPool.Factory(self.threadpool_prefix, self.threadpool_size)\n    val_iterator = itertools.islice(values, start_index, end_index)\n\n    pool.Start()\n    try:\n      for batch_index, batch in enumerate(\n          collection.Batch(val_iterator, self.batch_size)):\n        logging.debug(\"Processing batch %d out of %d\", batch_index,\n                      total_batch_count)\n\n        pool.AddTask(\n            target=self.ConvertBatch,\n            args=(batch,),\n            name=\"batch_%d\" % batch_index,\n            inline=False)\n\n    finally:\n      pool.Stop(join_timeout=3600)", "response": "Converts given collection to exported values."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef GetClientConfig(filename):\n  config_lib.SetPlatformArchContext()\n  config_lib.ParseConfigCommandLine()\n  context = list(grr_config.CONFIG.context)\n  context.append(\"Client Context\")\n  deployer = build.ClientRepacker()\n  # Disable timestamping so we can get a reproducible and cachable config file.\n  config_data = deployer.GetClientConfig(\n      context, validate=True, deploy_timestamp=False)\n  builder = build.ClientBuilder()\n  with open(filename, \"w\") as fd:\n    fd.write(config_data)\n    builder.WriteBuildYaml(fd, build_timestamp=False)", "response": "Write client config to filename."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef main(args):\n\n  grr_config.CONFIG.AddContext(contexts.CLIENT_BUILD_CONTEXT)\n\n  if args.subparser_name == \"generate_client_config\":\n    # We don't need a full init to just build a config.\n    GetClientConfig(args.client_config_output)\n    return\n\n  # TODO(user): Find out if adding the client-builder context is still\n  # necessary.\n  context = FLAGS.context\n  context.append(contexts.CLIENT_BUILD_CONTEXT)\n\n  config_lib.SetPlatformArchContext()\n  config_lib.ParseConfigCommandLine()\n\n  # Use basic console output logging so we can see what is happening.\n  logger = logging.getLogger()\n  handler = logging.StreamHandler()\n  handler.setLevel(logging.DEBUG if FLAGS.verbose else logging.INFO)\n  logger.handlers = [handler]\n\n  if args.subparser_name == \"build\":\n    if grr_config.CONFIG[\"Client.fleetspeak_enabled\"]:\n      if grr_config.CONFIG.ContextApplied(\"Platform:Darwin\"):\n        if not args.fleetspeak_service_config:\n          raise RuntimeError(\"--fleetspeak_service_config must be provided.\")\n        if not grr_config.CONFIG.Get(\"ClientBuilder.install_dir\"):\n          raise RuntimeError(\"ClientBuilder.install_dir must be set.\")\n        if not grr_config.CONFIG.Get(\"ClientBuilder.fleetspeak_plist_path\"):\n          raise RuntimeError(\"ClientBuilder.fleetspeak_plist_path must be set.\")\n      grr_config.CONFIG.Set(\"ClientBuilder.client_path\",\n                            \"grr_response_client.grr_fs_client\")\n    TemplateBuilder().BuildTemplate(\n        context=context,\n        output=args.output,\n        fleetspeak_service_config=args.fleetspeak_service_config)\n  elif args.subparser_name == \"repack\":\n    if args.debug_build:\n      context.append(\"DebugClientBuild Context\")\n    result_path = repacking.TemplateRepacker().RepackTemplate(\n        args.template,\n        args.output_dir,\n        context=context,\n        sign=args.sign,\n        signed_template=args.signed_template)\n\n    if not result_path:\n      raise ErrorDuringRepacking(\" \".join(sys.argv[:]))\n  elif args.subparser_name == \"repack_multiple\":\n\n    # Resolve globs manually on Windows.\n    templates = []\n    for template in args.templates:\n      if \"*\" in template:\n        templates.extend(glob.glob(template))\n      else:\n        # This could go through glob but then we'd swallow errors for\n        # non existing files.\n        templates.append(template)\n\n    repack_configs = []\n    for repack_config in args.repack_configs:\n      if \"*\" in repack_config:\n        repack_configs.extend(glob.glob(repack_config))\n      else:\n        # This could go through glob but then we'd swallow errors for\n        # non existing files.\n        repack_configs.append(repack_config)\n\n    MultiTemplateRepacker().RepackTemplates(\n        repack_configs,\n        templates,\n        args.output_dir,\n        config=FLAGS.config,\n        sign=args.sign,\n        signed_template=args.signed_template)\n  elif args.subparser_name == \"sign_template\":\n    repacking.TemplateRepacker().SignTemplate(\n        args.template, args.output_file, context=context)\n    if not os.path.exists(args.output_file):\n      raise RuntimeError(\"Signing failed: output not written\")", "response": "Launch the appropriate builder."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ngets instance of builder class based on flags.", "response": "def GetBuilder(self, context, fleetspeak_service_config):\n    \"\"\"Get instance of builder class based on flags.\"\"\"\n    try:\n      if \"Target:Darwin\" in context:\n        return builders.DarwinClientBuilder(\n            context=context,\n            fleetspeak_service_config=fleetspeak_service_config)\n      elif \"Target:Windows\" in context:\n        return builders.WindowsClientBuilder(context=context)\n      elif \"Target:LinuxDeb\" in context:\n        return builders.LinuxClientBuilder(context=context)\n      elif \"Target:LinuxRpm\" in context:\n        return builders.CentosClientBuilder(context=context)\n      else:\n        parser.error(\"Bad build context: %s\" % context)\n    except AttributeError:\n      raise RuntimeError(\"Unable to build for platform %s when running \"\n                         \"on current platform.\" % self.platform)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef BuildTemplate(self,\n                    context=None,\n                    output=None,\n                    fleetspeak_service_config=None):\n    \"\"\"Find template builder and call it.\"\"\"\n    context = context or []\n    context.append(\"Arch:%s\" % self.GetArch())\n    # Platform context has common platform settings, Target has template build\n    # specific stuff.\n    self.platform = platform.system()\n    context.append(\"Platform:%s\" % self.platform)\n    context.append(\"Target:%s\" % self.platform)\n    if \"Target:Linux\" in context:\n      context.append(self.GetPackageFormat())\n\n    template_path = None\n    if output:\n      template_path = os.path.join(\n          output,\n          grr_config.CONFIG.Get(\n              \"PyInstaller.template_filename\", context=context))\n\n    builder_obj = self.GetBuilder(context, fleetspeak_service_config)\n    builder_obj.MakeExecutableTemplate(output_file=template_path)", "response": "Build a template file."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nadd the repack config filename onto the base output directory.", "response": "def GetOutputDir(self, base_dir, config_filename):\n    \"\"\"Add the repack config filename onto the base output directory.\n\n    This allows us to repack lots of different configs to the same installer\n    name and still be able to distinguish them.\n\n    Args:\n      base_dir: output directory string\n      config_filename: the secondary config filename string\n\n    Returns:\n      String to be used as output directory for this repack.\n    \"\"\"\n    return os.path.join(base_dir,\n                        os.path.basename(config_filename.replace(\".yaml\", \"\")))"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef RepackTemplates(self,\n                      repack_configs,\n                      templates,\n                      output_dir,\n                      config=None,\n                      sign=False,\n                      signed_template=False):\n    \"\"\"Call repacker in a subprocess.\"\"\"\n    pool = multiprocessing.Pool(processes=10)\n    results = []\n    bulk_sign_installers = False\n    for repack_config in repack_configs:\n      for template in templates:\n        repack_args = [\"grr_client_build\"]\n        if config:\n          repack_args.extend([\"--config\", config])\n\n        repack_args.extend([\n            \"--secondary_configs\", repack_config, \"repack\", \"--template\",\n            template, \"--output_dir\",\n            self.GetOutputDir(output_dir, repack_config)\n        ])\n\n        # We only sign exes and rpms at the moment. The others will raise if we\n        # try to ask for signing.\n        passwd = None\n\n        if sign:\n          if template.endswith(\".exe.zip\"):\n            # This is for osslsigncode only.\n            if platform.system() != \"Windows\":\n              passwd = self.GetWindowsPassphrase()\n              repack_args.append(\"--sign\")\n            else:\n              bulk_sign_installers = True\n            if signed_template:\n              repack_args.append(\"--signed_template\")\n          elif template.endswith(\".rpm.zip\"):\n            bulk_sign_installers = True\n\n        print(\"Calling %s\" % \" \".join(repack_args))\n        results.append(\n            pool.apply_async(SpawnProcess, (repack_args,), dict(passwd=passwd)))\n\n        # Also build debug if it's windows.\n        if template.endswith(\".exe.zip\"):\n          debug_args = []\n          debug_args.extend(repack_args)\n          debug_args.append(\"--debug_build\")\n          print(\"Calling %s\" % \" \".join(debug_args))\n          results.append(\n              pool.apply_async(SpawnProcess, (debug_args,),\n                               dict(passwd=passwd)))\n\n    try:\n      pool.close()\n      # Workaround to handle keyboard kills\n      # http://stackoverflow.com/questions/1408356/keyboard-interrupts-with-pythons-multiprocessing-pool\n      # get will raise if the child raises.\n      for result_obj in results:\n        result_obj.get(9999)\n      pool.join()\n    except KeyboardInterrupt:\n      print(\"parent received control-c\")\n      pool.terminate()\n    except ErrorDuringRepacking:\n      pool.terminate()\n      raise\n\n    if bulk_sign_installers:\n      to_sign = {}\n      for root, _, files in os.walk(output_dir):\n        for f in files:\n          if f.endswith(\".exe\"):\n            to_sign.setdefault(\"windows\", []).append(os.path.join(root, f))\n          elif f.endswith(\".rpm\"):\n            to_sign.setdefault(\"rpm\", []).append(os.path.join(root, f))\n      if to_sign.get(\"windows\"):\n        signer = repacking.TemplateRepacker().GetSigner([\n            \"ClientBuilder Context\",\n            \"Platform:%s\" % platform.system(), \"Target:Windows\"\n        ])\n        signer.SignFiles(to_sign.get(\"windows\"))\n      if to_sign.get(\"rpm\"):\n        signer = repacking.TemplateRepacker().GetSigner([\n            \"ClientBuilder Context\",\n            \"Platform:%s\" % platform.system(), \"Target:Linux\", \"Target:LinuxRpm\"\n        ])\n        signer.AddSignatureToRPMs(to_sign.get(\"rpm\"))", "response": "Repacks the specified templates in a list of repacker configurations."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ninitializing all startup routines for the client.", "response": "def ClientInit():\n  \"\"\"Run all startup routines for the client.\"\"\"\n  metric_metadata = client_metrics.GetMetadata()\n  metric_metadata.extend(communicator.GetMetricMetadata())\n  stats_collector_instance.Set(\n      default_stats_collector.DefaultStatsCollector(metric_metadata))\n\n  config_lib.SetPlatformArchContext()\n  config_lib.ParseConfigCommandLine()\n\n  client_logging.LogInit()\n  all_parsers.Register()\n  registry.Init()\n\n  if not config.CONFIG.ContextApplied(contexts.CLIENT_BUILD_CONTEXT):\n    config.CONFIG.Persist(\"Client.labels\")\n    config.CONFIG.Persist(\"Client.proxy_servers\")\n    config.CONFIG.Persist(\"Client.tempdir_roots\")"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef Parse(self, cmd, args, stdout, stderr, return_val, time_taken,\n            knowledge_base):\n    \"\"\"Parse the sysctl output.\"\"\"\n    _ = stderr, time_taken, args, knowledge_base  # Unused.\n    self.CheckReturn(cmd, return_val)\n    result = rdf_protodict.AttributedDict()\n    # The KeyValueParser generates an ordered dict by default. The sysctl vals\n    # aren't ordering dependent, but there's no need to un-order it.\n    for k, v in iteritems(self.lexer.ParseToOrderedDict(stdout)):\n      key = k.replace(\".\", \"_\")\n      if len(v) == 1:\n        v = v[0]\n      result[key] = v\n    return [result]", "response": "Parse the sysctl output."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nparse the key currentcontrolset output.", "response": "def Parse(self, stat, unused_knowledge_base):\n    \"\"\"Parse the key currentcontrolset output.\"\"\"\n    value = stat.registry_data.GetValue()\n\n    if not str(value).isdigit() or int(value) > 999 or int(value) < 0:\n      raise parser.ParseError(\n          \"Invalid value for CurrentControlSet key %s\" % value)\n    yield rdfvalue.RDFString(\n        \"HKEY_LOCAL_MACHINE\\\\SYSTEM\\\\ControlSet%03d\" % int(value))"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef Parse(self, stat, knowledge_base):\n    value = stat.registry_data.GetValue()\n    if not value:\n      raise parser.ParseError(\"Invalid value for key %s\" % stat.pathspec.path)\n    value = artifact_utils.ExpandWindowsEnvironmentVariables(\n        value, knowledge_base)\n    if value:\n      yield rdfvalue.RDFString(value)", "response": "Parse the value of the stat."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef Parse(self, stat, _):\n    if isinstance(stat, rdf_client_fs.StatEntry):\n      value = stat.registry_data.GetValue()\n    elif isinstance(stat, rdfvalue.RDFString):\n      value = stat\n    if not value:\n      raise parser.ParseError(\"Invalid value for key %s\" % stat.pathspec.path)\n\n    systemdrive = value[0:2]\n    if re.match(r\"^[A-Za-z]:$\", systemdrive):\n      yield rdfvalue.RDFString(systemdrive)\n    else:\n      raise parser.ParseError(\n          \"Bad drive letter for key %s\" % stat.pathspec.path)", "response": "Parses a SystemDrive environment variable."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef Parse(self, stat, knowledge_base):\n    _ = knowledge_base  # Unused.\n    sid_str = stat.pathspec.Dirname().Basename()\n\n    if SID_RE.match(sid_str):\n      kb_user = rdf_client.User()\n      kb_user.sid = sid_str\n      if stat.pathspec.Basename() == \"ProfileImagePath\":\n        if stat.resident:\n          # Support old clients.\n          kb_user.homedir = utils.SmartUnicode(stat.resident)\n        else:\n          kb_user.homedir = stat.registry_data.GetValue()\n\n        kb_user.userprofile = kb_user.homedir\n        try:\n          # Assume username is the last component of the path. This is not\n          # robust, but other user artifacts will override it if there is a\n          # better match.\n          kb_user.username = kb_user.homedir.rsplit(\"\\\\\", 1)[-1]\n        except IndexError:\n          pass\n\n      yield kb_user", "response": "Parse each returned registry value."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nparse each returned registry value.", "response": "def ParseMultiple(self, stats, knowledge_base):\n    \"\"\"Parse each returned registry value.\"\"\"\n    user_dict = {}\n\n    for stat in stats:\n      sid_str = stat.pathspec.path.split(\"/\", 3)[2]\n      if SID_RE.match(sid_str):\n        if sid_str not in user_dict:\n          user_dict[sid_str] = rdf_client.User(sid=sid_str)\n\n        if stat.registry_data.GetValue():\n          # Look up in the mapping if we can use this entry to populate a user\n          # attribute, and if so, set it.\n          reg_key_name = stat.pathspec.Dirname().Basename()\n          if reg_key_name in self.key_var_mapping:\n            map_dict = self.key_var_mapping[reg_key_name]\n            reg_key = stat.pathspec.Basename()\n            kb_attr = map_dict.get(reg_key)\n            if kb_attr:\n              value = artifact_utils.ExpandWindowsEnvironmentVariables(\n                  stat.registry_data.GetValue(), knowledge_base)\n              value = artifact_utils.ExpandWindowsUserEnvironmentVariables(\n                  value, knowledge_base, sid=sid_str)\n              user_dict[sid_str].Set(kb_attr, value)\n\n    # Now yield each user we found.\n    return itervalues(user_dict)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef ParseMultiple(self, stats, knowledge_base):\n    _ = knowledge_base\n    services = {}\n    field_map = {\n        \"Description\": \"description\",\n        \"DisplayName\": \"display_name\",\n        \"Group\": \"group_name\",\n        \"DriverPackageId\": \"driver_package_id\",\n        \"ErrorControl\": \"error_control\",\n        \"ImagePath\": \"image_path\",\n        \"ObjectName\": \"object_name\",\n        \"Start\": \"startup_type\",\n        \"Type\": \"service_type\",\n        \"Parameters/ServiceDLL\": \"service_dll\"\n    }\n\n    # Field map key should be converted to lowercase because key aquired through\n    # self._GetKeyName could have some  characters in different case than the\n    # field map, e.g. ServiceDLL and ServiceDll.\n    field_map = {k.lower(): v for k, v in iteritems(field_map)}\n    for stat in stats:\n\n      # Ignore subkeys\n      if not stat.HasField(\"registry_data\"):\n        continue\n\n      service_name = self._GetServiceName(stat.pathspec.path)\n      reg_key = os.path.dirname(stat.pathspec.path)\n      service_info = rdf_client.WindowsServiceInformation(\n          name=service_name, registry_key=reg_key)\n      services.setdefault(service_name, service_info)\n\n      key = self._GetKeyName(stat.pathspec.path)\n\n      if key in field_map:\n        try:\n          services[service_name].Set(field_map[key],\n                                     stat.registry_data.GetValue())\n        except type_info.TypeValueError:\n\n          # Flatten multi strings into a simple string\n          if (stat.registry_type ==\n              rdf_client_fs.StatEntry.RegistryType.REG_MULTI_SZ):\n            services[service_name].Set(\n                field_map[key],\n                utils.SmartUnicode(stat.registry_data.GetValue()))\n          else:\n            # Log failures for everything else\n            # TODO(user): change this to yield a ParserAnomaly object.\n            dest_type = type(services[service_name].Get(field_map[key]))\n            logging.debug(\"Wrong type set for %s:%s, expected %s, got %s\",\n                          stat.pathspec.path, stat.registry_data.GetValue(),\n                          dest_type, type(stat.registry_data.GetValue()))\n\n    return itervalues(services)", "response": "Parses the service registry keys and returns WindowsServiceInformation objects."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nconvert the timezone to Olson format.", "response": "def Parse(self, stat, knowledge_base):\n    \"\"\"Convert the timezone to Olson format.\"\"\"\n    _ = knowledge_base\n    value = stat.registry_data.GetValue()\n    result = ZONE_LIST.get(value.strip())\n    if not result:\n      yield rdfvalue.RDFString(\"Unknown (%s)\" % value.strip())\n\n    yield rdfvalue.RDFString(result)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nconvert a generator with approval rows into ApprovalRequest objects.", "response": "def _ResponseToApprovalsWithGrants(response):\n  \"\"\"Converts a generator with approval rows into ApprovalRequest objects.\"\"\"\n  prev_triplet = None\n  cur_approval_request = None\n  for (approval_id_int, approval_timestamp, approval_request_bytes,\n       grantor_username, grant_timestamp) in response:\n\n    cur_triplet = (approval_id_int, approval_timestamp, approval_request_bytes)\n\n    if cur_triplet != prev_triplet:\n      prev_triplet = cur_triplet\n\n      if cur_approval_request:\n        yield cur_approval_request\n\n      cur_approval_request = mysql_utils.StringToRDFProto(\n          rdf_objects.ApprovalRequest, approval_request_bytes)\n      cur_approval_request.approval_id = _IntToApprovalID(approval_id_int)\n      cur_approval_request.timestamp = mysql_utils.TimestampToRDFDatetime(\n          approval_timestamp)\n\n    if grantor_username and grant_timestamp:\n      cur_approval_request.grants.append(\n          rdf_objects.ApprovalGrant(\n              grantor_username=grantor_username,\n              timestamp=mysql_utils.TimestampToRDFDatetime(grant_timestamp)))\n\n  if cur_approval_request:\n    yield cur_approval_request"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nwrite a GRR user object for a given username.", "response": "def WriteGRRUser(self,\n                   username,\n                   password=None,\n                   ui_mode=None,\n                   canary_mode=None,\n                   user_type=None,\n                   cursor=None):\n    \"\"\"Writes user object for a user with a given name.\"\"\"\n\n    values = {\"username\": username, \"username_hash\": mysql_utils.Hash(username)}\n\n    if password is not None:\n      values[\"password\"] = password.SerializeToString()\n    if ui_mode is not None:\n      values[\"ui_mode\"] = int(ui_mode)\n    if canary_mode is not None:\n      # TODO(amoser): This int conversion is dirty but necessary with\n      # the current MySQL driver.\n      # TODO: We can remove this once the bug is fixed.\n      values[\"canary_mode\"] = int(bool(canary_mode))\n    if user_type is not None:\n      values[\"user_type\"] = int(user_type)\n\n    query = \"INSERT INTO grr_users {cols} VALUES {vals}\".format(\n        cols=mysql_utils.Columns(values),\n        vals=mysql_utils.NamedPlaceholders(values))\n\n    updates = \", \".join(\"{0} = VALUES({0})\".format(col) for col in values)\n    query += \" ON DUPLICATE KEY UPDATE \" + updates\n\n    cursor.execute(query, values)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nconverting a database result row into a GRR user object.", "response": "def _RowToGRRUser(self, row):\n    \"\"\"Creates a GRR user object from a database result row.\"\"\"\n    username, password, ui_mode, canary_mode, user_type = row\n    result = rdf_objects.GRRUser(\n        username=username,\n        ui_mode=ui_mode,\n        canary_mode=canary_mode,\n        user_type=user_type)\n\n    if password:\n      result.password.ParseFromString(password)\n\n    return result"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nread a GRR user object corresponding to a given name.", "response": "def ReadGRRUser(self, username, cursor=None):\n    \"\"\"Reads a user object corresponding to a given name.\"\"\"\n    cursor.execute(\n        \"SELECT username, password, ui_mode, canary_mode, user_type \"\n        \"FROM grr_users WHERE username_hash = %s\", [mysql_utils.Hash(username)])\n\n    row = cursor.fetchone()\n    if row is None:\n      raise db.UnknownGRRUserError(username)\n\n    return self._RowToGRRUser(row)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef ReadGRRUsers(self, offset=0, count=None, cursor=None):\n    if count is None:\n      count = 18446744073709551615  # 2^64-1, as suggested by MySQL docs\n\n    cursor.execute(\n        \"SELECT username, password, ui_mode, canary_mode, user_type \"\n        \"FROM grr_users ORDER BY username ASC \"\n        \"LIMIT %s OFFSET %s\", [count, offset])\n    return [self._RowToGRRUser(row) for row in cursor.fetchall()]", "response": "Reads GRR users with optional pagination sorted by username."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ndelete the user and all related metadata with the given username.", "response": "def DeleteGRRUser(self, username, cursor=None):\n    \"\"\"Deletes the user and all related metadata with the given username.\"\"\"\n    cursor.execute(\"DELETE FROM grr_users WHERE username_hash = %s\",\n                   (mysql_utils.Hash(username),))\n\n    if cursor.rowcount == 0:\n      raise db.UnknownGRRUserError(username)"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nwrites an approval request object to the database.", "response": "def WriteApprovalRequest(self, approval_request, cursor=None):\n    \"\"\"Writes an approval request object.\"\"\"\n    # Copy the approval_request to ensure we don't modify the source object.\n    approval_request = approval_request.Copy()\n    # Generate random approval id.\n    approval_id_int = random.UInt64()\n\n    grants = approval_request.grants\n    approval_request.grants = None\n\n    expiry_time = approval_request.expiration_time\n\n    args = {\n        \"username_hash\": mysql_utils.Hash(approval_request.requestor_username),\n        \"approval_type\": int(approval_request.approval_type),\n        \"subject_id\": approval_request.subject_id,\n        \"approval_id\": approval_id_int,\n        \"expiration_time\": mysql_utils.RDFDatetimeToTimestamp(expiry_time),\n        \"approval_request\": approval_request.SerializeToString()\n    }\n    query = \"\"\"\n    INSERT INTO approval_request (username_hash, approval_type,\n                                  subject_id, approval_id, expiration_time,\n                                  approval_request)\n    VALUES (%(username_hash)s, %(approval_type)s,\n            %(subject_id)s, %(approval_id)s, FROM_UNIXTIME(%(expiration_time)s),\n            %(approval_request)s)\n    \"\"\"\n    cursor.execute(query, args)\n\n    for grant in grants:\n      self._GrantApproval(approval_request.requestor_username, approval_id_int,\n                          grant.grantor_username, cursor)\n\n    return _IntToApprovalID(approval_id_int)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _GrantApproval(self, requestor_username, approval_id, grantor_username,\n                     cursor):\n    \"\"\"Grants approval for a given request.\"\"\"\n    grant_args = {\n        \"username_hash\": mysql_utils.Hash(requestor_username),\n        \"approval_id\": approval_id,\n        \"grantor_username_hash\": mysql_utils.Hash(grantor_username),\n    }\n    grant_query = (\n        \"INSERT INTO approval_grant {columns} VALUES {values}\".format(\n            columns=mysql_utils.Columns(grant_args),\n            values=mysql_utils.NamedPlaceholders(grant_args)))\n    cursor.execute(grant_query, grant_args)", "response": "Grant approval for a given request."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef GrantApproval(self,\n                    requestor_username,\n                    approval_id,\n                    grantor_username,\n                    cursor=None):\n    \"\"\"Grants approval for a given request using given username.\"\"\"\n    self._GrantApproval(\n        requestor_username, _ApprovalIDToInt(approval_id), grantor_username,\n        cursor)", "response": "Grants approval for a given request using given username."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nread an approval request object with a given id.", "response": "def ReadApprovalRequest(self, requestor_username, approval_id, cursor=None):\n    \"\"\"Reads an approval request object with a given id.\"\"\"\n\n    query = (\"\"\"\n        SELECT\n            ar.approval_id,\n            UNIX_TIMESTAMP(ar.timestamp),\n            ar.approval_request,\n            u.username,\n            UNIX_TIMESTAMP(ag.timestamp)\n        FROM approval_request ar\n        LEFT JOIN approval_grant ag USING (username_hash, approval_id)\n        LEFT JOIN grr_users u ON u.username_hash = ag.grantor_username_hash\n        WHERE ar.approval_id = %s AND ar.username_hash = %s\n        \"\"\")\n\n    cursor.execute(\n        query,\n        [_ApprovalIDToInt(approval_id),\n         mysql_utils.Hash(requestor_username)])\n    res = cursor.fetchall()\n    if not res:\n      raise db.UnknownApprovalRequestError(\"Approval '%s' not found.\" %\n                                           approval_id)\n\n    approval_id_int, timestamp, approval_request_bytes, _, _ = res[0]\n\n    approval_request = mysql_utils.StringToRDFProto(rdf_objects.ApprovalRequest,\n                                                    approval_request_bytes)\n    approval_request.approval_id = _IntToApprovalID(approval_id_int)\n    approval_request.timestamp = mysql_utils.TimestampToRDFDatetime(timestamp)\n\n    for _, _, _, grantor_username, timestamp in res:\n      if not grantor_username:\n        continue\n\n      # Note: serialized approval_request objects are guaranteed to not\n      # have any grants.\n      approval_request.grants.append(\n          rdf_objects.ApprovalGrant(\n              grantor_username=grantor_username,\n              timestamp=mysql_utils.TimestampToRDFDatetime(timestamp)))\n\n    return approval_request"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreads approval requests of a given type for a given user.", "response": "def ReadApprovalRequests(self,\n                           requestor_username,\n                           approval_type,\n                           subject_id=None,\n                           include_expired=False,\n                           cursor=None):\n    \"\"\"Reads approval requests of a given type for a given user.\"\"\"\n\n    query = \"\"\"\n        SELECT\n            ar.approval_id,\n            UNIX_TIMESTAMP(ar.timestamp),\n            ar.approval_request,\n            u.username,\n            UNIX_TIMESTAMP(ag.timestamp)\n        FROM approval_request ar\n        LEFT JOIN approval_grant AS ag USING (username_hash, approval_id)\n        LEFT JOIN grr_users u ON u.username_hash = ag.grantor_username_hash\n        WHERE ar.username_hash = %s AND ar.approval_type = %s\n        \"\"\"\n\n    args = [mysql_utils.Hash(requestor_username), int(approval_type)]\n\n    if subject_id:\n      query += \" AND ar.subject_id = %s\"\n      args.append(subject_id)\n\n    query += \" ORDER BY ar.approval_id\"\n\n    ret = []\n    now = rdfvalue.RDFDatetime.Now()\n    cursor.execute(query, args)\n    for approval_request in _ResponseToApprovalsWithGrants(cursor.fetchall()):\n      if include_expired or approval_request.expiration_time >= now:\n        ret.append(approval_request)\n    return ret"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nwriting a notification for a given user.", "response": "def WriteUserNotification(self, notification, cursor=None):\n    \"\"\"Writes a notification for a given user.\"\"\"\n    # Copy the notification to ensure we don't modify the source object.\n    args = {\n        \"username_hash\":\n            mysql_utils.Hash(notification.username),\n        \"notification_state\":\n            int(notification.state),\n        \"notification\":\n            notification.SerializeToString(),\n    }\n    query = \"INSERT INTO user_notification {columns} VALUES {values}\".format(\n        columns=mysql_utils.Columns(args),\n        values=mysql_utils.NamedPlaceholders(args))\n    try:\n      cursor.execute(query, args)\n    except MySQLdb.IntegrityError:\n      raise db.UnknownGRRUserError(notification.username)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef ReadUserNotifications(self,\n                            username,\n                            state=None,\n                            timerange=None,\n                            cursor=None):\n    \"\"\"Reads notifications scheduled for a user within a given timerange.\"\"\"\n\n    query = (\"SELECT UNIX_TIMESTAMP(timestamp), \"\n             \"       notification_state, notification \"\n             \"FROM user_notification \"\n             \"WHERE username_hash = %s \")\n    args = [mysql_utils.Hash(username)]\n\n    if state is not None:\n      query += \"AND notification_state = %s \"\n      args.append(int(state))\n\n    if timerange is not None:\n      time_from, time_to = timerange  # pylint: disable=unpacking-non-sequence\n\n      if time_from is not None:\n        query += \"AND timestamp >= FROM_UNIXTIME(%s) \"\n        args.append(mysql_utils.RDFDatetimeToTimestamp(time_from))\n\n      if time_to is not None:\n        query += \"AND timestamp <= FROM_UNIXTIME(%s) \"\n        args.append(mysql_utils.RDFDatetimeToTimestamp(time_to))\n\n    query += \"ORDER BY timestamp DESC \"\n\n    ret = []\n    cursor.execute(query, args)\n\n    for timestamp, state, notification_ser in cursor.fetchall():\n      n = rdf_objects.UserNotification.FromSerializedString(notification_ser)\n      n.timestamp = mysql_utils.TimestampToRDFDatetime(timestamp)\n      n.state = state\n      ret.append(n)\n\n    return ret", "response": "Reads notifications scheduled for a given user within a given timerange."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef UpdateUserNotifications(self,\n                              username,\n                              timestamps,\n                              state=None,\n                              cursor=None):\n    \"\"\"Updates existing user notification objects.\"\"\"\n\n    query = (\"UPDATE user_notification \"\n             \"SET notification_state = %s \"\n             \"WHERE username_hash = %s\"\n             \"AND UNIX_TIMESTAMP(timestamp) IN {}\").format(\n                 mysql_utils.Placeholders(len(timestamps)))\n\n    args = [\n        int(state),\n        mysql_utils.Hash(username),\n    ] + [mysql_utils.RDFDatetimeToTimestamp(t) for t in timestamps]\n    cursor.execute(query, args)", "response": "Updates existing user notifications."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef SplitIntoComponents(str_in):\n\n  if str_in.startswith((\"\\\"\", \"'\")):\n    return shlex.split(str_in)\n  else:\n    components = str_in.split(\" \", 1)\n    if len(components) > 1:\n      return [components[0]] + SplitIntoComponents(components[1])\n    else:\n      return components", "response": "Splits a string into space - separated components."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef Detect(self, str_in):\n\n    components = SplitIntoComponents(str_in)\n\n    extracted_paths = set()\n    for extractor in self.extractors:\n      extracted_paths.update(extractor.Extract(components))\n\n    results = set(extracted_paths)\n\n    for post_processor in self.post_processors:\n      processed_results = set()\n      for result in results:\n        processed_results.update(post_processor.Process(result))\n      results = processed_results\n\n    return results", "response": "Detects paths in a given string."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef Init():\n  VFS_HANDLERS.clear()\n  _VFS_VIRTUALROOTS.clear()\n  vfs_virtualroots = config.CONFIG[\"Client.vfs_virtualroots\"]\n\n  VFS_HANDLERS[files.File.supported_pathtype] = files.File\n  VFS_HANDLERS[files.TempFile.supported_pathtype] = files.TempFile\n  VFS_HANDLERS[sleuthkit.TSKFile.supported_pathtype] = sleuthkit.TSKFile\n  if vfs_registry is not None:\n    VFS_HANDLERS[vfs_registry.RegistryFile\n                 .supported_pathtype] = vfs_registry.RegistryFile\n\n  for vfs_virtualroot in vfs_virtualroots:\n    try:\n      handler_string, root = vfs_virtualroot.split(\":\", 1)\n    except ValueError:\n      raise ValueError(\n          \"Badly formatted vfs virtual root: %s. Correct format is \"\n          \"os:/path/to/virtual_root\" % vfs_virtualroot)\n\n    handler_string = handler_string.upper()\n    handler = rdf_paths.PathSpec.PathType.enum_dict.get(handler_string)\n    if handler is None:\n      raise ValueError(\n          \"VFSHandler {} could not be registered, because it was not found in\"\n          \" PathSpec.PathType {}\".format(handler_string,\n                                         rdf_paths.PathSpec.PathType.enum_dict))\n\n    # We need some translation here, TSK needs an OS virtual root base. For\n    # every other handler we can just keep the type the same.\n    if handler == rdf_paths.PathSpec.PathType.TSK:\n      base_type = rdf_paths.PathSpec.PathType.OS\n    else:\n      base_type = handler\n    _VFS_VIRTUALROOTS[handler] = rdf_paths.PathSpec(\n        path=root, pathtype=base_type, is_virtualroot=True)", "response": "Initialize all vfs handlers."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef VFSOpen(pathspec,\n            progress_callback = None\n           ):\n  \"\"\"Expands pathspec to return an expanded Path.\n\n  A pathspec is a specification of how to access the file by recursively opening\n  each part of the path by different drivers. For example the following\n  pathspec:\n\n  pathtype: OS\n  path: \"/dev/sda1\"\n  nested_path {\n    pathtype: TSK\n    path: \"/home/image2.img\"\n    nested_path {\n      pathtype: TSK\n      path: \"/home/a.txt\"\n    }\n  }\n\n  Instructs the system to:\n  1) open /dev/sda1 using the OS driver.\n  2) Pass the obtained filelike object to the TSK driver to open\n  \"/home/image2.img\".\n  3) The obtained filelike object should be passed to the TSK driver to open\n  \"/home/a.txt\".\n\n  The problem remains how to get to this expanded path specification. Since the\n  server is not aware of all the files on the client, the server may request\n  this:\n\n  pathtype: OS\n  path: \"/dev/sda1\"\n  nested_path {\n    pathtype: TSK\n    path: \"/home/image2.img/home/a.txt\"\n  }\n\n  Or even this:\n\n  pathtype: OS\n  path: \"/dev/sda1/home/image2.img/home/a.txt\"\n\n  This function converts the pathspec requested by the server into an expanded\n  pathspec required to actually open the file. This is done by expanding each\n  component of the pathspec in turn.\n\n  Expanding the component is done by opening each leading directory in turn and\n  checking if it is a directory of a file. If its a file, we examine the file\n  headers to determine the next appropriate driver to use, and create a nested\n  pathspec.\n\n  Note that for some clients there might be a virtual root specified. This\n  is a directory that gets prepended to all pathspecs of a given\n  pathtype. For example if there is a virtual root defined as\n  [\"os:/virtualroot\"], a path specification like\n\n  pathtype: OS\n  path: \"/home/user/*\"\n\n  will get translated into\n\n  pathtype: OS\n  path: \"/virtualroot\"\n  is_virtualroot: True\n  nested_path {\n    pathtype: OS\n    path: \"/dev/sda1\"\n  }\n\n  Args:\n    pathspec: A Path() protobuf to normalize.\n    progress_callback: A callback to indicate that the open call is still\n      working but needs more time.\n\n  Returns:\n    The open filelike object. This will contain the expanded Path() protobuf as\n    the member fd.pathspec.\n\n  Raises:\n    IOError: if one of the path components can not be opened.\n\n  \"\"\"\n  # Initialize the dictionary of VFS handlers lazily, if not yet done.\n  if not VFS_HANDLERS:\n    Init()\n\n  fd = None\n\n  # Adjust the pathspec in case we are using a vfs_virtualroot.\n  vroot = _VFS_VIRTUALROOTS.get(pathspec.pathtype)\n\n  # If we have a virtual root for this vfs handler, we need to prepend\n  # it to the incoming pathspec except if the pathspec is explicitly\n  # marked as containing a virtual root already or if it isn't marked but\n  # the path already contains the virtual root.\n  if (not vroot or pathspec.is_virtualroot or\n      pathspec.CollapsePath().startswith(vroot.CollapsePath())):\n    # No virtual root but opening changes the pathspec so we always work on a\n    # copy.\n    working_pathspec = pathspec.Copy()\n  else:\n    # We're in a virtual root, put the target pathspec inside the virtual root\n    # as a nested path.\n    working_pathspec = vroot.Copy()\n    working_pathspec.last.nested_path = pathspec.Copy()\n\n  # For each pathspec step, we get the handler for it and instantiate it with\n  # the old object, and the current step.\n  while working_pathspec:\n    component = working_pathspec.Pop()\n    try:\n      handler = VFS_HANDLERS[component.pathtype]\n    except KeyError:\n      raise UnsupportedHandlerError(component.pathtype)\n\n    # Open the component.\n    fd = handler.Open(\n        fd=fd,\n        component=component,\n        handlers=dict(VFS_HANDLERS),\n        pathspec=working_pathspec,\n        progress_callback=progress_callback)\n\n  if fd is None:\n    raise ValueError(\"VFSOpen cannot be called with empty PathSpec.\")\n\n  return fd", "response": "This function is used to create a pathspec that is used by the server to open the file."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nopening multiple files specified by given path - specs.", "response": "def VFSMultiOpen(pathspecs, progress_callback=None):\n  \"\"\"Opens multiple files specified by given path-specs.\n\n  See documentation for `VFSOpen` for more information.\n\n  Args:\n    pathspecs: A list of pathspec instances of files to open.\n    progress_callback: A callback function to call to notify about progress\n\n  Returns:\n    A context manager yielding file-like objects.\n  \"\"\"\n  precondition.AssertIterableType(pathspecs, rdf_paths.PathSpec)\n\n  vfs_open = functools.partial(VFSOpen, progress_callback=progress_callback)\n  return context.MultiContext(map(vfs_open, pathspecs))"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nread from the VFS and returns the contents.", "response": "def ReadVFS(pathspec, offset, length, progress_callback=None):\n  \"\"\"Read from the VFS and return the contents.\n\n  Args:\n    pathspec: path to read from\n    offset: number of bytes to skip\n    length: number of bytes to read\n    progress_callback: A callback to indicate that the open call is still\n      working but needs more time.\n\n  Returns:\n    VFS file contents\n  \"\"\"\n  fd = VFSOpen(pathspec, progress_callback=progress_callback)\n  fd.Seek(offset)\n  return fd.Read(length)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef WriteRow(self, values):\n    precondition.AssertIterableType(values, text)\n\n    if compatibility.PY2:\n      self._csv.writerow([value.encode(\"utf-8\") for value in values])\n    else:\n      self._csv.writerow(values)", "response": "Writes a single row to the underlying buffer."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef WriteRow(self, values):\n    precondition.AssertDictType(values, text, text)\n\n    row = []\n    for column in self._columns:\n      try:\n        value = values[column]\n      except KeyError:\n        raise ValueError(\"Row does not contain required column `%s`\" % column)\n\n      row.append(value)\n\n    self._writer.WriteRow(row)", "response": "Writes a single row to the underlying buffer."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn True if the last run failed.", "response": "def _IsCronJobFailing(self, cron_job):\n    \"\"\"Returns True if the last run failed.\"\"\"\n    status = cron_job.Get(cron_job.Schema.LAST_RUN_STATUS)\n    if status is None:\n      return False\n\n    return status.status != rdf_cronjobs.CronJobRunStatus.Status.OK"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ninitializing the object from an ApiFlow object.", "response": "def InitFromApiFlow(self, f, cron_job_id=None):\n    \"\"\"Shortcut method for easy legacy cron jobs support.\"\"\"\n    if f.flow_id:\n      self.run_id = f.flow_id\n    elif f.urn:\n      self.run_id = f.urn.Basename()\n    self.started_at = f.started_at\n    self.cron_job_id = cron_job_id\n\n    flow_state_enum = api_plugins_flow.ApiFlow.State\n    cron_enum = rdf_cronjobs.CronJobRun.CronJobRunStatus\n    errors_map = {\n        flow_state_enum.RUNNING: cron_enum.RUNNING,\n        flow_state_enum.TERMINATED: cron_enum.FINISHED,\n        flow_state_enum.ERROR: cron_enum.ERROR,\n        flow_state_enum.CLIENT_CRASHED: cron_enum.ERROR\n    }\n    self.status = errors_map[f.state]\n\n    if f.state != f.State.RUNNING:\n      self.finished_at = f.last_active_at\n\n    if f.context.kill_timestamp:\n      self.status = self.Status.LIFETIME_EXCEEDED\n\n    if f.context.HasField(\"status\"):\n      self.log_message = f.context.status\n\n    if f.context.HasField(\"backtrace\"):\n      self.backtrace = f.context.backtrace\n\n    return self"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nparse a timerange argument and always returns non - None timerange.", "response": "def _ParseTimeRange(self, timerange):\n    \"\"\"Parses a timerange argument and always returns non-None timerange.\"\"\"\n    if timerange is None:\n      timerange = (None, None)\n\n    from_time, to_time = timerange\n    if not from_time:\n      from_time = rdfvalue.RDFDatetime().FromSecondsSinceEpoch(0)\n\n    if not to_time:\n      to_time = rdfvalue.RDFDatetime().FromSecondsSinceEpoch(sys.maxsize)\n\n    return (from_time, to_time)"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncreates an object copy by serializing and deserializing it.", "response": "def _DeepCopy(self, obj):\n    \"\"\"Creates an object copy by serializing/deserializing it.\n\n    RDFStruct.Copy() doesn't deep-copy repeated fields which may lead to\n    hard to catch bugs.\n\n    Args:\n      obj: RDFValue to be copied.\n\n    Returns:\n      A deep copy of the passed RDFValue.\n    \"\"\"\n    precondition.AssertType(obj, rdfvalue.RDFValue)\n\n    return obj.__class__.FromSerializedString(obj.SerializeToString())"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef ReadAPIAuditEntries(self,\n                          username=None,\n                          router_method_names=None,\n                          min_timestamp=None,\n                          max_timestamp=None):\n    \"\"\"Returns audit entries stored in the database.\"\"\"\n    results = []\n\n    for entry in self.api_audit_entries:\n      if username is not None and entry.username != username:\n        continue\n\n      if (router_method_names and\n          entry.router_method_name not in router_method_names):\n        continue\n\n      if min_timestamp is not None and entry.timestamp < min_timestamp:\n        continue\n\n      if max_timestamp is not None and entry.timestamp > max_timestamp:\n        continue\n\n      results.append(entry)\n\n    return sorted(results, key=lambda entry: entry.timestamp)", "response": "Reads the audit entries stored in the database."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef CountAPIAuditEntriesByUserAndDay(self,\n                                       min_timestamp=None,\n                                       max_timestamp=None):\n    \"\"\"Returns audit entry counts grouped by user and calendar day.\"\"\"\n    results = collections.Counter()\n    for entry in self.api_audit_entries:\n      if min_timestamp is not None and entry.timestamp < min_timestamp:\n        continue\n\n      if max_timestamp is not None and entry.timestamp > max_timestamp:\n        continue\n\n      # Truncate DateTime by removing the time-part to allow grouping by date.\n      day = rdfvalue.RDFDatetime.FromDate(entry.timestamp.AsDatetime().date())\n      results[(entry.username, day)] += 1\n\n    return dict(results)", "response": "Returns a dictionary of API audit entries grouped by user and calendar day."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef WriteAPIAuditEntry(self, entry):\n    copy = entry.Copy()\n    copy.timestamp = rdfvalue.RDFDatetime.Now()\n    self.api_audit_entries.append(copy)", "response": "Writes an API audit entry to the database."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\napplies instant output plugin to a multi - type collection.", "response": "def ApplyPluginToMultiTypeCollection(plugin, output_collection,\n                                     source_urn=None):\n  \"\"\"Applies instant output plugin to a multi-type collection.\n\n  Args:\n    plugin: InstantOutputPlugin instance.\n    output_collection: MultiTypeCollection instance.\n    source_urn: If not None, override source_urn for collection items. This has\n      to be used when exporting flow results - their GrrMessages don't have\n      \"source\" attribute set.\n\n  Yields:\n    Bytes chunks, as generated by the plugin.\n  \"\"\"\n  for chunk in plugin.Start():\n    yield chunk\n\n  for stored_type_name in sorted(output_collection.ListStoredTypes()):\n    stored_cls = rdfvalue.RDFValue.classes[stored_type_name]\n\n    # pylint: disable=cell-var-from-loop\n    def GetValues():\n      for timestamp, value in output_collection.ScanByType(stored_type_name):\n        _ = timestamp\n        if source_urn:\n          value.source = source_urn\n        yield value\n\n    # pylint: enable=cell-var-from-loop\n\n    for chunk in plugin.ProcessValues(stored_cls, GetValues):\n      yield chunk\n\n  for chunk in plugin.Finish():\n    yield chunk"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\napplies instant output plugin to a collection of results.", "response": "def ApplyPluginToTypedCollection(plugin, type_names, fetch_fn):\n  \"\"\"Applies instant output plugin to a collection of results.\n\n  Args:\n    plugin: InstantOutputPlugin instance.\n    type_names: List of type names (strings) to be processed.\n    fetch_fn: Function that takes a type name as an argument and returns\n      available items (FlowResult) corresponding to this type. Items are\n      returned as a generator\n\n  Yields:\n    Bytes chunks, as generated by the plugin.\n  \"\"\"\n  for chunk in plugin.Start():\n    yield chunk\n\n  def GetValues(tn):\n    for v in fetch_fn(tn):\n      yield v\n\n  for type_name in sorted(type_names):\n    stored_cls = rdfvalue.RDFValue.classes[type_name]\n\n    for chunk in plugin.ProcessValues(stored_cls,\n                                      functools.partial(GetValues, type_name)):\n      yield chunk\n\n  for chunk in plugin.Finish():\n    yield chunk"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef output_file_name(self):\n\n    safe_path = re.sub(r\":|/\", \"_\", self.source_urn.Path().lstrip(\"/\"))\n    return \"results_%s%s\" % (safe_path, self.output_file_extension)", "response": "Name of the file where plugin s output should be written to."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nfetch metadata for a given list of clients.", "response": "def _GetMetadataForClients(self, client_urns):\n    \"\"\"Fetches metadata for a given list of clients.\"\"\"\n\n    result = {}\n    metadata_to_fetch = set()\n\n    for urn in client_urns:\n      try:\n        result[urn] = self._cached_metadata[urn]\n      except KeyError:\n        metadata_to_fetch.add(urn)\n\n    if metadata_to_fetch:\n      if data_store.RelationalDBEnabled():\n        client_ids = set(urn.Basename() for urn in metadata_to_fetch)\n        infos = data_store.REL_DB.MultiReadClientFullInfo(client_ids)\n\n        fetched_metadata = [\n            export.GetMetadata(client_id, info)\n            for client_id, info in infos.items()\n        ]\n      else:\n        client_fds = aff4.FACTORY.MultiOpen(\n            metadata_to_fetch, mode=\"r\", token=self.token)\n\n        fetched_metadata = [\n            export.GetMetadataLegacy(client_fd, token=self.token)\n            for client_fd in client_fds\n        ]\n\n      for metadata in fetched_metadata:\n        metadata.source_urn = self.source_urn\n\n        self._cached_metadata[metadata.client_urn] = metadata\n        result[metadata.client_urn] = metadata\n        metadata_to_fetch.remove(metadata.client_urn)\n\n      for urn in metadata_to_fetch:\n        default_mdata = export.ExportedMetadata(source_urn=self.source_urn)\n        result[urn] = default_mdata\n        self._cached_metadata[urn] = default_mdata\n\n    return [result[urn] for urn in client_urns]"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _GenerateSingleTypeIteration(self, next_types, processed_types,\n                                   converted_responses):\n    \"\"\"Yields responses of a given type only.\n\n    _GenerateSingleTypeIteration iterates through converted_responses and\n    only yields responses of the same type. The type is either popped from\n    next_types or inferred from the first item of converted_responses.\n    The type is added to a set of processed_types.\n\n    Along the way _GenerateSingleTypeIteration updates next_types set.\n    All newly encountered and not previously processed types are added to\n    next_types set.\n\n    Calling _GenerateSingleTypeIteration multiple times allows doing\n    multiple passes on converted responses and emitting converted responses\n    of the same type continuously (so that they can be written into\n    the same file by the plugin).\n\n    Args:\n      next_types: List of value type classes that will be used in further\n        iterations.\n      processed_types: List of value type classes that have been used already.\n      converted_responses: Iterable with values to iterate over.\n\n    Yields:\n      Values from converted_response with the same type. Type is either\n      popped from the next_types set or inferred from the first\n      converted_responses value.\n    \"\"\"\n    if not next_types:\n      current_type = None\n    else:\n      current_type = next_types.pop()\n      processed_types.add(current_type)\n\n    for converted_response in converted_responses:\n      if not current_type:\n        current_type = converted_response.__class__\n        processed_types.add(current_type)\n\n      if converted_response.__class__ != current_type:\n        if converted_response.__class__ not in processed_types:\n          next_types.add(converted_response.__class__)\n        continue\n\n      yield converted_response", "response": "Generates a single type iteration."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ngenerating converted values from given messages.", "response": "def _GenerateConvertedValues(self, converter, grr_messages):\n    \"\"\"Generates converted values using given converter from given messages.\n\n    Groups values in batches of BATCH_SIZE size and applies the converter\n    to each batch.\n\n    Args:\n      converter: ExportConverter instance.\n      grr_messages: An iterable (a generator is assumed) with GRRMessage values.\n\n    Yields:\n      Values generated by the converter.\n\n    Raises:\n      ValueError: if any of the GrrMessage objects doesn't have \"source\" set.\n    \"\"\"\n    for batch in collection.Batch(grr_messages, self.BATCH_SIZE):\n      metadata_items = self._GetMetadataForClients([gm.source for gm in batch])\n      batch_with_metadata = zip(metadata_items, [gm.payload for gm in batch])\n\n      for result in converter.BatchConvert(\n          batch_with_metadata, token=self.token):\n        yield result"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nparsing a categorized path into type and list of components.", "response": "def ParseCategorizedPath(path):\n  \"\"\"Parses a categorized path string into type and list of components.\"\"\"\n  components = tuple(component for component in path.split(\"/\") if component)\n  if components[0:2] == (\"fs\", \"os\"):\n    return PathInfo.PathType.OS, components[2:]\n  elif components[0:2] == (\"fs\", \"tsk\"):\n    return PathInfo.PathType.TSK, components[2:]\n  elif components[0:1] == (\"registry\",):\n    return PathInfo.PathType.REGISTRY, components[1:]\n  elif components[0:1] == (\"temp\",):\n    return PathInfo.PathType.TEMP, components[1:]\n  else:\n    raise ValueError(\"Incorrect path: '%s'\" % path)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef ToCategorizedPath(path_type, components):\n  try:\n    prefix = {\n        PathInfo.PathType.OS: (\"fs\", \"os\"),\n        PathInfo.PathType.TSK: (\"fs\", \"tsk\"),\n        PathInfo.PathType.REGISTRY: (\"registry\",),\n        PathInfo.PathType.TEMP: (\"temp\",),\n    }[path_type]\n  except KeyError:\n    raise ValueError(\"Unknown path type: `%s`\" % path_type)\n\n  return \"/\".join(itertools.chain(prefix, components))", "response": "Translates a path type and a list of components to a categorized path."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef GetGRRVersionString(self):\n    client_info = self.startup_info.client_info\n    client_name = client_info.client_description or client_info.client_name\n    if client_info.client_version > 0:\n      client_version = str(client_info.client_version)\n    else:\n      client_version = _UNKNOWN_GRR_VERSION\n    return \" \".join([client_name, client_version])", "response": "Retrieves the GRR version as a string."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef GetMacAddresses(self):\n    result = set()\n    for interface in self.interfaces:\n      if (interface.mac_address and\n          interface.mac_address != b\"\\x00\" * len(interface.mac_address)):\n        result.add(Text(interface.mac_address.human_readable_address))\n    return sorted(result)", "response": "Returns a list of MAC addresses from all interfaces."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a list of IP addresses from all interfaces.", "response": "def GetIPAddresses(self):\n    \"\"\"IP addresses from all interfaces.\"\"\"\n    result = []\n    filtered_ips = [\"127.0.0.1\", \"::1\", \"fe80::1\"]\n\n    for interface in self.interfaces:\n      for address in interface.addresses:\n        if address.human_readable_address not in filtered_ips:\n          result.append(Text(address.human_readable_address))\n    return sorted(result)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ngetting a client summary object.", "response": "def GetSummary(self):\n    \"\"\"Gets a client summary object.\n\n    Returns:\n      rdf_client.ClientSummary\n    Raises:\n      ValueError: on bad cloud type\n    \"\"\"\n    summary = rdf_client.ClientSummary()\n    summary.client_id = self.client_id\n    summary.timestamp = self.timestamp\n\n    summary.system_info.release = self.os_release\n    summary.system_info.version = str(self.os_version or \"\")\n    summary.system_info.kernel = self.kernel\n    summary.system_info.machine = self.arch\n    summary.system_info.install_date = self.install_time\n    kb = self.knowledge_base\n    if kb:\n      summary.system_info.fqdn = kb.fqdn\n      summary.system_info.system = kb.os\n      summary.users = kb.users\n      summary.interfaces = self.interfaces\n      summary.client_info = self.startup_info.client_info\n      if kb.os_release:\n        summary.system_info.release = kb.os_release\n        if kb.os_major_version:\n          summary.system_info.version = \"%d.%d\" % (kb.os_major_version,\n                                                   kb.os_minor_version)\n\n    hwi = self.hardware_info\n    if hwi:\n      summary.serial_number = hwi.serial_number\n      summary.system_manufacturer = hwi.system_manufacturer\n      summary.system_uuid = hwi.system_uuid\n\n    cloud_instance = self.cloud_instance\n    if cloud_instance:\n      summary.cloud_type = cloud_instance.cloud_type\n      if cloud_instance.cloud_type == \"GOOGLE\":\n        summary.cloud_instance_id = cloud_instance.google.unique_id\n      elif cloud_instance.cloud_type == \"AMAZON\":\n        summary.cloud_instance_id = cloud_instance.amazon.instance_id\n      else:\n        raise ValueError(\"Bad cloud type: %s\" % cloud_instance.cloud_type)\n    return summary"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nconstruct a path info corresponding to the parent of current path.", "response": "def GetParent(self):\n    \"\"\"Constructs a path info corresponding to the parent of current path.\n\n    The root path (represented by an empty list of components, corresponds to\n    `/` on Unix-like systems) does not have a parent.\n\n    Returns:\n      Instance of `rdf_objects.PathInfo` or `None` if parent does not exist.\n    \"\"\"\n    if self.root:\n      return None\n\n    return PathInfo(\n        components=self.components[:-1],\n        path_type=self.path_type,\n        directory=True)"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nyields all ancestors of a path.", "response": "def GetAncestors(self):\n    \"\"\"Yields all ancestors of a path.\n\n    The ancestors are returned in order from closest to the farthest one.\n\n    Yields:\n      Instances of `rdf_objects.PathInfo`.\n    \"\"\"\n    current = self\n    while True:\n      current = current.GetParent()\n      if current is None:\n        return\n      yield current"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef UpdateFrom(self, src):\n    if not isinstance(src, PathInfo):\n      raise TypeError(\"expected `%s` but got `%s`\" % (PathInfo, type(src)))\n    if self.path_type != src.path_type:\n      raise ValueError(\n          \"src [%s] does not represent the same path type as self [%s]\" %\n          (src.path_type, self.path_type))\n    if self.components != src.components:\n      raise ValueError(\"src [%s] does not represent the same path as self [%s]\"\n                       % (src.components, self.components))\n\n    if src.HasField(\"stat_entry\"):\n      self.stat_entry = src.stat_entry\n\n    self.last_stat_entry_timestamp = max(self.last_stat_entry_timestamp,\n                                         src.last_stat_entry_timestamp)\n    self.directory = self.directory or src.directory", "response": "Updates the internal state of this object from src."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nconvert a reference into an URN.", "response": "def ToURN(self):\n    \"\"\"Converts a reference into an URN.\"\"\"\n\n    if self.path_type in [PathInfo.PathType.OS, PathInfo.PathType.TSK]:\n      return rdfvalue.RDFURN(self.client_id).Add(\"fs\").Add(\n          self.path_type.name.lower()).Add(\"/\".join(self.path_components))\n    elif self.path_type == PathInfo.PathType.REGISTRY:\n      return rdfvalue.RDFURN(self.client_id).Add(\"registry\").Add(\"/\".join(\n          self.path_components))\n    elif self.path_type == PathInfo.PathType.TEMP:\n      return rdfvalue.RDFURN(self.client_id).Add(\"temp\").Add(\"/\".join(\n          self.path_components))\n\n    raise ValueError(\"Unsupported path type: %s\" % self.path_type)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef ToPath(self):\n\n    if self.path_type == PathInfo.PathType.OS:\n      return os.path.join(\"fs\", \"os\", *self.path_components)\n    elif self.path_type == PathInfo.PathType.TSK:\n      return os.path.join(\"fs\", \"tsk\", *self.path_components)\n    elif self.path_type == PathInfo.PathType.REGISTRY:\n      return os.path.join(\"registry\", *self.path_components)\n    elif self.path_type == PathInfo.PathType.TEMP:\n      return os.path.join(\"temp\", *self.path_components)\n\n    raise ValueError(\"Unsupported path type: %s\" % self.path_type)", "response": "Converts a reference into a VFS file path."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef LogInit():\n  logging.debug(\"Initializing client logging subsystem.\")\n\n  # The root logger.\n  logger = logging.getLogger()\n  memory_handlers = [\n      m for m in logger.handlers\n      if m.__class__.__name__ == \"PreLoggingMemoryHandler\"\n  ]\n\n  # Clear all handers.\n  logger.handlers = list(GetLogHandlers())\n  SetLogLevels()\n\n  # Now flush the old messages into the log files.\n  for handler in memory_handlers:\n    for record in handler.buffer:\n      logger.handle(record)", "response": "Configure the logging subsystem."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nupdates ApiClient records to include info from Fleetspeak.", "response": "def UpdateClientsFromFleetspeak(clients):\n  \"\"\"Updates ApiClient records to include info from Fleetspeak.\"\"\"\n  if not fleetspeak_connector.CONN or not fleetspeak_connector.CONN.outgoing:\n    # FS not configured, or an outgoing connection is otherwise unavailable.\n    return\n  id_map = {}\n  for client in clients:\n    if client.fleetspeak_enabled:\n      id_map[fleetspeak_utils.GRRIDToFleetspeakID(client.client_id)] = client\n  if not id_map:\n    return\n  res = fleetspeak_connector.CONN.outgoing.ListClients(\n      admin_pb2.ListClientsRequest(client_ids=list(iterkeys(id_map))))\n  for read in res.clients:\n    api_client = id_map[read.client_id]\n    api_client.last_seen_at = fleetspeak_utils.TSToRDFDatetime(\n        read.last_contact_time)\n    api_client.last_clock = fleetspeak_utils.TSToRDFDatetime(read.last_clock)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nremoving labels with given names from a given client object.", "response": "def RemoveClientLabels(self, client, labels_names):\n    \"\"\"Removes labels with given names from a given client object.\"\"\"\n\n    affected_owners = set()\n    for label in client.GetLabels():\n      if label.name in labels_names and label.owner != \"GRR\":\n        affected_owners.add(label.owner)\n\n    for owner in affected_owners:\n      client.RemoveLabels(labels_names, owner=owner)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _NormalizeTime(self, time):\n    if isinstance(time, rdfvalue.RDFDatetime):\n      return time.AsMicrosecondsSinceEpoch()\n    if isinstance(time, rdfvalue.Duration):\n      return time.microseconds\n    return int(time)", "response": "Normalize a time to be an int measured in microseconds."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nadd value at timestamp.", "response": "def Append(self, value, timestamp):\n    \"\"\"Adds value at timestamp.\n\n    Values must be added in order of increasing timestamp.\n\n    Args:\n      value: An observed value.\n      timestamp: The timestamp at which value was observed.\n\n    Raises:\n      RuntimeError: If timestamp is smaller than the previous timstamp.\n    \"\"\"\n\n    timestamp = self._NormalizeTime(timestamp)\n    if self.data and timestamp < self.data[-1][1]:\n      raise RuntimeError(\"Next timestamp must be larger.\")\n    self.data.append([value, timestamp])"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nadd multiple value< = timestamp pairs to the end of the sequence.", "response": "def MultiAppend(self, value_timestamp_pairs):\n    \"\"\"Adds multiple value<->timestamp pairs.\n\n    Args:\n      value_timestamp_pairs: Tuples of (value, timestamp).\n    \"\"\"\n    for value, timestamp in value_timestamp_pairs:\n      self.Append(value, timestamp)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef FilterRange(self, start_time=None, stop_time=None):\n\n    start_time = self._NormalizeTime(start_time)\n    stop_time = self._NormalizeTime(stop_time)\n    self.data = [\n        p for p in self.data\n        if (start_time is None or p[1] >= start_time) and\n        (stop_time is None or p[1] < stop_time)\n    ]", "response": "Filter the series to lie between start_time and stop_time."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nnormalizing the series to have a fixed period over a fixed time range.", "response": "def Normalize(self, period, start_time, stop_time, mode=NORMALIZE_MODE_GAUGE):\n    \"\"\"Normalize the series to have a fixed period over a fixed time range.\n\n    Supports two modes, depending on the type of data:\n\n      NORMALIZE_MODE_GAUGE: support gauge values. If multiple original data\n        points lie within an output interval, the output value is an average of\n        the original data point.  if no original data points lie within an\n        output interval, the output value is None.\n\n      NORMALIZE_MODE_COUNTER: supports counter values. Assumes that the sequence\n        is already increasing (typically, MakeIncreasing will have been\n        called). Each output value is the largest value seen during or before\n        the corresponding output interval.\n\n    Args:\n\n      period: The desired time between points. Should be an rdfvalue.Duration or\n        a count of microseconds.\n\n      start_time: The first timestamp will be at start_time. Should be an\n        rdfvalue.RDFDatetime or a count of microseconds since epoch.\n\n      stop_time: The last timestamp will be at stop_time - period. Should be an\n        rdfvalue.RDFDatetime or a count of microseconds since epoch.\n\n      mode: The type of normalization to perform. May be NORMALIZE_MODE_GAUGE or\n        NORMALIZE_MODE_COUNTER.\n\n    Raises:\n      RuntimeError: In case the sequence timestamps are misordered.\n\n    \"\"\"\n    period = self._NormalizeTime(period)\n    start_time = self._NormalizeTime(start_time)\n    stop_time = self._NormalizeTime(stop_time)\n    if not self.data:\n      return\n\n    self.FilterRange(start_time, stop_time)\n\n    grouped = {}\n    for value, timestamp in self.data:\n      offset = timestamp - start_time\n      shifted_offset = offset - (offset % period)\n      grouped.setdefault(shifted_offset, []).append(value)\n\n    self.data = []\n    last_value = None\n    for offset in range(0, stop_time - start_time, period):\n      g = grouped.get(offset)\n      if mode == NORMALIZE_MODE_GAUGE:\n        v = None\n        if g:\n          v = sum(g) / len(g)\n        self.data.append([v, offset + start_time])\n      else:\n        if g:\n          for v in g:\n            if v < last_value:\n              raise RuntimeError(\"Next value must not be smaller.\")\n            last_value = v\n        self.data.append([last_value, offset + start_time])"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef MakeIncreasing(self):\n    offset = 0\n    last_value = None\n    for p in self.data:\n      if last_value and last_value > p[0]:\n        # Assume that it was only reset once.\n        offset += last_value\n      last_value = p[0]\n      if offset:\n        p[0] += offset", "response": "Makes the time series increasing."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef ToDeltas(self):\n    if len(self.data) < 2:\n      self.data = []\n      return\n    for i in range(0, len(self.data) - 1):\n      if self.data[i][0] is None or self.data[i + 1][0] is None:\n        self.data[i][0] = None\n      else:\n        self.data[i][0] = self.data[i + 1][0] - self.data[i][0]\n    del self.data[-1]", "response": "Convert the sequence of differences between points."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef Add(self, other):\n    if len(self.data) != len(other.data):\n      raise RuntimeError(\"Can only add series of identical lengths.\")\n    for i in range(len(self.data)):\n      if self.data[i][1] != other.data[i][1]:\n        raise RuntimeError(\"Timestamp mismatch.\")\n      if self.data[i][0] is None and other.data[i][0] is None:\n        continue\n      self.data[i][0] = (self.data[i][0] or 0) + (other.data[i][0] or 0)", "response": "Adds other to self pointwise."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn the arithmatic mean of all values.", "response": "def Mean(self):\n    \"\"\"Return the arithmatic mean of all values.\"\"\"\n    values = [v for v, _ in self.data if v is not None]\n    if not values:\n      return None\n\n    # TODO(hanuszczak): Why do we return a floored division result instead of\n    # the exact value?\n    return sum(values) // len(values)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef Validate(self):\n    self._ValidateReturnedTypes()\n    self._ValidatePaths()\n    self._ValidateType()\n    self._ValidateRequiredAttributes()\n    self._ValidateCommandArgs()", "response": "Check the source is well constructed."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef ToPrimitiveDict(self):\n    artifact_dict = super(Artifact, self).ToPrimitiveDict()\n\n    # ArtifactName is not JSON-serializable, so convert name to string.\n    artifact_dict[\"name\"] = utils.SmartStr(self.name)\n\n    # Convert proto enum to simple strings so they get rendered in the GUI\n    # properly\n    for source in artifact_dict[\"sources\"]:\n      if \"type\" in source:\n        source[\"type\"] = str(source[\"type\"])\n      if \"key_value_pairs\" in source[\"attributes\"]:\n        outarray = []\n        for indict in source[\"attributes\"][\"key_value_pairs\"]:\n          outarray.append(dict(indict))\n        source[\"attributes\"][\"key_value_pairs\"] = outarray\n\n    # Repeated fields that have not been set should return as empty lists.\n    for field in self.required_repeated_fields:\n      if field not in artifact_dict:\n        artifact_dict[field] = []\n    return artifact_dict", "response": "Handle dict generation specifically for Artifacts."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef Validate(self):\n    # All filters can handle the input type.\n    bad_filters = []\n    for f in self.filters:\n      try:\n        f.Validate()\n      except DefinitionError as e:\n        bad_filters.append(\"%s: %s\" % (f.expression, e))\n    if bad_filters:\n      raise DefinitionError(\n          \"Filters with invalid expressions: %s\" % \", \".join(bad_filters))", "response": "Verifies this filter set can process the result data."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef Parse(self, raw_data):\n    self.results = set()\n    if not self.filters:\n      self.results.update(raw_data)\n    else:\n      for f in self.filters:\n        self.results.update(f.Parse(raw_data))\n    return list(self.results)", "response": "Take the data and yield results that passed through the filters."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ntake the results and yield results that passed through the filters.", "response": "def Parse(self, raw_data):\n    \"\"\"Take the results and yield results that passed through the filters.\n\n    The output of each filter is used as the input for successive filters.\n\n    Args:\n      raw_data: An iterable series of rdf values.\n\n    Returns:\n      A list of rdf values that matched all filters.\n    \"\"\"\n    self.results = raw_data\n    for f in self.filters:\n      self.results = f.Parse(self.results)\n    return self.results"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning an initialized filter. Only initialize filters once.", "response": "def GetFilter(cls, filter_name):\n    \"\"\"Return an initialized filter. Only initialize filters once.\n\n    Args:\n      filter_name: The name of the filter, as a string.\n\n    Returns:\n      an initialized instance of the filter.\n\n    Raises:\n      DefinitionError if the type of filter has not been defined.\n    \"\"\"\n    # Check if the filter is defined in the registry.\n    try:\n      filt_cls = cls.GetPlugin(filter_name)\n    except KeyError:\n      raise DefinitionError(\"Filter %s does not exist.\" % filter_name)\n\n    return filt_cls()"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _GetVal(self, obj, key):\n    if \".\" in key:\n      lhs, rhs = key.split(\".\", 1)\n      obj2 = getattr(obj, lhs, None)\n      if obj2 is None:\n        return None\n      return self._GetVal(obj2, rhs)\n    else:\n      return getattr(obj, key, None)", "response": "Recursively walks down an attribute chain to the actual result data."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef ParseObjs(self, objs, expression):\n    filt = self._Compile(expression)\n    for result in filt.Filter(objs):\n      yield result", "response": "Parse one or more objects using an objectfilter expression."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ngenerates lambdas for uid and gid comparison.", "response": "def _Comparator(self, operator):\n    \"\"\"Generate lambdas for uid and gid comparison.\"\"\"\n    if operator == \"=\":\n      return lambda x, y: x == y\n    elif operator == \">=\":\n      return lambda x, y: x >= y\n    elif operator == \">\":\n      return lambda x, y: x > y\n    elif operator == \"<=\":\n      return lambda x, y: x <= y\n    elif operator == \"<\":\n      return lambda x, y: x < y\n    elif operator == \"!\":\n      return lambda x, y: x != y\n    raise DefinitionError(\"Invalid comparison operator %s\" % operator)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _Initialize(self):\n\n    if self.cfg.mask:\n      self.mask = int(self.cfg.mask[0], 8)\n    else:\n      self.mask = 0o7777\n    if self.cfg.mode:\n      self.mode = int(self.cfg.mode[0], 8)\n      self.matchers.append(self._MatchMode)\n\n    if self.cfg.gid:\n      for gid in self.cfg.gid:\n        matched = self._UID_GID_RE.match(gid)\n        if matched:\n          o, v = matched.groups()\n          self.gid_matchers.append((self._Comparator(o), int(v)))\n      self.matchers.append(self._MatchGid)\n\n    if self.cfg.uid:\n      for uid in self.cfg.uid:\n        matched = self._UID_GID_RE.match(uid)\n        if matched:\n          o, v = matched.groups()\n          self.uid_matchers.append((self._Comparator(o), int(v)))\n      self.matchers.append(self._MatchUid)\n\n    if self.cfg.file_re:\n      self.file_re = re.compile(self.cfg.file_re[0])\n      self.matchers.append(self._MatchFile)\n\n    if self.cfg.path_re:\n      self.path_re = re.compile(self.cfg.path_re[0])\n      self.matchers.append(self._MatchPath)\n\n    if self.cfg.file_type:\n      self.file_type = self._TYPES.get(self.cfg.file_type[0].upper())\n      self.matchers.append(self._MatchType)", "response": "Initialize the filter configuration from a validated configuration."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nparses one or more objects by testing if they match the given expression.", "response": "def ParseObjs(self, objs, expression):\n    \"\"\"Parse one or more objects by testing if it has matching stat results.\n\n    Args:\n      objs: An iterable of objects that should be checked.\n      expression: A StatFilter expression, e.g.:\n        \"uid:>0 gid:=0 file_type:link\"\n\n    Yields:\n      matching objects.\n    \"\"\"\n    self.Validate(expression)\n    for obj in objs:\n      if not isinstance(obj, rdf_client_fs.StatEntry):\n        continue\n      # If all match conditions pass, yield the object.\n      for match in self.matchers:\n        if not match(obj):\n          break\n      else:\n        yield obj"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nvalidates that a parsed rule entry is valid for fschecker.", "response": "def Validate(self, expression):\n    \"\"\"Validates that a parsed rule entry is valid for fschecker.\n\n    Args:\n      expression: A rule expression.\n\n    Raises:\n      DefinitionError: If the filter definition could not be validated.\n\n    Returns:\n      True if the expression validated OK.\n    \"\"\"\n    parsed = self._Load(expression)\n\n    if not parsed:\n      raise DefinitionError(\"Empty StatFilter expression.\")\n\n    bad_keys = set(parsed) - self._KEYS\n    if bad_keys:\n      raise DefinitionError(\"Invalid parameters: %s\" % \",\".join(bad_keys))\n\n    if self.cfg.mask and not self.cfg.mode:\n      raise DefinitionError(\"mode can only be set when mask is also defined.\")\n\n    if self.cfg.mask:\n      if len(self.cfg.mask) > 1:\n        raise DefinitionError(\"Too many mask values defined.\")\n      if not self._PERM_RE.match(self.cfg.mask[0]):\n        raise DefinitionError(\"mask=%s is not octal, e.g. 0600\" % self.cfg.mask)\n\n    if self.cfg.mode:\n      if len(self.cfg.mode) > 1:\n        raise DefinitionError(\"Too many mode values defined.\")\n      if not self._PERM_RE.match(self.cfg.mode[0]):\n        raise DefinitionError(\"mode=%s is not octal, e.g. 0600\" % self.cfg.mode)\n\n    if self.cfg.gid:\n      for gid in self.cfg.gid:\n        matched = self._UID_GID_RE.match(gid)\n        if not matched:\n          raise DefinitionError(\"gid: %s is not an integer preceded by \"\n                                \"!, >, < or =.\" % gid)\n\n    if self.cfg.uid:\n      for uid in self.cfg.uid:\n        matched = self._UID_GID_RE.match(uid)\n        if not matched:\n          raise DefinitionError(\"uid: %s is not an integer preceded by \"\n                                \"!, >, < or =.\" % uid)\n\n    if self.cfg.file_re:\n      if len(self.cfg.file_re) > 1:\n        raise DefinitionError(\"Too many regexes defined: %s\" % self.cfg.file_re)\n      try:\n        self.file_re = re.compile(self.cfg.file_re[0])\n      except (re.error, TypeError) as e:\n        raise DefinitionError(\"Invalid file regex: %s\" % e)\n\n    if self.cfg.path_re:\n      if len(self.cfg.path_re) > 1:\n        raise DefinitionError(\"Too many regexes defined: %s\" % self.cfg.path_re)\n      try:\n        self.path_re = re.compile(self.cfg.path_re[0])\n      except (re.error, TypeError) as e:\n        raise DefinitionError(\"Invalid path regex: %s\" % e)\n\n    if self.cfg.file_type:\n      if len(self.cfg.file_type) > 1:\n        raise DefinitionError(\n            \"Too many file types defined: %s\" % self.cfg.file_type)\n      file_type = self.cfg.file_type[0].upper()\n      if file_type not in self._TYPES:\n        raise DefinitionError(\"Unsupported file type %s\" % file_type)\n\n    self._Initialize()\n    if not self.matchers:\n      raise DefinitionError(\"StatFilter has no actions: %s\" % expression)\n    return True"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef ParseObjs(self, objs, type_names):\n    for obj in objs:\n      for type_name in self._RDFTypes(type_names):\n        if isinstance(obj, self._GetClass(type_name)):\n          yield obj", "response": "Parse one or more objects by testing if they are a known RDF class."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nvalidate that the given types are RDFValues.", "response": "def Validate(self, type_names):\n    \"\"\"Filtered types need to be RDFValues.\"\"\"\n    errs = [n for n in self._RDFTypes(type_names) if not self._GetClass(n)]\n    if errs:\n      raise DefinitionError(\"Undefined RDF Types: %s\" % \",\".join(errs))"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef main(argv):\n  del argv  # Unused.\n\n  if flags.FLAGS.version:\n    print(\"GRR worker {}\".format(config_server.VERSION[\"packageversion\"]))\n    return\n\n  config.CONFIG.AddContext(contexts.WORKER_CONTEXT,\n                           \"Context applied when running a worker.\")\n\n  # Initialise flows and config_lib\n  server_startup.Init()\n\n  fleetspeak_connector.Init()\n\n\n  token = access_control.ACLToken(username=\"GRRWorker\").SetUID()\n  worker_obj = worker_lib.GRRWorker(token=token)\n  worker_obj.Run()", "response": "Main function for GRR worker."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef SetName(obj, name):\n  # Not doing type assertion on obj, since it may be a mock object used\n  # in tests.\n  precondition.AssertType(name, str)\n\n  if PY2:\n    obj.__name__ = name.encode(\"ascii\")\n  else:\n    obj.__name__ = name", "response": "A compatibility wrapper for setting object s name."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef MakeType(name, base_classes,\n             namespace):\n  \"\"\"A compatibility wrapper for the `type` built-in function.\n\n  In Python 2 `type` (used as a type constructor) requires the name argument to\n  be a `bytes` object whereas in Python 3 it is required to be an `unicode`\n  object. Since class name is human readable text rather than arbitrary stream\n  of bytes, the Python 3 behaviour is considered to be the sane one.\n\n  Once support for Python 2 is dropped all invocations of this call can be\n  replaced with the `type` built-in.\n\n  Args:\n    name: A name of the type to create.\n    base_classes: A tuple of base classes that the returned type is supposed to\n      derive from.\n    namespace: A dictionary of methods and fields that the returned type is\n      supposed to contain.\n\n  Returns:\n    A new type with specified parameters.\n  \"\"\"\n  precondition.AssertType(name, str)\n\n  if PY2:\n    name = name.encode(\"ascii\")\n\n  return type(name, base_classes, namespace)", "response": "A compatibility wrapper for the type function."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef FormatTime(fmt, stime = None):\n  precondition.AssertType(fmt, str)\n  precondition.AssertOptionalType(stime, time.struct_time)\n\n  # TODO(hanuszczak): https://github.com/google/pytype/issues/127\n  # pytype: disable=wrong-arg-types\n  # We need this because second parameter is not a keyword argument, so method\n  # must be explicitly called with or without it.\n  if stime is None:\n    strftime = time.strftime\n  else:\n    strftime = lambda fmt: time.strftime(fmt, stime)\n\n  if PY2:\n    return strftime(fmt.encode(\"ascii\")).decode(\"ascii\")\n  else:\n    return strftime(fmt)", "response": "A compatibility wrapper for the strftime function."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef ShlexSplit(string):\n  precondition.AssertType(string, Text)\n\n  if PY2:\n    string = string.encode(\"utf-8\")\n\n  parts = shlex.split(string)\n\n  if PY2:\n    # TODO(hanuszczak): https://github.com/google/pytype/issues/127\n    # pytype: disable=attribute-error\n    parts = [part.decode(\"utf-8\") for part in parts]\n    # pytype: enable=attribute-error\n\n  return parts", "response": "A wrapper for shlex. split that works with unicode objects."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef Dump(obj,\n         sort_keys = False,\n         encoder = None):\n  \"\"\"Stringifies a Python object into its JSON representation.\n\n  Args:\n    obj: A Python object to convert to JSON.\n    sort_keys: If True, output dictionaries keys in sorted (ascending) order.\n    encoder: An (optional) encoder class to use.\n\n  Returns:\n    A JSON representation of the given object.\n  \"\"\"\n  # Python 2 json.dumps expects separators as a tuple of bytes, while\n  # Python 3 expects them to be a tuple of unicode strings. Pytype\n  # is too dumb to infer the result of the if statement that sets\n  # _SEPARATORS and complains when running in Python 3 mode.\n  text = json.dumps(\n      obj,\n      indent=2,\n      sort_keys=sort_keys,\n      ensure_ascii=False,\n      cls=encoder,\n      separators=_SEPARATORS)  # pytype: disable=wrong-arg-types\n\n  # `text` is an instance of `bytes` if the object to serialize does not contain\n  # any unicode characters, otherwise it is `unicode`. See [1] for details.\n  #\n  # [1]: https://bugs.python.org/issue13769\n  if compatibility.PY2 and isinstance(text, bytes):\n    text = text.decode(\"utf-8\")  # pytype: disable=attribute-error\n\n  return text", "response": "Stringifies a Python object into its JSON representation."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef DropPrivileges():\n  if config.CONFIG[\"Server.username\"]:\n    try:\n      os.setuid(pwd.getpwnam(config.CONFIG[\"Server.username\"]).pw_uid)\n    except (KeyError, OSError):\n      logging.exception(\"Unable to switch to user %s\",\n                        config.CONFIG[\"Server.username\"])\n      raise", "response": "Attempt to drop privileges if required."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nrunning all required startup routines and initialization hooks.", "response": "def Init():\n  \"\"\"Run all required startup routines and initialization hooks.\"\"\"\n  global INIT_RAN\n  if INIT_RAN:\n    return\n\n  # Set up a temporary syslog handler so we have somewhere to log problems\n  # with ConfigInit() which needs to happen before we can start our create our\n  # proper logging setup.\n  syslog_logger = logging.getLogger(\"TempLogger\")\n  if os.path.exists(\"/dev/log\"):\n    handler = logging.handlers.SysLogHandler(address=\"/dev/log\")\n  else:\n    handler = logging.handlers.SysLogHandler()\n  syslog_logger.addHandler(handler)\n\n  try:\n    config_lib.SetPlatformArchContext()\n    config_lib.ParseConfigCommandLine()\n  except config_lib.Error:\n    syslog_logger.exception(\"Died during config initialization\")\n    raise\n\n  metric_metadata = server_metrics.GetMetadata()\n  metric_metadata.extend(communicator.GetMetricMetadata())\n\n  stats_collector = prometheus_stats_collector.PrometheusStatsCollector(\n      metric_metadata, registry=prometheus_client.REGISTRY)\n  stats_collector_instance.Set(stats_collector)\n\n  server_logging.ServerLoggingStartupInit()\n\n  bs_registry_init.RegisterBlobStores()\n  all_decoders.Register()\n  all_parsers.Register()\n  registry.Init()\n\n  # Exempt config updater from this check because it is the one responsible for\n  # setting the variable.\n  if not config.CONFIG.ContextApplied(\"ConfigUpdater Context\"):\n    if not config.CONFIG.Get(\"Server.initialized\"):\n      raise RuntimeError(\"Config not initialized, run \\\"grr_config_updater\"\n                         \" initialize\\\". If the server is already configured,\"\n                         \" add \\\"Server.initialized: True\\\" to your config.\")\n\n  INIT_RAN = True"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef VersionPath():\n\n  # Try to get a version.ini. It should be in the resources if the code\n  # was packed with \"pip sdist\". It will be 2 levels up from grr_response_core\n  # if the code was installed via \"pip install -e\".\n  version_ini = (\n      package.ResourcePath(\"grr-response-core\", \"version.ini\") or\n      package.ResourcePath(\"grr-response-core\", \"../../version.ini\"))\n\n  if not os.path.exists(version_ini):\n    raise RuntimeError(\"Can't find version.ini at %s\" % version_ini)\n\n  return version_ini", "response": "Returns a path to version. ini."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef Version():\n\n  version_ini = VersionPath()\n\n  config = configparser.SafeConfigParser()\n  config.read(version_ini)\n  return dict(\n      packageversion=config.get(\"Version\", \"packageversion\"),\n      major=config.getint(\"Version\", \"major\"),\n      minor=config.getint(\"Version\", \"minor\"),\n      revision=config.getint(\"Version\", \"revision\"),\n      release=config.getint(\"Version\", \"release\"))", "response": "Return a dict with GRR version information."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef Regions(self,\n              skip_executable_regions=False,\n              skip_shared_regions=False,\n              skip_readonly_regions=False):\n    \"\"\"Iterates over the readable regions for this process.\n\n    We use mach_vm_region_recurse here to get a fine grained view of\n    the process' memory space. The algorithm is that for some regions,\n    the function returns is_submap=True which means that there are\n    actually subregions that we need to examine by increasing the\n    depth and calling the function again. For example, there are two\n    regions, addresses 1000-2000 and 2000-3000 where 1000-2000 has two\n    subregions, 1100-1200 and 1300-1400. In that case we would call:\n\n    mvrr(address=0, depth=0)       -> (1000-2000, is_submap=True)\n    mvrr(address=0, depth=1)       -> (1100-1200, is_submap=False)\n    mvrr(address=1200, depth=1)    -> (1300-1400, is_submap=False)\n    mvrr(address=1400, depth=1)    -> (2000-3000, is_submap=False)\n\n    At this point, we know we went out of the original submap which\n    ends at 2000. We need to recheck the region at 2000, it could be\n    submap = True at depth 0 so we call\n\n    mvrr(address=1400, depth=0)    -> (2000-3000, is_submap=False)\n\n    Args:\n      skip_executable_regions: Skips executable sections.\n      skip_shared_regions: Skips shared sections. Includes mapped files.\n      skip_readonly_regions: Skips readonly sections.\n\n    Yields:\n      Pairs (address, length) for each identified region.\n    \"\"\"\n\n    address = ctypes.c_ulong(0)\n    mapsize = ctypes.c_ulong(0)\n    count = ctypes.c_uint32(submap_info_size)\n    sub_info = vm_region_submap_short_info_data_64()\n    depth = 0\n    depth_end_addresses = {}\n\n    while True:\n      c_depth = ctypes.c_uint32(depth)\n\n      r = libc.mach_vm_region_recurse(self.task, ctypes.pointer(address),\n                                      ctypes.pointer(mapsize),\n                                      ctypes.pointer(c_depth),\n                                      ctypes.pointer(sub_info),\n                                      ctypes.pointer(count))\n\n      # If we get told \"invalid address\", we have crossed into kernel land...\n      if r == 1:\n        break\n\n      if r != 0:\n        raise process_error.ProcessError(\"Error in mach_vm_region, ret=%s\" % r)\n\n      if depth > 0 and address.value >= depth_end_addresses[depth]:\n        del depth_end_addresses[depth]\n        depth -= 1\n        continue\n\n      p = sub_info.protection\n      if skip_executable_regions and p & VM_PROT_EXECUTE:\n        address.value += mapsize.value\n        continue\n\n      if skip_shared_regions and sub_info.share_mode in [\n          SM_COW, SM_SHARED, SM_TRUESHARED\n      ]:\n        address.value += mapsize.value\n        continue\n\n      if not p & VM_PROT_READ:\n        address.value += mapsize.value\n        continue\n\n      writable = p & VM_PROT_WRITE\n      if skip_readonly_regions and not writable:\n        address.value += mapsize.value\n        continue\n\n      if sub_info.is_submap:\n        depth += 1\n        depth_end_addresses[depth] = address.value + mapsize.value\n      else:\n        yield address.value, mapsize.value\n        address.value += mapsize.value", "response": "Iterates over the readable regions of this process."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreads at most num_bytes starting from offset <address >.", "response": "def ReadBytes(self, address, num_bytes):\n    \"\"\"Reads at most num_bytes starting from offset <address>.\"\"\"\n    pdata = ctypes.c_void_p(0)\n    data_cnt = ctypes.c_uint32(0)\n\n    ret = libc.mach_vm_read(self.task, ctypes.c_ulonglong(address),\n                            ctypes.c_longlong(num_bytes), ctypes.pointer(pdata),\n                            ctypes.pointer(data_cnt))\n    if ret:\n      raise process_error.ProcessError(\"Error in mach_vm_read, ret=%s\" % ret)\n    buf = ctypes.string_at(pdata.value, data_cnt.value)\n    libc.vm_deallocate(self.mytask, pdata, data_cnt)\n    return buf"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nconverting a binary SID to its string representation.", "response": "def BinarySIDtoStringSID(sid):\n  \"\"\"Converts a binary SID to its string representation.\n\n  https://msdn.microsoft.com/en-us/library/windows/desktop/aa379597.aspx\n\n  The byte representation of an SID is as follows:\n    Offset  Length  Description\n    00      01      revision\n    01      01      sub-authority count\n    02      06      authority (big endian)\n    08      04      subauthority #1 (little endian)\n    0b      04      subauthority #2 (little endian)\n    ...\n\n  Args:\n    sid: A `bytes` instance for a SID to convert.\n\n  Returns:\n    A `unicode` representation of given SID.\n\n  Raises:\n    ValueError: If the binary SID is malformed.\n  \"\"\"\n  precondition.AssertType(sid, bytes)\n\n  # TODO: This seemingly no-op is actually not a no-op. The reason\n  # is that `sid` might be either `bytes` from the future package or `str` (e.g.\n  # a bytes literal on Python 2). This call ensures that we get a `bytes` object\n  # with Python 3 semantics. Once GRR's codebase is ready to drop support for\n  # Python 2 this line can be removed as indeed then it would be a no-op.\n  sid = bytes(sid)\n\n  if not sid:\n    return u\"\"\n\n  str_sid_components = [sid[0]]\n  # Now decode the 48-byte portion\n\n  if len(sid) >= 8:\n    subauthority_count = sid[1]\n\n    identifier_authority = struct.unpack(\">H\", sid[2:4])[0]\n    identifier_authority <<= 32\n    identifier_authority |= struct.unpack(\">L\", sid[4:8])[0]\n    str_sid_components.append(identifier_authority)\n\n    start = 8\n    for i in range(subauthority_count):\n      authority = sid[start:start + 4]\n      if not authority:\n        break\n\n      if len(authority) < 4:\n        message = (\"In binary SID '%s', component %d has been truncated. \"\n                   \"Expected 4 bytes, found %d: (%s)\")\n        message %= (sid, i, len(authority), authority)\n        raise ValueError(message)\n\n      str_sid_components.append(struct.unpack(\"<L\", authority)[0])\n      start += 4\n\n  return u\"S-%s\" % (u\"-\".join(map(str, str_sid_components)))"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nparses WMI Event Consumers.", "response": "def ParseMultiple(self, result_dicts):\n    \"\"\"Parse WMI Event Consumers.\"\"\"\n    for result_dict in result_dicts:\n      wmi_dict = result_dict.ToDict()\n\n      try:\n        creator_sid_bytes = bytes(wmi_dict[\"CreatorSID\"])\n        wmi_dict[\"CreatorSID\"] = BinarySIDtoStringSID(creator_sid_bytes)\n      except ValueError:\n        # We recover from corrupt SIDs by outputting it raw as a string\n        wmi_dict[\"CreatorSID\"] = compatibility.Repr(wmi_dict[\"CreatorSID\"])\n      except KeyError:\n        pass\n\n      for output_type in self.output_types:\n        anomalies = []\n\n        output = rdfvalue.RDFValue.classes[output_type]()\n        for k, v in iteritems(wmi_dict):\n          try:\n            output.Set(k, v)\n          except AttributeError as e:\n            # Skip any attribute we don't know about\n            anomalies.append(\"Unknown field %s, with value %s\" % (k, v))\n          except ValueError as e:\n            anomalies.append(\"Invalid value %s for field %s: %s\" % (v, k, e))\n\n        # Yield anomalies first to help with debugging\n        if anomalies:\n          yield rdf_anomaly.Anomaly(\n              type=\"PARSER_ANOMALY\",\n              generated_by=self.__class__.__name__,\n              finding=anomalies)\n\n        # Raise if the parser generated no output but there were fields.\n        if wmi_dict and not output:\n          raise ValueError(\"Non-empty dict %s returned empty output.\" %\n                           wmi_dict)\n\n        yield output"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nparses the WMI packages output.", "response": "def ParseMultiple(self, result_dicts):\n    \"\"\"Parse the WMI packages output.\"\"\"\n    status = rdf_client.SoftwarePackage.InstallState.INSTALLED\n    packages = []\n    for result_dict in result_dicts:\n      packages.append(\n          rdf_client.SoftwarePackage(\n              name=result_dict[\"Name\"],\n              description=result_dict[\"Description\"],\n              version=result_dict[\"Version\"],\n              install_state=status))\n\n    if packages:\n      yield rdf_client.SoftwarePackages(packages=packages)"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ntakes a US format date and return epoch.", "response": "def AmericanDateToEpoch(self, date_str):\n    \"\"\"Take a US format date and return epoch.\"\"\"\n    try:\n      epoch = time.strptime(date_str, \"%m/%d/%Y\")\n      return int(calendar.timegm(epoch)) * 1000000\n    except ValueError:\n      return 0"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef ParseMultiple(self, result_dicts):\n    status = rdf_client.SoftwarePackage.InstallState.INSTALLED\n\n    packages = []\n    for result_dict in result_dicts:\n      result = result_dict.ToDict()\n\n      # InstalledOn comes back in a godawful format such as '7/10/2013'.\n      installed_on = self.AmericanDateToEpoch(result.get(\"InstalledOn\", \"\"))\n      packages.append(\n          rdf_client.SoftwarePackage(\n              name=result.get(\"HotFixID\"),\n              description=result.get(\"Caption\"),\n              installed_by=result.get(\"InstalledBy\"),\n              install_state=status,\n              installed_on=installed_on))\n\n    if packages:\n      yield rdf_client.SoftwarePackages(packages=packages)", "response": "Parse the WMI packages output."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nparse the WMI Win32_UserAccount output.", "response": "def ParseMultiple(self, result_dicts):\n    \"\"\"Parse the WMI Win32_UserAccount output.\"\"\"\n    for result_dict in result_dicts:\n      kb_user = rdf_client.User()\n      for wmi_key, kb_key in iteritems(self.account_mapping):\n        try:\n          kb_user.Set(kb_key, result_dict[wmi_key])\n        except KeyError:\n          pass\n      # We need at least a sid or a username.  If these are missing its likely\n      # we retrieved just the userdomain for an AD account that has a name\n      # collision with a local account that is correctly populated.  We drop the\n      # bogus domain account.\n      if kb_user.sid or kb_user.username:\n        yield kb_user"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef ParseMultiple(self, result_dicts):\n    for result_dict in result_dicts:\n      result = result_dict.ToDict()\n      winvolume = rdf_client_fs.WindowsVolume(\n          drive_letter=result.get(\"DeviceID\"),\n          drive_type=result.get(\"DriveType\"))\n\n      try:\n        size = int(result.get(\"Size\"))\n      except (ValueError, TypeError):\n        size = None\n\n      try:\n        free_space = int(result.get(\"FreeSpace\"))\n      except (ValueError, TypeError):\n        free_space = None\n\n      # Since we don't get the sector sizes from WMI, we just set them at 1 byte\n      yield rdf_client_fs.Volume(\n          windowsvolume=winvolume,\n          name=result.get(\"VolumeName\"),\n          file_system_type=result.get(\"FileSystem\"),\n          serial_number=result.get(\"VolumeSerialNumber\"),\n          sectors_per_allocation_unit=1,\n          bytes_per_sector=1,\n          total_allocation_units=size,\n          actual_available_allocation_units=free_space)", "response": "Parse the WMI packages output."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef ParseMultiple(self, result_dicts):\n    for result_dict in result_dicts:\n      # Currently we are only grabbing the Identifying Number\n      # as the serial number (catches the unique number for VMs).\n      # This could be changed to include more information from\n      # Win32_ComputerSystemProduct.\n\n      yield rdf_client.HardwareInfo(\n          serial_number=result_dict[\"IdentifyingNumber\"],\n          system_manufacturer=result_dict[\"Vendor\"])", "response": "Parse the WMI output to get Identifying Number."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef WMITimeStrToRDFDatetime(self, timestr):\n    # We use manual parsing here because the time functions provided (datetime,\n    # dateutil) do not properly deal with timezone information.\n    offset_minutes = timestr[21:]\n    year = timestr[:4]\n    month = timestr[4:6]\n    day = timestr[6:8]\n    hours = timestr[8:10]\n    minutes = timestr[10:12]\n    seconds = timestr[12:14]\n    microseconds = timestr[15:21]\n\n    unix_seconds = calendar.timegm(\n        tuple(map(int, [year, month, day, hours, minutes, seconds])))\n    unix_seconds -= int(offset_minutes) * 60\n    return rdfvalue.RDFDatetime(unix_seconds * 1e6 + int(microseconds))", "response": "Converts a WMI time string to an RDFDatetime object."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef ParseMultiple(self, result_dicts):\n    for result_dict in result_dicts:\n      args = {\"ifname\": result_dict[\"Description\"]}\n      args[\"mac_address\"] = binascii.unhexlify(\n          result_dict[\"MACAddress\"].replace(\":\", \"\"))\n\n      self._ConvertIPs([(\"IPAddress\", \"addresses\"),\n                        (\"DefaultIPGateway\", \"ip_gateway_list\"),\n                        (\"DHCPServer\", \"dhcp_server_list\")], result_dict, args)\n\n      if \"DHCPLeaseExpires\" in result_dict:\n        args[\"dhcp_lease_expires\"] = self.WMITimeStrToRDFDatetime(\n            result_dict[\"DHCPLeaseExpires\"])\n\n      if \"DHCPLeaseObtained\" in result_dict:\n        args[\"dhcp_lease_obtained\"] = self.WMITimeStrToRDFDatetime(\n            result_dict[\"DHCPLeaseObtained\"])\n\n      yield rdf_client_network.Interface(**args)\n\n      yield rdf_client_network.DNSClientConfiguration(\n          dns_server=result_dict[\"DNSServerSearchOrder\"],\n          dns_suffix=result_dict[\"DNSDomainSuffixSearchOrder\"])", "response": "Parse the WMI packages output."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _ClientIDFromSessionID(session_id):\n\n  parts = session_id.Split(4)\n  client_id = parts[0]\n  if re.match(r\"C\\.[0-9a-f]{16}\", client_id):\n    return client_id\n\n  # Maybe it's a legacy hunt id (aff4:/hunts/<hunt_id>/<client_id>/...\n  client_id = parts[2]\n  if re.match(r\"C\\.[0-9a-f]{16}\", client_id):\n    return client_id\n\n  raise ValueError(\"Unable to parse client id from session_id: %s\" % session_id)", "response": "Extracts the client id from a session id."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef Parse(self):\n    for data in self.Query(self.EVENTS_QUERY):\n      (timestamp, agent_bundle_identifier, agent_name, url, sender,\n       sender_address, type_number, title, referrer, referrer_alias) = data\n      yield [\n          timestamp, \"OSX_QUARANTINE\", url, referrer, title, agent_name,\n          agent_bundle_identifier, sender, sender_address, type_number,\n          referrer_alias\n      ]", "response": "Yields a list of events from the history."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef RetrieveIP4Info(self, ip):\n    if ip.is_private:\n      return (IPInfo.INTERNAL, \"Internal IP address.\")\n    try:\n      # It's an external IP, let's try to do a reverse lookup.\n      res = socket.getnameinfo((str(ip), 0), socket.NI_NAMEREQD)\n      return (IPInfo.EXTERNAL, res[0])\n    except (socket.error, socket.herror, socket.gaierror):\n      return (IPInfo.EXTERNAL, \"Unknown IP address.\")", "response": "Retrieves information for an IP4 address."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get(self, blocking=True):\n    if self.closed:\n      raise PoolAlreadyClosedError(\"Connection pool is already closed.\")\n\n    # NOTE: Once we acquire capacity from the semaphore, it is essential that we\n    # return it eventually. On success, this responsibility is delegated to\n    # _ConnectionProxy.\n    if not self.limiter.acquire(blocking=blocking):\n      return None\n    c = None\n    # pop is atomic, but if we did a check first, it would not be atomic with\n    # the pop.\n    try:\n      c = self.idle_conns.pop()\n    except IndexError:\n      # Create a connection, release the pool allocation if it fails.\n      try:\n        c = self.connect_func()\n      except Exception:\n        self.limiter.release()\n        raise\n    return _ConnectionProxy(self, c)", "response": "Gets a connection from the pool."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef CheckTemplates(self, base_dir, version):\n    major_minor = \".\".join(version.split(\".\")[0:2])\n    templates = glob.glob(\n        os.path.join(base_dir, \"templates/*%s*.zip\" % major_minor))\n    required_templates = set(\n        [x.replace(\"maj.minor\", major_minor) for x in self.REQUIRED_TEMPLATES])\n\n    # Client templates have an extra version digit, e.g. 3.1.0.0\n    templates_present = set([\n        re.sub(r\"_%s[^_]+_\" % major_minor, \"_%s_\" % major_minor,\n               os.path.basename(x)) for x in templates\n    ])\n\n    difference = required_templates - templates_present\n    if difference:\n      raise RuntimeError(\"Missing templates %s\" % difference)", "response": "Verify that we have at least one template that matches maj. minor version."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _TerminateFlow(rdf_flow,\n                   reason=None,\n                   flow_state=rdf_flow_objects.Flow.FlowState.ERROR):\n  \"\"\"Does the actual termination.\"\"\"\n  flow_cls = registry.FlowRegistry.FlowClassByName(rdf_flow.flow_class_name)\n  flow_obj = flow_cls(rdf_flow)\n\n  if not flow_obj.IsRunning():\n    # Nothing to do.\n    return\n\n  logging.info(\"Terminating flow %s on %s, reason: %s\", rdf_flow.flow_id,\n               rdf_flow.client_id, reason)\n\n  rdf_flow.flow_state = flow_state\n  rdf_flow.error_message = reason\n  flow_obj.NotifyCreatorOfError()\n\n  data_store.REL_DB.UpdateFlow(\n      rdf_flow.client_id,\n      rdf_flow.flow_id,\n      flow_obj=rdf_flow,\n      processing_on=None,\n      processing_since=None,\n      processing_deadline=None)\n  data_store.REL_DB.DeleteAllFlowRequestsAndResponses(rdf_flow.client_id,\n                                                      rdf_flow.flow_id)", "response": "Does the actual termination."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nterminating a flow and all of its children.", "response": "def TerminateFlow(client_id,\n                  flow_id,\n                  reason=None,\n                  flow_state=rdf_flow_objects.Flow.FlowState.ERROR):\n  \"\"\"Terminates a flow and all of its children.\n\n  Args:\n    client_id: Client ID of a flow to terminate.\n    flow_id: Flow ID of a flow to terminate.\n    reason: String with a termination reason.\n    flow_state: Flow state to be assigned to a flow after termination. Defaults\n      to FlowState.ERROR.\n  \"\"\"\n\n  to_terminate = [data_store.REL_DB.ReadFlowObject(client_id, flow_id)]\n\n  while to_terminate:\n    next_to_terminate = []\n    for rdf_flow in to_terminate:\n      _TerminateFlow(rdf_flow, reason=reason, flow_state=flow_state)\n      next_to_terminate.extend(\n          data_store.REL_DB.ReadChildFlowObjects(rdf_flow.client_id,\n                                                 rdf_flow.flow_id))\n    to_terminate = next_to_terminate"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef CallClient(self,\n                 action_cls,\n                 request=None,\n                 next_state=None,\n                 request_data=None,\n                 **kwargs):\n    \"\"\"Calls the client asynchronously.\n\n    This sends a message to the client to invoke an Action. The run action may\n    send back many responses that will be queued by the framework until a status\n    message is sent by the client. The status message will cause the entire\n    transaction to be committed to the specified state.\n\n    Args:\n       action_cls: The function to call on the client.\n       request: The request to send to the client. If not specified, we create a\n         new RDFValue using the kwargs.\n       next_state: The state in this flow, that responses to this message should\n         go to.\n       request_data: A dict which will be available in the RequestState\n         protobuf. The Responses object maintains a reference to this protobuf\n         for use in the execution of the state method. (so you can access this\n         data by responses.request).\n       **kwargs: These args will be used to construct the client action argument\n         rdfvalue.\n\n    Raises:\n       ValueError: The request passed to the client does not have the correct\n                   type.\n    \"\"\"\n    try:\n      action_identifier = action_registry.ID_BY_ACTION_STUB[action_cls]\n    except KeyError:\n      raise ValueError(\"Action class %s not known.\" % action_cls)\n\n    if action_cls.in_rdfvalue is None:\n      if request:\n        raise ValueError(\"Client action %s does not expect args.\" % action_cls)\n    else:\n      if request is None:\n        # Create a new rdf request.\n        request = action_cls.in_rdfvalue(**kwargs)\n      else:\n        # Verify that the request type matches the client action requirements.\n        if not isinstance(request, action_cls.in_rdfvalue):\n          raise ValueError(\"Client action expected %s but got %s\" %\n                           (action_cls.in_rdfvalue, type(request)))\n\n    outbound_id = self.GetNextOutboundId()\n\n    # Create a flow request.\n    flow_request = rdf_flow_objects.FlowRequest(\n        client_id=self.rdf_flow.client_id,\n        flow_id=self.rdf_flow.flow_id,\n        request_id=outbound_id,\n        next_state=next_state)\n\n    if request_data is not None:\n      flow_request.request_data = rdf_protodict.Dict().FromDict(request_data)\n\n    cpu_limit_ms = None\n    network_bytes_limit = None\n\n    if self.rdf_flow.cpu_limit:\n      cpu_usage = self.rdf_flow.cpu_time_used\n      cpu_limit_ms = 1000 * max(\n          self.rdf_flow.cpu_limit - cpu_usage.user_cpu_time -\n          cpu_usage.system_cpu_time, 0)\n\n      if cpu_limit_ms == 0:\n        raise flow.FlowError(\"CPU limit exceeded for {} {}.\".format(\n            self.rdf_flow.flow_class_name, self.rdf_flow.flow_id))\n\n    if self.rdf_flow.network_bytes_limit:\n      network_bytes_limit = max(\n          self.rdf_flow.network_bytes_limit - self.rdf_flow.network_bytes_sent,\n          0)\n      if network_bytes_limit == 0:\n        raise flow.FlowError(\"Network limit exceeded for {} {}.\".format(\n            self.rdf_flow.flow_class_name, self.rdf_flow.flow_id))\n\n    client_action_request = rdf_flows.ClientActionRequest(\n        client_id=self.rdf_flow.client_id,\n        flow_id=self.rdf_flow.flow_id,\n        request_id=outbound_id,\n        action_identifier=action_identifier,\n        action_args=request,\n        cpu_limit_ms=cpu_limit_ms,\n        network_bytes_limit=network_bytes_limit)\n\n    self.flow_requests.append(flow_request)\n    self.client_action_requests.append(client_action_request)", "response": "Calls the client asynchronously."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncalls a flow and returns the id of the new flow.", "response": "def CallFlow(self,\n               flow_name=None,\n               next_state=None,\n               request_data=None,\n               client_id=None,\n               base_session_id=None,\n               **kwargs):\n    \"\"\"Creates a new flow and send its responses to a state.\n\n    This creates a new flow. The flow may send back many responses which will be\n    queued by the framework until the flow terminates. The final status message\n    will cause the entire transaction to be committed to the specified state.\n\n    Args:\n       flow_name: The name of the flow to invoke.\n       next_state: The state in this flow, that responses to this message should\n         go to.\n       request_data: Any dict provided here will be available in the\n         RequestState protobuf. The Responses object maintains a reference to\n         this protobuf for use in the execution of the state method. (so you can\n         access this data by responses.request). There is no format mandated on\n         this data but it may be a serialized protobuf.\n       client_id: If given, the flow is started for this client.\n       base_session_id: A URN which will be used to build a URN.\n       **kwargs: Arguments for the child flow.\n\n    Returns:\n       The flow_id of the child flow which was created.\n\n    Raises:\n      ValueError: The requested next state does not exist.\n    \"\"\"\n    if not getattr(self, next_state):\n      raise ValueError(\"Next state %s is invalid.\" % next_state)\n\n    flow_request = rdf_flow_objects.FlowRequest(\n        client_id=self.rdf_flow.client_id,\n        flow_id=self.rdf_flow.flow_id,\n        request_id=self.GetNextOutboundId(),\n        next_state=next_state)\n\n    if request_data is not None:\n      flow_request.request_data = rdf_protodict.Dict().FromDict(request_data)\n\n    self.flow_requests.append(flow_request)\n\n    flow_cls = registry.FlowRegistry.FlowClassByName(flow_name)\n\n    flow.StartFlow(\n        client_id=self.rdf_flow.client_id,\n        flow_cls=flow_cls,\n        parent_flow_obj=self,\n        **kwargs)"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nallows this flow to send a message to its parent flow.", "response": "def SendReply(self, response, tag=None):\n    \"\"\"Allows this flow to send a message to its parent flow.\n\n    If this flow does not have a parent, the message is ignored.\n\n    Args:\n      response: An RDFValue() instance to be sent to the parent.\n      tag: If specified, tag the result with this tag.\n\n    Raises:\n      ValueError: If responses is not of the correct type.\n    \"\"\"\n    if not isinstance(response, rdfvalue.RDFValue):\n      raise ValueError(\"SendReply can only send RDFValues\")\n\n    if self.rdf_flow.parent_flow_id:\n      response = rdf_flow_objects.FlowResponse(\n          client_id=self.rdf_flow.client_id,\n          request_id=self.rdf_flow.parent_request_id,\n          response_id=self.GetNextResponseId(),\n          payload=response,\n          flow_id=self.rdf_flow.parent_flow_id,\n          tag=tag)\n\n      self.flow_responses.append(response)\n    else:\n      reply = rdf_flow_objects.FlowResult(\n          client_id=self.rdf_flow.client_id,\n          flow_id=self.rdf_flow.flow_id,\n          hunt_id=self.rdf_flow.parent_hunt_id,\n          payload=response,\n          tag=tag)\n      self.replies_to_write.append(reply)\n      self.replies_to_process.append(reply)\n\n    self.rdf_flow.num_replies_sent += 1"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef SaveResourceUsage(self, status):\n    user_cpu = status.cpu_time_used.user_cpu_time\n    system_cpu = status.cpu_time_used.system_cpu_time\n    self.rdf_flow.cpu_time_used.user_cpu_time += user_cpu\n    self.rdf_flow.cpu_time_used.system_cpu_time += system_cpu\n\n    self.rdf_flow.network_bytes_sent += status.network_bytes_sent\n\n    if self.rdf_flow.cpu_limit:\n      user_cpu_total = self.rdf_flow.cpu_time_used.user_cpu_time\n      system_cpu_total = self.rdf_flow.cpu_time_used.system_cpu_time\n      if self.rdf_flow.cpu_limit < (user_cpu_total + system_cpu_total):\n        # We have exceeded our limit, stop this flow.\n        raise flow.FlowError(\"CPU limit exceeded for {} {}.\".format(\n            self.rdf_flow.flow_class_name, self.rdf_flow.flow_id))\n\n    if (self.rdf_flow.network_bytes_limit and\n        self.rdf_flow.network_bytes_limit < self.rdf_flow.network_bytes_sent):\n      # We have exceeded our byte limit, stop this flow.\n      raise flow.FlowError(\"Network bytes limit exceeded {} {}.\".format(\n          self.rdf_flow.flow_class_name, self.rdf_flow.flow_id))", "response": "Method to tally resources."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef Error(self, error_message=None, backtrace=None, status=None):\n    stats_collector_instance.Get().IncrementCounter(\n        \"flow_errors\", fields=[compatibility.GetName(self.__class__)])\n\n    client_id = self.rdf_flow.client_id\n    flow_id = self.rdf_flow.flow_id\n\n    logging.error(\"Error in flow %s on %s: %s, %s\", flow_id, client_id,\n                  error_message, backtrace)\n\n    if self.rdf_flow.parent_flow_id or self.rdf_flow.parent_hunt_id:\n      status_msg = rdf_flow_objects.FlowStatus(\n          client_id=client_id,\n          request_id=self.rdf_flow.parent_request_id,\n          response_id=self.GetNextResponseId(),\n          cpu_time_used=self.rdf_flow.cpu_time_used,\n          network_bytes_sent=self.rdf_flow.network_bytes_sent,\n          error_message=error_message,\n          flow_id=self.rdf_flow.parent_flow_id,\n          backtrace=backtrace)\n\n      if status is not None:\n        status_msg.status = status\n      else:\n        status_msg.status = rdf_flow_objects.FlowStatus.Status.ERROR\n\n      if self.rdf_flow.parent_flow_id:\n        self.flow_responses.append(status_msg)\n      elif self.rdf_flow.parent_hunt_id:\n        db_compat.ProcessHuntFlowError(\n            self.rdf_flow,\n            error_message=error_message,\n            backtrace=backtrace,\n            status_msg=status_msg)\n\n    self.rdf_flow.flow_state = self.rdf_flow.FlowState.ERROR\n    if backtrace is not None:\n      self.rdf_flow.backtrace = backtrace\n    if error_message is not None:\n      self.rdf_flow.error_message = error_message\n\n    self.NotifyCreatorOfError()", "response": "Terminates this flow with an error."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nmarking this flow as done.", "response": "def MarkDone(self, status=None):\n    \"\"\"Marks this flow as done.\"\"\"\n    stats_collector_instance.Get().IncrementCounter(\n        \"flow_completions\", fields=[compatibility.GetName(self.__class__)])\n\n    # Notify our parent flow or hunt that we are done (if there's a parent flow\n    # or hunt).\n    if self.rdf_flow.parent_flow_id or self.rdf_flow.parent_hunt_id:\n      status = rdf_flow_objects.FlowStatus(\n          client_id=self.rdf_flow.client_id,\n          request_id=self.rdf_flow.parent_request_id,\n          response_id=self.GetNextResponseId(),\n          status=rdf_flow_objects.FlowStatus.Status.OK,\n          cpu_time_used=self.rdf_flow.cpu_time_used,\n          network_bytes_sent=self.rdf_flow.network_bytes_sent,\n          flow_id=self.rdf_flow.parent_flow_id)\n      if self.rdf_flow.parent_flow_id:\n        self.flow_responses.append(status)\n      elif self.rdf_flow.parent_hunt_id:\n        db_compat.ProcessHuntFlowDone(self.rdf_flow, status_msg=status)\n\n    self.rdf_flow.flow_state = self.rdf_flow.FlowState.FINISHED\n\n    if self.ShouldSendNotifications():\n      self.NotifyAboutEnd()"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nlogs the message using the standard logging.", "response": "def Log(self, format_str, *args):\n    \"\"\"Logs the message using the flow's standard logging.\n\n    Args:\n      format_str: Format string\n      *args: arguments to the format string\n    \"\"\"\n    log_entry = rdf_flow_objects.FlowLogEntry(\n        client_id=self.rdf_flow.client_id,\n        flow_id=self.rdf_flow.flow_id,\n        hunt_id=self.rdf_flow.parent_hunt_id,\n        message=format_str % args)\n    data_store.REL_DB.WriteFlowLogEntries([log_entry])\n    if self.rdf_flow.parent_hunt_id:\n      db_compat.ProcessHuntFlowLog(self.rdf_flow, format_str % args)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef RunStateMethod(self, method_name, request=None, responses=None):\n    if self.rdf_flow.pending_termination:\n      self.Error(error_message=self.rdf_flow.pending_termination.reason)\n      return\n\n    client_id = self.rdf_flow.client_id\n\n    deadline = self.rdf_flow.processing_deadline\n    if deadline and rdfvalue.RDFDatetime.Now() > deadline:\n      raise flow.FlowError(\"Processing time for flow %s on %s expired.\" %\n                           (self.rdf_flow.flow_id, self.rdf_flow.client_id))\n\n    self.rdf_flow.current_state = method_name\n    if request and responses:\n      logging.debug(\"Running %s for flow %s on %s, %d responses.\", method_name,\n                    self.rdf_flow.flow_id, client_id, len(responses))\n    else:\n      logging.debug(\"Running %s for flow %s on %s\", method_name,\n                    self.rdf_flow.flow_id, client_id)\n\n    try:\n      try:\n        method = getattr(self, method_name)\n      except AttributeError:\n        raise ValueError(\"Flow %s has no state method %s\" %\n                         (self.__class__.__name__, method_name))\n\n      # Prepare a responses object for the state method to use:\n      responses = flow_responses.Responses.FromResponses(\n          request=request, responses=responses)\n\n      if responses.status is not None:\n        self.SaveResourceUsage(responses.status)\n\n      stats_collector_instance.Get().IncrementCounter(\"grr_worker_states_run\")\n\n      if method_name == \"Start\":\n        stats_collector_instance.Get().IncrementCounter(\n            \"flow_starts\", fields=[self.rdf_flow.flow_class_name])\n        method()\n      else:\n        method(responses)\n\n      if self.replies_to_process:\n        if self.rdf_flow.parent_hunt_id and not self.rdf_flow.parent_flow_id:\n          self._ProcessRepliesWithHuntOutputPlugins(self.replies_to_process)\n        else:\n          self._ProcessRepliesWithFlowOutputPlugins(self.replies_to_process)\n\n        self.replies_to_process = []\n\n    # We don't know here what exceptions can be thrown in the flow but we have\n    # to continue. Thus, we catch everything.\n    except Exception as e:  # pylint: disable=broad-except\n      # This flow will terminate now\n      stats_collector_instance.Get().IncrementCounter(\n          \"flow_errors\", fields=[self.rdf_flow.flow_class_name])\n      logging.exception(\"Flow %s on %s raised %s.\", self.rdf_flow.flow_id,\n                        client_id, utils.SmartUnicode(e))\n\n      self.Error(\n          error_message=utils.SmartUnicode(e), backtrace=traceback.format_exc())", "response": "Runs the given method on the current state."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nprocess all requests that are due to run.", "response": "def ProcessAllReadyRequests(self):\n    \"\"\"Processes all requests that are due to run.\n\n    Returns:\n      The number of processed requests.\n    \"\"\"\n    request_dict = data_store.REL_DB.ReadFlowRequestsReadyForProcessing(\n        self.rdf_flow.client_id,\n        self.rdf_flow.flow_id,\n        next_needed_request=self.rdf_flow.next_request_to_process)\n    if not request_dict:\n      return 0\n\n    processed = 0\n    while self.rdf_flow.next_request_to_process in request_dict:\n      request, responses = request_dict[self.rdf_flow.next_request_to_process]\n      self.RunStateMethod(request.next_state, request, responses)\n      self.rdf_flow.next_request_to_process += 1\n      processed += 1\n      self.completed_requests.append(request)\n\n    if processed and self.IsRunning() and not self.outstanding_requests:\n      self.RunStateMethod(\"End\")\n      if (self.rdf_flow.flow_state == self.rdf_flow.FlowState.RUNNING and\n          not self.outstanding_requests):\n        self.MarkDone()\n\n    self.PersistState()\n\n    if not self.IsRunning():\n      # All requests and responses can now be deleted.\n      self._ClearAllRequestsAndResponses()\n\n    return processed"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nprocessing replies with output plugins.", "response": "def _ProcessRepliesWithFlowOutputPlugins(self, replies):\n    \"\"\"Processes replies with output plugins.\"\"\"\n    created_output_plugins = []\n    for index, output_plugin_state in enumerate(\n        self.rdf_flow.output_plugins_states):\n      plugin_descriptor = output_plugin_state.plugin_descriptor\n      output_plugin_cls = plugin_descriptor.GetPluginClass()\n      output_plugin = output_plugin_cls(\n          source_urn=self.rdf_flow.long_flow_id,\n          args=plugin_descriptor.plugin_args,\n          token=access_control.ACLToken(username=self.rdf_flow.creator))\n\n      try:\n        # TODO(user): refactor output plugins to use FlowResponse\n        # instead of GrrMessage.\n        output_plugin.ProcessResponses(\n            output_plugin_state.plugin_state,\n            [r.AsLegacyGrrMessage() for r in replies])\n        output_plugin.Flush(output_plugin_state.plugin_state)\n        output_plugin.UpdateState(output_plugin_state.plugin_state)\n\n        data_store.REL_DB.WriteFlowOutputPluginLogEntries([\n            rdf_flow_objects.FlowOutputPluginLogEntry(\n                client_id=self.rdf_flow.client_id,\n                flow_id=self.rdf_flow.flow_id,\n                hunt_id=self.rdf_flow.parent_hunt_id,\n                output_plugin_id=\"%d\" % index,\n                log_entry_type=rdf_flow_objects.FlowOutputPluginLogEntry\n                .LogEntryType.LOG,\n                message=\"Processed %d replies.\" % len(replies))\n        ])\n\n        self.Log(\"Plugin %s successfully processed %d flow replies.\",\n                 plugin_descriptor, len(replies))\n\n        created_output_plugins.append(output_plugin)\n      except Exception as e:  # pylint: disable=broad-except\n        logging.exception(\"Plugin %s failed to process %d replies: %s\",\n                          plugin_descriptor, len(replies),\n                          traceback.format_exc())\n        created_output_plugins.append(None)\n\n        data_store.REL_DB.WriteFlowOutputPluginLogEntries([\n            rdf_flow_objects.FlowOutputPluginLogEntry(\n                client_id=self.rdf_flow.client_id,\n                flow_id=self.rdf_flow.flow_id,\n                hunt_id=self.rdf_flow.parent_hunt_id,\n                output_plugin_id=\"%d\" % index,\n                log_entry_type=rdf_flow_objects.FlowOutputPluginLogEntry\n                .LogEntryType.ERROR,\n                message=\"Error while processing %d replies: %s\" %\n                (len(replies), str(e)))\n        ])\n\n        self.Log(\"Plugin %s failed to process %d replies due to: %s\",\n                 plugin_descriptor, len(replies), e)\n\n    return created_output_plugins"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _SignedBinaryIDFromURN(binary_urn\n                          ):\n  \"\"\"Converts an AFF4 URN for a signed binary to a SignedBinaryID.\"\"\"\n  if binary_urn.RelativeName(GetAFF4PythonHackRoot()):\n    return rdf_objects.SignedBinaryID(\n        binary_type=rdf_objects.SignedBinaryID.BinaryType.PYTHON_HACK,\n        path=binary_urn.RelativeName(GetAFF4PythonHackRoot()))\n  elif binary_urn.RelativeName(GetAFF4ExecutablesRoot()):\n    return rdf_objects.SignedBinaryID(\n        binary_type=rdf_objects.SignedBinaryID.BinaryType.EXECUTABLE,\n        path=binary_urn.RelativeName(GetAFF4ExecutablesRoot()))\n  else:\n    raise ValueError(\"Unable to determine type of signed binary: %s.\" %\n                     binary_urn)", "response": "Converts an AFF4 URN for a signed binary to a SignedBinaryID."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nconverts a SignedBinaryID to the equivalent AFF4 URN.", "response": "def _SignedBinaryURNFromID(binary_id\n                          ):\n  \"\"\"Converts a SignedBinaryID to the equivalent AFF4 URN.\"\"\"\n  binary_type = binary_id.binary_type\n  if binary_type == rdf_objects.SignedBinaryID.BinaryType.PYTHON_HACK:\n    return GetAFF4PythonHackRoot().Add(binary_id.path)\n  elif binary_type == rdf_objects.SignedBinaryID.BinaryType.EXECUTABLE:\n    return GetAFF4ExecutablesRoot().Add(binary_id.path)\n  else:\n    raise ValueError(\"Unknown binary type %s.\" % binary_type)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef WriteSignedBinary(binary_urn,\n                      binary_content,\n                      private_key,\n                      public_key,\n                      chunk_size = 1024,\n                      token = None):\n  \"\"\"Signs a binary and saves it to the datastore.\n\n  If a signed binary with the given URN already exists, its contents will get\n  overwritten.\n\n  Args:\n    binary_urn: URN that should serve as a unique identifier for the binary.\n    binary_content: Contents of the binary, as raw bytes.\n    private_key: Key that should be used for signing the binary contents.\n    public_key: Key that should be used to verify the signature generated using\n      the private key.\n    chunk_size: Size, in bytes, of the individual blobs that the binary contents\n      will be split to before saving to the datastore.\n    token: ACL token to use with the legacy (non-relational) datastore.\n  \"\"\"\n  if _ShouldUseLegacyDatastore():\n    collects.GRRSignedBlob.NewFromContent(\n        binary_content,\n        binary_urn,\n        chunk_size=chunk_size,\n        token=token,\n        private_key=private_key,\n        public_key=public_key)\n\n  if data_store.RelationalDBEnabled():\n    blob_references = rdf_objects.BlobReferences()\n    for chunk_offset in range(0, len(binary_content), chunk_size):\n      chunk = binary_content[chunk_offset:chunk_offset + chunk_size]\n      blob_rdf = rdf_crypto.SignedBlob()\n      blob_rdf.Sign(chunk, private_key, verify_key=public_key)\n      blob_id = data_store.BLOBS.WriteBlobWithUnknownHash(\n          blob_rdf.SerializeToString())\n      blob_references.items.Append(\n          rdf_objects.BlobReference(\n              offset=chunk_offset, size=len(chunk), blob_id=blob_id))\n    data_store.REL_DB.WriteSignedBinaryReferences(\n        _SignedBinaryIDFromURN(binary_urn), blob_references)", "response": "Signs a binary and saves it to the datastore."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nwriting signed blobs to the datastore.", "response": "def WriteSignedBinaryBlobs(binary_urn,\n                           blobs,\n                           token = None):\n  \"\"\"Saves signed blobs to the datastore.\n\n  If a signed binary with the given URN already exists, its contents will get\n  overwritten.\n\n  Args:\n    binary_urn: RDFURN that should serve as a unique identifier for the binary.\n    blobs: An Iterable of signed blobs to write to the datastore.\n    token: ACL token to use with the legacy (non-relational) datastore.\n  \"\"\"\n  if _ShouldUseLegacyDatastore():\n    aff4.FACTORY.Delete(binary_urn, token=token)\n    with data_store.DB.GetMutationPool() as mutation_pool:\n      with aff4.FACTORY.Create(\n          binary_urn,\n          collects.GRRSignedBlob,\n          mode=\"w\",\n          mutation_pool=mutation_pool,\n          token=token) as fd:\n        for blob in blobs:\n          fd.Add(blob, mutation_pool=mutation_pool)\n\n  if data_store.RelationalDBEnabled():\n    blob_references = rdf_objects.BlobReferences()\n    current_offset = 0\n    for blob in blobs:\n      blob_id = data_store.BLOBS.WriteBlobWithUnknownHash(\n          blob.SerializeToString())\n      blob_references.items.Append(\n          rdf_objects.BlobReference(\n              offset=current_offset, size=len(blob.data), blob_id=blob_id))\n      current_offset += len(blob.data)\n    data_store.REL_DB.WriteSignedBinaryReferences(\n        _SignedBinaryIDFromURN(binary_urn), blob_references)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ndelete the signed binary with the given urn from the datastore.", "response": "def DeleteSignedBinary(binary_urn,\n                       token = None):\n  \"\"\"Deletes the binary with the given urn from the datastore.\n\n  Args:\n    binary_urn: RDFURN that serves as a unique identifier for the binary.\n    token: ACL token to use with the legacy (non-relational) datastore.\n\n  Raises:\n    SignedBinaryNotFoundError: If the signed binary does not exist.\n  \"\"\"\n  if _ShouldUseLegacyDatastore():\n    try:\n      aff4.FACTORY.Open(\n          binary_urn, aff4_type=aff4.AFF4Stream, mode=\"r\", token=token)\n    except aff4.InstantiationError:\n      raise SignedBinaryNotFoundError(binary_urn)\n    aff4.FACTORY.Delete(binary_urn, token=token)\n\n  if data_store.RelationalDBEnabled():\n    try:\n      data_store.REL_DB.ReadSignedBinaryReferences(\n          _SignedBinaryIDFromURN(binary_urn))\n    except db.UnknownSignedBinaryError:\n      if _ShouldUseLegacyDatastore():\n        # Migration of data isn't complete yet (we haven't started reading\n        # exclusively from the relational DB), so this is probably ok.\n        return\n      else:\n        raise SignedBinaryNotFoundError(binary_urn)\n    data_store.REL_DB.DeleteSignedBinaryReferences(\n        _SignedBinaryIDFromURN(binary_urn))"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns the URNs for all signed binaries in the datastore.", "response": "def FetchURNsForAllSignedBinaries(token\n                                 ):\n  \"\"\"Returns URNs for all signed binaries in the datastore.\n\n  Args:\n    token: ACL token to use with the legacy (non-relational) datastore.\n  \"\"\"\n  if _ShouldUseLegacyDatastore():\n    urns = []\n    aff4_roots = [GetAFF4PythonHackRoot(), GetAFF4ExecutablesRoot()]\n    for _, descendant_urns in aff4.FACTORY.RecursiveMultiListChildren(\n        aff4_roots):\n      urns.extend(descendant_urns)\n    aff4_streams = aff4.FACTORY.MultiOpen(\n        urns, aff4_type=collects.GRRSignedBlob, mode=\"r\", token=token)\n    return [stream.urn for stream in aff4_streams]\n  else:\n    return [\n        _SignedBinaryURNFromID(i)\n        for i in data_store.REL_DB.ReadIDsForAllSignedBinaries()\n    ]"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef FetchBlobsForSignedBinary(\n    binary_urn,\n    token = None\n):\n  \"\"\"Retrieves blobs for the given binary from the datastore.\n\n  Args:\n    binary_urn: RDFURN that uniquely identifies the binary.\n    token: ACL token to use with the legacy (non-relational) datastore.\n\n  Returns:\n    A tuple containing an iterator for all the binary's blobs and an\n    RDFDatetime representing when the binary's contents were saved\n    to the datastore.\n\n  Raises:\n    SignedBinaryNotFoundError: If no signed binary with the given URN exists.\n  \"\"\"\n  if _ShouldUseLegacyDatastore():\n    try:\n      aff4_stream = aff4.FACTORY.Open(\n          binary_urn, aff4_type=collects.GRRSignedBlob, mode=\"r\", token=token)\n    except aff4.InstantiationError:\n      raise SignedBinaryNotFoundError(binary_urn)\n    timestamp = aff4_stream.Get(aff4_stream.Schema.TYPE).age\n    return (blob for blob in aff4_stream), timestamp\n  else:\n    try:\n      references, timestamp = data_store.REL_DB.ReadSignedBinaryReferences(\n          _SignedBinaryIDFromURN(binary_urn))\n    except db.UnknownSignedBinaryError:\n      raise SignedBinaryNotFoundError(binary_urn)\n    blob_ids = [r.blob_id for r in references.items]\n    raw_blobs = (data_store.BLOBS.ReadBlob(blob_id) for blob_id in blob_ids)\n    blobs = (\n        rdf_crypto.SignedBlob.FromSerializedString(raw_blob)\n        for raw_blob in raw_blobs)\n    return blobs, timestamp", "response": "Retrieves blobs for the given signed binary."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nfetching the size of the given signed binary.", "response": "def FetchSizeOfSignedBinary(binary_urn,\n                            token = None\n                           ):\n  \"\"\"Returns the size of the given binary (in bytes).\n\n  Args:\n    binary_urn: RDFURN that uniquely identifies the binary.\n    token: ACL token to use with the legacy (non-relational) datastore.\n\n  Raises:\n    SignedBinaryNotFoundError: If no signed binary with the given URN exists.\n  \"\"\"\n  if _ShouldUseLegacyDatastore():\n    try:\n      aff4_stream = aff4.FACTORY.Open(\n          binary_urn, aff4_type=collects.GRRSignedBlob, mode=\"r\", token=token)\n      return aff4_stream.size\n    except aff4.InstantiationError:\n      raise SignedBinaryNotFoundError(binary_urn)\n  else:\n    try:\n      references, _ = data_store.REL_DB.ReadSignedBinaryReferences(\n          _SignedBinaryIDFromURN(binary_urn))\n    except db.UnknownSignedBinaryError:\n      raise SignedBinaryNotFoundError(binary_urn)\n    last_reference = references.items[-1]\n    return last_reference.offset + last_reference.size"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning an iterator over all the blobs in the given binary.", "response": "def StreamSignedBinaryContents(blob_iterator,\n                               chunk_size = 1024\n                              ):\n  \"\"\"Yields the contents of the given binary in chunks of the given size.\n\n  Args:\n    blob_iterator: An Iterator over all the binary's blobs.\n    chunk_size: Size, in bytes, of the chunks to yield.\n  \"\"\"\n  all_blobs_read = False\n  byte_buffer = io.BytesIO()\n  while not all_blobs_read or byte_buffer.getvalue():\n    while not all_blobs_read and byte_buffer.tell() < chunk_size:\n      try:\n        blob = next(blob_iterator)\n      except StopIteration:\n        all_blobs_read = True\n        break\n      byte_buffer.write(blob.data)\n    if byte_buffer.tell() > 0:\n      # Yield a chunk of the signed binary and reset the buffer to contain\n      # only data that hasn't been sent yet.\n      byte_buffer.seek(0)\n      yield byte_buffer.read(chunk_size)\n      byte_buffer = io.BytesIO(byte_buffer.read())\n      byte_buffer.seek(0, io.SEEK_END)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ncreate or overwrites blobs.", "response": "def WriteBlobs(self, blob_id_data_map):\n    \"\"\"Creates or overwrites blobs.\"\"\"\n\n    urns = {self._BlobUrn(blob_id): blob_id for blob_id in blob_id_data_map}\n\n    mutation_pool = data_store.DB.GetMutationPool()\n\n    existing = aff4.FACTORY.MultiOpen(\n        urns, aff4_type=aff4.AFF4MemoryStreamBase, mode=\"r\")\n\n    for blob_urn, blob_id in iteritems(urns):\n      if blob_urn in existing:\n        logging.debug(\"Blob %s already stored.\", blob_id)\n        continue\n\n      with aff4.FACTORY.Create(\n          blob_urn,\n          aff4.AFF4UnversionedMemoryStream,\n          mode=\"w\",\n          mutation_pool=mutation_pool) as fd:\n        content = blob_id_data_map[blob_id]\n        fd.Write(content)\n\n      logging.debug(\"Got blob %s (length %s)\", blob_id.AsHexString(),\n                    len(content))\n\n    mutation_pool.Flush()"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef CheckBlobsExist(self, blob_ids):\n    res = {blob_id: False for blob_id in blob_ids}\n\n    urns = {self._BlobUrn(blob_id): blob_id for blob_id in blob_ids}\n\n    existing = aff4.FACTORY.MultiOpen(\n        urns, aff4_type=aff4.AFF4MemoryStreamBase, mode=\"r\")\n\n    for blob in existing:\n      res[urns[blob.urn]] = True\n\n    return res", "response": "Checks if blobs for the given digests already exist."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef NamedPlaceholders(iterable):\n  placeholders = \", \".join(\"%({})s\".format(key) for key in sorted(iterable))\n  return \"({})\".format(placeholders)", "response": "Returns a string containing named placeholders from all elements of the given iterable."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef Columns(iterable):\n  columns = sorted(iterable)\n  return \"({})\".format(\", \".join(\"`{}`\".format(col) for col in columns))", "response": "Returns a string containing a tuple of column names for MySQL INSERTs."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nconvert MySQL TIMESTAMP columns to RDFDatetime objects.", "response": "def TimestampToRDFDatetime(timestamp):\n  \"\"\"Converts MySQL `TIMESTAMP(6)` columns to datetime objects.\"\"\"\n  # TODO(hanuszczak): `timestamp` should be of MySQL type `Decimal`. However,\n  # it is unclear where this type is actually defined and how to import it in\n  # order to have a type assertion.\n  if timestamp is None:\n    return None\n  else:\n    micros = int(1000000 * timestamp)\n    return rdfvalue.RDFDatetime.FromMicrosecondsSinceEpoch(micros)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef ComponentsToPath(components):\n  precondition.AssertIterableType(components, Text)\n  for component in components:\n    if not component:\n      raise ValueError(\"Empty path component in: {}\".format(components))\n\n    if \"/\" in component:\n      raise ValueError(\"Path component with '/' in: {}\".format(components))\n\n  if components:\n    return \"/\" + \"/\".join(components)\n  else:\n    return \"\"", "response": "Converts a list of path components to a canonical MySQL path representation."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef PathToComponents(path):\n  precondition.AssertType(path, Text)\n  if path and not path.startswith(\"/\"):\n    raise ValueError(\"Path '{}' is not absolute\".format(path))\n\n  if path:\n    return tuple(path.split(\"/\")[1:])\n  else:\n    return ()", "response": "Converts a MySQL path representation to a list of components."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nlogs any invalid run states found.", "response": "def _LogInvalidRunLevels(states, valid):\n  \"\"\"Log any invalid run states found.\"\"\"\n  invalid = set()\n  for state in states:\n    if state not in valid:\n      invalid.add(state)\n  if invalid:\n    logging.warning(\"Invalid init runlevel(s) encountered: %s\",\n                    \", \".join(invalid))"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\naccept a string and returns a list of strings of numeric LSB runlevels.", "response": "def GetRunlevelsLSB(states):\n  \"\"\"Accepts a string and returns a list of strings of numeric LSB runlevels.\"\"\"\n  if not states:\n    return set()\n  valid = set([\"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\"])\n  _LogInvalidRunLevels(states, valid)\n  return valid.intersection(set(states.split()))"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef GetRunlevelsNonLSB(states):\n  if not states:\n    return set()\n  convert_table = {\n      \"0\": \"0\",\n      \"1\": \"1\",\n      \"2\": \"2\",\n      \"3\": \"3\",\n      \"4\": \"4\",\n      \"5\": \"5\",\n      \"6\": \"6\",\n      # SysV, Gentoo, Solaris, HP-UX all allow an alpha variant\n      # for single user. https://en.wikipedia.org/wiki/Runlevel\n      \"S\": \"1\",\n      \"s\": \"1\"\n  }\n  _LogInvalidRunLevels(states, convert_table)\n  return set([convert_table[s] for s in states.split() if s in convert_table])", "response": "Accepts a string and returns a list of strings of numeric LSB runlevels."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nparse the INSserv file and returns a list of services and facilities.", "response": "def _ParseInsserv(self, data):\n    \"\"\"/etc/insserv.conf* entries define system facilities.\n\n    Full format details are in man 8 insserv, but the basic structure is:\n      $variable          facility1 facility2\n      $second_variable   facility3 $variable\n\n    Any init script that specifies Required-Start: $second_variable needs to be\n    expanded to facility1 facility2 facility3.\n\n    Args:\n      data: A string of insserv definitions.\n    \"\"\"\n    p = config_file.FieldParser()\n    entries = p.ParseEntries(data)\n    raw = {e[0]: e[1:] for e in entries}\n    # Now expand out the facilities to services.\n    facilities = {}\n    for k, v in iteritems(raw):\n      # Remove interactive tags.\n      k = k.replace(\"<\", \"\").replace(\">\", \"\")\n      facilities[k] = v\n    for k, vals in iteritems(facilities):\n      self.insserv[k] = []\n      for v in vals:\n        self.insserv[k].extend(self._InsservExpander(facilities, v))"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nextracts entries from the xinetd config files.", "response": "def _ProcessEntries(self, fd):\n    \"\"\"Extract entries from the xinetd config files.\"\"\"\n    p = config_file.KeyValueParser(kv_sep=\"{\", term=\"}\", sep=None)\n    data = utils.ReadFileBytesAsUnicode(fd)\n    entries = p.ParseEntries(data)\n    for entry in entries:\n      for section, cfg in iteritems(entry):\n        # The parser returns a list of configs. There will only be one.\n        if cfg:\n          cfg = cfg[0].strip()\n        else:\n          cfg = \"\"\n        self._ParseSection(section, cfg)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ninterpolating configurations with defaults to generate actual configs.", "response": "def _GenConfig(self, cfg):\n    \"\"\"Interpolate configurations with defaults to generate actual configs.\"\"\"\n    # Some setting names may have a + or - suffix. These indicate that the\n    # settings modify the default values.\n    merged = self.default.copy()\n    for setting, vals in iteritems(cfg):\n      option, operator = (setting.split(None, 1) + [None])[:2]\n      vals = set(vals)\n      default = set(self.default.get(option, []))\n      # If there is an operator, updated values accordingly.\n      if operator == \"+\":\n        vals = default.union(vals)\n      elif operator == \"-\":\n        vals = default.difference(vals)\n      merged[option] = list(vals)\n    return rdf_protodict.AttributedDict(**merged)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef ParseMultiple(self, stats, unused_file_obj, unused_kb):\n    services = {}\n    for stat_entry in stats:\n      path = stat_entry.pathspec.path\n      runlevel = self.runlevel_re.match(os.path.dirname(path))\n      runscript = self.runscript_re.match(os.path.basename(path))\n      if runlevel and runscript:\n        svc = runscript.groupdict()\n        service = services.setdefault(\n            svc[\"name\"],\n            rdf_client.LinuxServiceInformation(\n                name=svc[\"name\"], start_mode=\"INIT\"))\n        runlvl = GetRunlevelsNonLSB(runlevel.group(1))\n        if svc[\"action\"] == \"S\" and runlvl:\n          service.start_on.append(runlvl.pop())\n          service.starts = True\n        elif runlvl:\n          service.stop_on.append(runlvl.pop())\n        if not stat.S_ISLNK(int(stat_entry.st_mode)):\n          yield rdf_anomaly.Anomaly(\n              type=\"PARSER_ANOMALY\",\n              finding=[path],\n              explanation=\"Startup script is not a symlink.\")\n    for svc in itervalues(services):\n      yield svc", "response": "Parse a single file."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef CheckClientLabels(client_id,\n                      labels_whitelist=None,\n                      labels_owners_whitelist=None,\n                      token=None):\n  \"\"\"Checks a given client against labels/owners whitelists.\"\"\"\n\n  labels_whitelist = labels_whitelist or []\n  labels_owners_whitelist = labels_owners_whitelist or []\n\n  if data_store.RelationalDBEnabled():\n    labels = data_store.REL_DB.ReadClientLabels(str(client_id))\n  else:\n    with aff4.FACTORY.Open(\n        client_id.ToClientURN(), aff4_type=aff4_grr.VFSGRRClient,\n        token=token) as fd:\n      labels = fd.GetLabels()\n\n  for label in labels:\n    if (label.name in labels_whitelist and\n        label.owner in labels_owners_whitelist):\n      return\n\n  raise access_control.UnauthorizedAccess(\n      \"Client %s doesn't have necessary labels.\" % utils.SmartStr(client_id))", "response": "Checks a given client against labels and owners whitelists."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _IsRetryable(error):\n  if not isinstance(error, MySQLdb.OperationalError):\n    return False\n  if not error.args:\n    return False\n  code = error.args[0]\n  return code in _RETRYABLE_ERRORS", "response": "Returns whether the error is likely to be retryable."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _CheckCollation(cursor):\n\n  # Do not fail for wrong collation, because changing it is harder than changing\n  # the character set. Some providers only allow changing character set and then\n  # use the default collation. Also, misconfigured collation is not expected to\n  # have major negative impacts, since it only affects string sort order for\n  # some locales.\n\n  cur_collation_connection = _ReadVariable(\"collation_connection\", cursor)\n  if cur_collation_connection != COLLATION:\n    logging.warning(\"Require MySQL collation_connection of %s, got %s.\",\n                    COLLATION, cur_collation_connection)\n\n  cur_collation_database = _ReadVariable(\"collation_database\", cursor)\n  if cur_collation_database != COLLATION:\n    logging.warning(\n        \"Require MySQL collation_database of %s, got %s.\"\n        \" To create your database, use: %s\", COLLATION, cur_collation_database,\n        CREATE_DATABASE_QUERY)", "response": "Checks MySQL collation and warns if misconfigured."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nenforces a sane UTF - 8 encoding for the database connection.", "response": "def _CheckConnectionEncoding(cursor):\n  \"\"\"Enforces a sane UTF-8 encoding for the database connection.\"\"\"\n  cur_character_set = _ReadVariable(\"character_set_connection\", cursor)\n  if cur_character_set != CHARACTER_SET:\n    raise EncodingEnforcementError(\n        \"Require MySQL character_set_connection of {}, got {}.\".format(\n            CHARACTER_SET, cur_character_set))"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _CheckDatabaseEncoding(cursor):\n  cur_character_set = _ReadVariable(\"character_set_database\", cursor)\n  if cur_character_set != CHARACTER_SET:\n    raise EncodingEnforcementError(\n        \"Require MySQL character_set_database of {}, got {}.\"\n        \" To create your database, use: {}\".format(CHARACTER_SET,\n                                                   cur_character_set,\n                                                   CREATE_DATABASE_QUERY))", "response": "Enforces a sane UTF - 8 encoding for the database."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nsetting the max_allowed_packet variable globally for new connections.", "response": "def _SetPacketSizeForFollowingConnections(cursor):\n  \"\"\"Sets max_allowed_packet globally for new connections (not current!).\"\"\"\n  cur_packet_size = int(_ReadVariable(\"max_allowed_packet\", cursor))\n\n  if cur_packet_size < MAX_PACKET_SIZE:\n    logging.warning(\n        \"MySQL max_allowed_packet of %d is required, got %d. Overwriting.\",\n        MAX_PACKET_SIZE, cur_packet_size)\n    _SetGlobalVariable(\"max_allowed_packet\", MAX_PACKET_SIZE, cursor)"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nchecks that MySQL packet size is big enough for expected query size.", "response": "def _CheckPacketSize(cursor):\n  \"\"\"Checks that MySQL packet size is big enough for expected query size.\"\"\"\n  cur_packet_size = int(_ReadVariable(\"max_allowed_packet\", cursor))\n  if cur_packet_size < MAX_PACKET_SIZE:\n    raise Error(\n        \"MySQL max_allowed_packet of {0} is required, got {1}. \"\n        \"Please set max_allowed_packet={0} in your MySQL config.\".format(\n            MAX_PACKET_SIZE, cur_packet_size))"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _CheckLogFileSize(cursor):\n\n  # Do not fail, because users might not be able to change this for their\n  # database. Instead, warn the user about the impacts.\n\n  innodb_log_file_size = int(_ReadVariable(\"innodb_log_file_size\", cursor))\n  required_size = 10 * mysql_blobs.BLOB_CHUNK_SIZE\n  if innodb_log_file_size < required_size:\n    # See MySQL error 1118: The size of BLOB/TEXT data inserted in one\n    # transaction is greater than 10% of redo log size. Increase the redo log\n    # size using innodb_log_file_size.\n    max_blob_size = innodb_log_file_size / 10\n    max_blob_size_mib = max_blob_size / 2**20\n    logging.warning(\n        \"MySQL innodb_log_file_size of %d is required, got %d. \"\n        \"Storing Blobs bigger than %.4f MiB will fail.\", required_size,\n        innodb_log_file_size, max_blob_size_mib)", "response": "Warns if MySQL log file size is not large enough for blob insertions."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _IsMariaDB(cursor):\n  for variable in [\"version\", \"version_comment\"]:\n    cursor.execute(\"SHOW VARIABLES LIKE %s;\", (variable,))\n    version = cursor.fetchone()\n    if version and \"MariaDB\" in version[1]:\n      return True\n  return False", "response": "Checks if we are running against MariaDB."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nconnecting to the given MySQL host and create a utf8mb4_unicode_ci database.", "response": "def _SetupDatabase(host=None,\n                   port=None,\n                   user=None,\n                   password=None,\n                   database=None,\n                   client_key_path=None,\n                   client_cert_path=None,\n                   ca_cert_path=None):\n  \"\"\"Connect to the given MySQL host and create a utf8mb4_unicode_ci database.\n\n  Args:\n    host: The hostname to connect to.\n    port: The port to connect to.\n    user: The username to connect as.\n    password: The password to connect with.\n    database: The database name to create.\n    client_key_path: The path of the client private key file.\n    client_cert_path: The path of the client public key certificate file.\n    ca_cert_path: The path of the Certificate Authority (CA) certificate file.\n  \"\"\"\n  with contextlib.closing(\n      _Connect(\n          host=host,\n          port=port,\n          user=user,\n          password=password,\n          # No database should be specified in a connection that intends\n          # to create a database.\n          database=None,\n          client_key_path=client_key_path,\n          client_cert_path=client_cert_path,\n          ca_cert_path=ca_cert_path)) as conn:\n    with contextlib.closing(conn.cursor()) as cursor:\n      try:\n        cursor.execute(CREATE_DATABASE_QUERY.format(database))\n      except MySQLdb.MySQLError as e:\n        #  Statement might fail if database exists, this is fine.\n        if e.args[0] != mysql_error_constants.DB_CREATE_EXISTS:\n          raise\n\n      cursor.execute(\"USE {}\".format(database))\n      _CheckCollation(cursor)\n\n  def _MigrationConnect():\n    return _Connect(\n        host=host,\n        port=port,\n        user=user,\n        password=password,\n        database=database,\n        client_key_path=client_key_path,\n        client_cert_path=client_cert_path,\n        ca_cert_path=ca_cert_path)\n\n  mysql_migration.ProcessMigrations(_MigrationConnect,\n                                    config.CONFIG[\"Mysql.migrations_dir\"])"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _GetConnectionArgs(host=None,\n                       port=None,\n                       user=None,\n                       password=None,\n                       database=None,\n                       client_key_path=None,\n                       client_cert_path=None,\n                       ca_cert_path=None):\n  \"\"\"Builds connection arguments for MySQLdb.Connect function.\"\"\"\n  connection_args = dict(\n      autocommit=False, use_unicode=True, charset=CHARACTER_SET)\n\n  if host is not None:\n    connection_args[\"host\"] = host\n\n  if port is not None:\n    connection_args[\"port\"] = port\n\n  if user is not None:\n    connection_args[\"user\"] = user\n\n  if password is not None:\n    connection_args[\"passwd\"] = password\n\n  if database is not None:\n    connection_args[\"db\"] = database\n\n  if client_key_path is not None:\n    connection_args[\"ssl\"] = {\n        \"key\": client_key_path,\n        \"cert\": client_cert_path,\n        \"ca\": ca_cert_path,\n    }\n\n  return connection_args", "response": "Builds connection arguments for MySQLdb. Connect function."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nconnects to MySQL and check if server fulfills requirements.", "response": "def _Connect(host=None,\n             port=None,\n             user=None,\n             password=None,\n             database=None,\n             client_key_path=None,\n             client_cert_path=None,\n             ca_cert_path=None):\n  \"\"\"Connect to MySQL and check if server fulfills requirements.\"\"\"\n  connection_args = _GetConnectionArgs(\n      host=host,\n      port=port,\n      user=user,\n      password=password,\n      database=database,\n      client_key_path=client_key_path,\n      client_cert_path=client_cert_path,\n      ca_cert_path=ca_cert_path)\n\n  conn = MySQLdb.Connect(**connection_args)\n  with contextlib.closing(conn.cursor()) as cursor:\n    _CheckForSSL(cursor)\n    _SetMariaDBMode(cursor)\n    _SetSqlMode(cursor)\n    _SetBinlogFormat(cursor)\n    _SetPacketSizeForFollowingConnections(cursor)\n    _SetEncoding(cursor)\n    _CheckConnectionEncoding(cursor)\n    _CheckLogFileSize(cursor)\n\n  return conn"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nrun a function within a transaction.", "response": "def _RunInTransaction(self, function, readonly=False):\n    \"\"\"Runs function within a transaction.\n\n    Allocates a connection, begins a transaction on it and passes the connection\n    to function.\n\n    If function finishes without raising, the transaction is committed.\n\n    If function raises, the transaction will be rolled back, if a retryable\n    database error is raised, the operation may be repeated.\n\n    Args:\n      function: A function to be run, must accept a single MySQLdb.connection\n        parameter.\n      readonly: Indicates that only a readonly (snapshot) transaction is\n        required.\n\n    Returns:\n      The value returned by the last call to function.\n\n    Raises: Any exception raised by function.\n    \"\"\"\n    start_query = \"START TRANSACTION;\"\n    if readonly:\n      start_query = \"START TRANSACTION WITH CONSISTENT SNAPSHOT, READ ONLY;\"\n\n    for retry_count in range(_MAX_RETRY_COUNT):\n      with contextlib.closing(self.pool.get()) as connection:\n        try:\n          with contextlib.closing(connection.cursor()) as cursor:\n            cursor.execute(start_query)\n\n          ret = function(connection)\n\n          if not readonly:\n            connection.commit()\n          return ret\n        except MySQLdb.OperationalError as e:\n          connection.rollback()\n          # Re-raise if this was the last attempt.\n          if retry_count >= _MAX_RETRY_COUNT or not _IsRetryable(e):\n            raise\n      # Simple delay, with jitter.\n      #\n      # TODO(user): Move to something more elegant, e.g. integrate a\n      # general retry or backoff library.\n      time.sleep(random.uniform(1.0, 2.0) * math.pow(1.5, retry_count))\n    # Shouldn't happen, because we should have re-raised whatever caused the\n    # last try to fail.\n    raise Exception(\"Looped ended early - last exception swallowed.\")"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef WriteClientGraphSeries(self, graph_series,\n                             client_label,\n                             timestamp):\n    \"\"\"See db.Database.\"\"\"\n    series_key = (client_label, graph_series.report_type, timestamp.Copy())\n    self.client_graph_series[series_key] = graph_series.Copy()", "response": "Writes a graph series to the client graph series table."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef ReadAllClientGraphSeries(\n      self,\n      client_label,\n      report_type,\n      time_range = None,\n  ):\n    \"\"\"See db.Database.\"\"\"\n    series_with_timestamps = {}\n    for series_key, series in iteritems(self.client_graph_series):\n      series_label, series_type, timestamp = series_key\n      if series_label == client_label and series_type == report_type:\n        if time_range is not None and not time_range.Includes(timestamp):\n          continue\n        series_with_timestamps[timestamp.Copy()] = series.Copy()\n    return series_with_timestamps", "response": "Reads all client graph series for a given client label and report type."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef ReadMostRecentClientGraphSeries(self, client_label,\n                                      report_type\n                                     ):\n    \"\"\"See db.Database.\"\"\"\n    series_with_timestamps = self.ReadAllClientGraphSeries(\n        client_label, report_type)\n    if not series_with_timestamps:\n      return None\n    _, latest_series = list(sorted(iteritems(series_with_timestamps)))[-1]\n    return latest_series", "response": "Reads the most recent series for a given client label and report type."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef Set(self,\n          subject,\n          attribute,\n          value,\n          timestamp=None,\n          replace=True,\n          sync=True):\n    \"\"\"Set the value into the data store.\"\"\"\n    subject = utils.SmartUnicode(subject)\n\n    _ = sync\n    attribute = utils.SmartUnicode(attribute)\n\n    if timestamp is None or timestamp == self.NEWEST_TIMESTAMP:\n      timestamp = time.time() * 1000000\n\n    if subject not in self.subjects:\n      self.subjects[subject] = {}\n\n    if replace or attribute not in self.subjects[subject]:\n      self.subjects[subject][attribute] = []\n\n    encoded_value = self._value_converter.Encode(attribute, value)\n    self.subjects[subject][attribute].append([encoded_value, int(timestamp)])\n    self.subjects[subject][attribute].sort(key=lambda x: x[1])", "response": "Sets the value into the data store."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef ResolvePrefix(self, subject, attribute_prefix, timestamp=None,\n                    limit=None):\n    \"\"\"Resolve all attributes for a subject starting with a prefix.\"\"\"\n    subject = utils.SmartUnicode(subject)\n\n    if timestamp in [None, self.NEWEST_TIMESTAMP, self.ALL_TIMESTAMPS]:\n      start, end = 0, (2**63) - 1\n    # Does timestamp represent a range?\n    elif isinstance(timestamp, (list, tuple)):\n      start, end = timestamp  # pylint: disable=unpacking-non-sequence\n    else:\n      raise ValueError(\"Invalid timestamp: %s\" % timestamp)\n\n    start = int(start)\n    end = int(end)\n\n    # TODO(hanuszczak): Make this function accept only one attribute prefix and\n    # only a unicode object.\n    if isinstance(attribute_prefix, string_types):\n      attribute_prefix = [attribute_prefix]\n\n    try:\n      record = self.subjects[subject]\n    except KeyError:\n      return []\n\n    # Holds all the attributes which matched. Keys are attribute names, values\n    # are lists of timestamped data.\n    results = {}\n    nr_results = 0\n    for prefix in attribute_prefix:\n      for attribute, values in iteritems(record):\n        if limit and nr_results >= limit:\n          break\n        # TODO(hanuszczak): After resolving the TODO comment above this call to\n        # `unicode` should be redundant.\n        if str(attribute).startswith(prefix):\n          for encoded_value, ts in values:\n            results_list = results.setdefault(attribute, [])\n            # If we are always after the latest ts we clear older ones.\n            if (results_list and timestamp in [self.NEWEST_TIMESTAMP, None] and\n                results_list[0][1] < ts):\n              results_list = []\n              results[attribute] = results_list\n\n            # Timestamp outside the range, drop it.\n            elif ts < start or ts > end:\n              continue\n\n            value = self._value_converter.Decode(attribute, encoded_value)\n            results_list.append((attribute, ts, value))\n            nr_results += 1\n            if limit and nr_results >= limit:\n              break\n\n    result = []\n    for attribute_name, values in sorted(iteritems(results)):\n      # Values are triples of (attribute_name, timestamp, data). We want to\n      # sort by timestamp.\n      for _, ts, data in sorted(values, key=lambda x: x[1], reverse=True):\n        # Return triples (attribute_name, data, timestamp).\n        result.append((attribute_name, data, ts))\n    return result", "response": "Resolves all attributes for a subject starting with a prefix."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef Process(self, fs_msg, context):\n    try:\n      if fs_msg.message_type == \"GrrMessage\":\n        grr_message = rdf_flows.GrrMessage.FromSerializedString(\n            fs_msg.data.value)\n        self._ProcessGRRMessages(fs_msg.source.client_id, [grr_message])\n      elif fs_msg.message_type == \"MessageList\":\n        packed_messages = rdf_flows.PackedMessageList.FromSerializedString(\n            fs_msg.data.value)\n        message_list = communicator.Communicator.DecompressMessageList(\n            packed_messages)\n        self._ProcessGRRMessages(fs_msg.source.client_id, message_list.job)\n      else:\n        logging.error(\"Received message with unrecognized message_type: %s\",\n                      fs_msg.message_type)\n        context.set_code(grpc.StatusCode.INVALID_ARGUMENT)\n    except Exception as e:\n      logging.error(\"Exception processing message: %s\", str(e))\n      raise", "response": "Processes a single fleetspeak message."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nprocesses messages from GRR clients.", "response": "def _ProcessGRRMessages(self, fs_client_id, grr_messages):\n    \"\"\"Handles messages from GRR clients received via Fleetspeak.\n\n    This method updates the last-ping timestamp of the client before beginning\n    processing.\n\n    Args:\n      fs_client_id: The Fleetspeak client-id for the client.\n      grr_messages: An Iterable of GrrMessages.\n    \"\"\"\n    grr_client_id = fleetspeak_utils.FleetspeakIDToGRRID(fs_client_id)\n    for grr_message in grr_messages:\n      grr_message.source = grr_client_id\n      grr_message.auth_state = (\n          rdf_flows.GrrMessage.AuthorizationState.AUTHENTICATED)\n    client_is_new = self.frontend.EnrolFleetspeakClient(client_id=grr_client_id)\n    if not client_is_new and data_store.RelationalDBEnabled():\n      data_store.REL_DB.WriteClientMetadata(\n          grr_client_id, last_ping=rdfvalue.RDFDatetime.Now())\n    self.frontend.ReceiveMessages(\n        client_id=grr_client_id, messages=grr_messages)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nperform all checks on a host using acquired artifacts. Checks are selected based on the artifacts available and the host attributes (e.g. os_name/cpe/labels) provided as either parameters, or in the knowledgebase artifact. A KnowledgeBase artifact should be provided that contains, at a minimum: - OS - Hostname or IP Other knowldegebase attributes may be required for specific checks. CPE is currently unused, pending addition of a CPE module in the GRR client. Labels are arbitrary string labels attached to a client. Args: host_data: A dictionary with artifact names as keys, and rdf data as values. os_name: An OS name (optional). cpe: A CPE string (optional). labels: An iterable of labels (optional). exclude_checks: A list of check ids not to run. A check id in this list will not get run even if included in restrict_checks. restrict_checks: A list of check ids that may be run, if appropriate. Returns: A CheckResults object that contains results for all checks that were performed on the host.", "response": "def CheckHost(host_data,\n              os_name=None,\n              cpe=None,\n              labels=None,\n              exclude_checks=None,\n              restrict_checks=None):\n  \"\"\"Perform all checks on a host using acquired artifacts.\n\n  Checks are selected based on the artifacts available and the host attributes\n  (e.g. os_name/cpe/labels) provided as either parameters, or in the\n  knowledgebase artifact.\n\n  A KnowledgeBase artifact should be provided that contains, at a minimum:\n  - OS\n  - Hostname or IP\n  Other knowldegebase attributes may be required for specific checks.\n\n  CPE is currently unused, pending addition of a CPE module in the GRR client.\n\n  Labels are arbitrary string labels attached to a client.\n\n  Args:\n    host_data: A dictionary with artifact names as keys, and rdf data as values.\n    os_name: An OS name (optional).\n    cpe: A CPE string (optional).\n    labels: An iterable of labels (optional).\n    exclude_checks: A list of check ids not to run. A check id in this list\n                    will not get run even if included in restrict_checks.\n    restrict_checks: A list of check ids that may be run, if appropriate.\n\n  Returns:\n    A CheckResults object that contains results for all checks that were\n      performed on the host.\n  \"\"\"\n  # Get knowledgebase, os_name from hostdata\n  kb = host_data.get(\"KnowledgeBase\")\n  if os_name is None:\n    os_name = kb.os\n  if cpe is None:\n    # TODO(user): Get CPE (requires new artifact/parser)\n    pass\n  if labels is None:\n    # TODO(user): Get labels (see grr/lib/export.py for acquisition\n    # from client)\n    pass\n  return CheckRegistry.Process(\n      host_data,\n      os_name=os_name,\n      cpe=cpe,\n      labels=labels,\n      restrict_checks=restrict_checks,\n      exclude_checks=exclude_checks)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef LoadCheckFromFile(file_path, check_id, overwrite_if_exists=True):\n  configs = LoadConfigsFromFile(file_path)\n  conf = configs.get(check_id)\n  check = Check(**conf)\n  check.Validate()\n  CheckRegistry.RegisterCheck(\n      check,\n      source=\"file:%s\" % file_path,\n      overwrite_if_exists=overwrite_if_exists)\n  logging.debug(\"Loaded check %s from %s\", check.check_id, file_path)\n  return check", "response": "Load a single check from a file."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef LoadChecksFromFiles(file_paths, overwrite_if_exists=True):\n  loaded = []\n  for file_path in file_paths:\n    configs = LoadConfigsFromFile(file_path)\n    for conf in itervalues(configs):\n      check = Check(**conf)\n      # Validate will raise if the check doesn't load.\n      check.Validate()\n      loaded.append(check)\n      CheckRegistry.RegisterCheck(\n          check,\n          source=\"file:%s\" % file_path,\n          overwrite_if_exists=overwrite_if_exists)\n      logging.debug(\"Loaded check %s from %s\", check.check_id, file_path)\n  return loaded", "response": "Load the checks defined in the specified files."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nloading all yaml files in the specified directories.", "response": "def LoadChecksFromDirs(dir_paths, overwrite_if_exists=True):\n  \"\"\"Load checks from all yaml files in the specified directories.\"\"\"\n  loaded = []\n  for dir_path in dir_paths:\n    cfg_files = glob.glob(os.path.join(dir_path, \"*.yaml\"))\n    loaded.extend(LoadChecksFromFiles(cfg_files, overwrite_if_exists))\n  return loaded"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef Render(self, rdf_data):\n    report_data = rdf_data[:self.max_results]\n    results = [self.hinter.Render(rdf) for rdf in report_data]\n    extra = len(rdf_data) - len(report_data)\n    if extra > 0:\n      results.append(\"...plus another %d issues.\" % extra)\n    return results", "response": "Processes data according to formatting rules."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef Parse(self, rdf_data):\n    if self._filter:\n      return list(self._filter.Parse(rdf_data, self.expression))\n    return rdf_data", "response": "Parses the RDF data into a list of items."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nvalidates the filter exists and has valid filter and hint expressions.", "response": "def Validate(self):\n    \"\"\"The filter exists, and has valid filter and hint expressions.\"\"\"\n    if self.type not in filters.Filter.classes:\n      raise DefinitionError(\"Undefined filter type %s\" % self.type)\n    self._filter.Validate(self.expression)\n    Validate(self.hint, \"Filter has invalid hint\")"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef Parse(self, rdf_data):\n    if not isinstance(rdf_data, (list, set)):\n      raise ProcessingError(\"Bad host data format: %s\" % type(rdf_data))\n    if self.baseline:\n      comparison = self.baseliner.Parse(rdf_data)\n    else:\n      comparison = rdf_data\n    found = self.handler.Parse(comparison)\n    results = self.hint.Render(found)\n    return self.matcher.Detect(comparison, results)", "response": "Process rdf data through filters and test if results match expectations."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncheck the test set is well constructed.", "response": "def Validate(self):\n    \"\"\"Check the test set is well constructed.\"\"\"\n    Validate(self.target, \"Probe has invalid target\")\n    self.baseliner.Validate()\n    self.handler.Validate()\n    self.hint.Validate()"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef Parse(self, conditions, host_data):\n    processed = []\n    probes = self.triggers.Calls(conditions)\n    for p in probes:\n      # Get the data required for the probe. A probe can use a result_context\n      # (e.g. Parsers, Anomalies, Raw), to identify the data that is needed\n      # from the artifact collection results.\n      artifact_data = host_data.get(p.artifact)\n      if not p.result_context:\n        rdf_data = artifact_data[\"PARSER\"]\n      else:\n        rdf_data = artifact_data.get(str(p.result_context))\n      try:\n        result = p.Parse(rdf_data)\n      except ProcessingError as e:\n        raise ProcessingError(\"Bad artifact %s: %s\" % (p.artifact, e))\n      if result:\n        processed.append(result)\n    # Matcher compares the number of probes that triggered with results.\n    return self.matcher.Detect(probes, processed)", "response": "Runs probes that evaluate whether collected data has an issue."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef Validate(self):\n    ValidateMultiple(self.probe, \"Method has invalid probes\")\n    Validate(self.target, \"Method has invalid target\")\n    Validate(self.hint, \"Method has invalid hint\")", "response": "Check the Method is well constructed."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef ExtendAnomalies(self, other):\n    for o in other:\n      if o is not None:\n        self.anomaly.Extend(list(o.anomaly))", "response": "Extend anomalies from another CheckResult."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ndetermining if the check uses the specified artifact.", "response": "def UsesArtifact(self, artifacts):\n    \"\"\"Determines if the check uses the specified artifact.\n\n    Args:\n      artifacts: Either a single artifact name, or a list of artifact names\n\n    Returns:\n      True if the check uses a specific artifact.\n    \"\"\"\n    # If artifact is a single string, see if it is in the list of artifacts\n    # as-is. Otherwise, test whether any of the artifacts passed in to this\n    # function exist in the list of artifacts.\n    if isinstance(artifacts, string_types):\n      return artifacts in self.artifacts\n    else:\n      return any(True for artifact in artifacts if artifact in self.artifacts)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef Parse(self, conditions, host_data):\n    result = CheckResult(check_id=self.check_id)\n    methods = self.SelectChecks(conditions)\n    result.ExtendAnomalies([m.Parse(conditions, host_data) for m in methods])\n    return result", "response": "Runs methods that evaluate whether collected host_data has an issue."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef Validate(self):\n    if not self.check_id:\n      raise DefinitionError(\"Check has missing check_id value\")\n    cls_name = self.check_id\n    if not self.method:\n      raise DefinitionError(\"Check %s has no methods\" % cls_name)\n    ValidateMultiple(self.method,\n                     \"Check %s has invalid method definitions\" % cls_name)", "response": "Validate the check method."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef Detect(self, baseline, host_data):\n    result = CheckResult()\n    for detector in self.detectors:\n      finding = detector(baseline, host_data)\n      if finding:\n        result.ExtendAnomalies([finding])\n    if result:\n      return result", "response": "Runs through detectors and returns anomalies if any detector triggers the issue."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef Issue(self, state, results):\n    result = CheckResult()\n    # If there are CheckResults we're aggregating methods or probes.\n    # Merge all current results into one CheckResult.\n    # Otherwise, the results are raw host data.\n    # Generate a new CheckResult and add the specific findings.\n    if results and all(isinstance(r, CheckResult) for r in results):\n      result.ExtendAnomalies(results)\n    else:\n      result.anomaly = [\n          rdf_anomaly.Anomaly(\n              type=anomaly_pb2.Anomaly.AnomalyType.Name(\n                  anomaly_pb2.Anomaly.ANALYSIS_ANOMALY),\n              symptom=self.hint.Problem(state),\n              finding=self.hint.Render(results),\n              explanation=self.hint.Fix())\n      ]\n    return result", "response": "Issues an anomalous findings."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nregister a check in the registry.", "response": "def RegisterCheck(cls, check, source=\"unknown\", overwrite_if_exists=False):\n    \"\"\"Adds a check to the registry, refresh the trigger to check map.\"\"\"\n    if not overwrite_if_exists and check.check_id in cls.checks:\n      raise DefinitionError(\n          \"Check named %s already exists and \"\n          \"overwrite_if_exists is set to False.\" % check.check_id)\n    check.loaded_from = source\n    cls.checks[check.check_id] = check\n    cls.triggers.Update(check.triggers, check)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _AsList(arg):\n    if (isinstance(arg, string_types) or\n        not isinstance(arg, collections.Iterable)):\n      return [arg]\n    else:\n      return list(arg)", "response": "Encapsulates an argument in a list if it s not already iterable."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef Conditions(cls, artifact=None, os_name=None, cpe=None, labels=None):\n    artifact = cls._AsList(artifact)\n    os_name = cls._AsList(os_name)\n    cpe = cls._AsList(cpe)\n    labels = cls._AsList(labels)\n    for condition in itertools.product(artifact, os_name, cpe, labels):\n      yield condition", "response": "Generates a series of condition tuples."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef FindChecks(cls,\n                 artifact=None,\n                 os_name=None,\n                 cpe=None,\n                 labels=None,\n                 restrict_checks=None):\n    \"\"\"Takes targeting info, identifies relevant checks.\n\n    FindChecks will return results when a host has the conditions necessary for\n    a check to occur. Conditions with partial results are not returned. For\n    example, FindChecks will not return checks that if a check targets\n    os_name=[\"Linux\"], labels=[\"foo\"] and a host only has the os_name=[\"Linux\"]\n    attribute.\n\n    Args:\n      artifact: 0+ artifact names.\n      os_name: 0+ OS names.\n      cpe: 0+ CPE identifiers.\n      labels: 0+ GRR labels.\n      restrict_checks: A list of check ids to restrict check processing to.\n\n    Returns:\n      the check_ids that apply.\n    \"\"\"\n    check_ids = set()\n    conditions = list(cls.Conditions(artifact, os_name, cpe, labels))\n    for chk_id, chk in iteritems(cls.checks):\n      if restrict_checks and chk_id not in restrict_checks:\n        continue\n      for condition in conditions:\n        if chk.triggers.Match(*condition):\n          check_ids.add(chk_id)\n          break  # No need to keep checking other conditions.\n    return check_ids", "response": "Returns a list of check ids that apply to the given set of conditions."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef SelectArtifacts(cls,\n                      os_name=None,\n                      cpe=None,\n                      labels=None,\n                      restrict_checks=None):\n    \"\"\"Takes targeting info, identifies artifacts to fetch.\n\n    Args:\n      os_name: 0+ OS names.\n      cpe: 0+ CPE identifiers.\n      labels: 0+ GRR labels.\n      restrict_checks: A list of check ids whose artifacts should be fetched.\n\n    Returns:\n      the artifacts that should be collected.\n    \"\"\"\n    results = set()\n    for condition in cls.Conditions(None, os_name, cpe, labels):\n      trigger = condition[1:]\n      for chk in itervalues(cls.checks):\n        if restrict_checks and chk.check_id not in restrict_checks:\n          continue\n        results.update(chk.triggers.Artifacts(*trigger))\n    return results", "response": "Takes targeting info identifies artifacts to fetch."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nprocessing the host data and returns a list of CheckResult messages.", "response": "def Process(cls,\n              host_data,\n              os_name=None,\n              cpe=None,\n              labels=None,\n              exclude_checks=None,\n              restrict_checks=None):\n    \"\"\"Runs checks over all host data.\n\n    Args:\n      host_data: The data collected from a host, mapped to artifact name.\n      os_name: 0+ OS names.\n      cpe: 0+ CPE identifiers.\n      labels: 0+ GRR labels.\n      exclude_checks: A list of check ids not to run. A check id in this list\n                      will not get run even if included in restrict_checks.\n      restrict_checks: A list of check ids that may be run, if appropriate.\n\n    Yields:\n      A CheckResult message for each check that was performed.\n    \"\"\"\n    # All the conditions that apply to this host.\n    artifacts = list(iterkeys(host_data))\n    check_ids = cls.FindChecks(artifacts, os_name, cpe, labels)\n    conditions = list(cls.Conditions(artifacts, os_name, cpe, labels))\n    for check_id in check_ids:\n      # skip if check in list of excluded checks\n      if exclude_checks and check_id in exclude_checks:\n        continue\n      if restrict_checks and check_id not in restrict_checks:\n        continue\n      try:\n        chk = cls.checks[check_id]\n        yield chk.Parse(conditions, host_data)\n      except ProcessingError as e:\n        logging.warning(\"Check ID %s raised: %s\", check_id, e)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a pseudo - random 32 - bit unsigned integer.", "response": "def UInt32():\n  \"\"\"Returns a pseudo-random 32-bit unsigned integer.\"\"\"\n  with _mutex:\n    try:\n      return _random_buffer.pop()\n    except IndexError:\n      data = os.urandom(struct.calcsize(\"=L\") * _random_buffer_size)\n      _random_buffer.extend(\n          struct.unpack(\"=\" + \"L\" * _random_buffer_size, data))\n      return _random_buffer.pop()"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nsetting up some vars for the directories we use.", "response": "def SetupVars(self):\n    \"\"\"Set up some vars for the directories we use.\"\"\"\n    # Python paths chosen to match appveyor:\n    # http://www.appveyor.com/docs/installed-software#python\n\n    self.python_dir_64 = args.python64_dir\n    self.python_dir_32 = args.python32_dir\n\n    self.virtualenv_bin64 = os.path.join(self.python_dir_64,\n                                         r\"Scripts\\virtualenv.exe\")\n    self.virtualenv_bin32 = os.path.join(self.python_dir_32,\n                                         r\"Scripts\\virtualenv.exe\")\n\n    if args.virtualenv_64bit:\n      self.virtualenv64 = args.virtualenv_64bit\n    else:\n      self.virtualenv64 = os.path.join(args.build_dir, \"python_64\")\n    if args.virtualenv_32bit:\n      self.virtualenv32 = args.virtualenv_32bit\n    else:\n      self.virtualenv32 = os.path.join(args.build_dir, \"python_32\")\n\n    self.grr_client_build64 = os.path.join(self.virtualenv64,\n                                           r\"Scripts\\grr_client_build.exe\")\n    self.grr_client_build32 = os.path.join(self.virtualenv32,\n                                           r\"Scripts\\grr_client_build.exe\")\n    self.pip64 = os.path.join(self.virtualenv64, r\"Scripts\\pip.exe\")\n    self.pip32 = os.path.join(self.virtualenv32, r\"Scripts\\pip.exe\")\n\n    self.virtualenv_python64 = os.path.join(self.virtualenv64,\n                                            r\"Scripts\\python.exe\")\n    self.virtualenv_python32 = os.path.join(self.virtualenv32,\n                                            r\"Scripts\\python.exe\")\n\n    self.git = r\"C:\\Program Files\\Git\\bin\\git.exe\"\n\n    self.install_path = r\"C:\\Windows\\System32\\GRR\"\n    self.service_name = \"GRR Monitor\"\n\n    self.expect_service_running = args.expect_service_running"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef Clean(self):\n    # os.unlink doesn't work effectively, use the shell to delete.\n    if os.path.exists(args.build_dir):\n      subprocess.call(\"rd /s /q %s\" % args.build_dir, shell=True)\n    if os.path.exists(args.output_dir):\n      subprocess.call(\"rd /s /q %s\" % args.output_dir, shell=True)\n\n    os.makedirs(args.build_dir)\n    os.makedirs(args.output_dir)\n\n    # Create virtualenvs.\n    if not args.virtualenv_64bit:\n      subprocess.check_call([self.virtualenv_bin64, self.virtualenv64])\n    if args.build_32 and not args.virtualenv_32bit:\n      subprocess.check_call([self.virtualenv_bin32, self.virtualenv32])\n\n    # Currently this should do nothing as we will already have a modern pip\n    # installed, but we leave this here so if we get broken by pip again it's\n    # just a simple case of searching for pip>=8.1.1 and adding an upper limit\n    # cap in all those places.\n\n    cmd = [\"-m\", \"pip\", \"install\"]\n    if args.wheel_dir:\n      cmd += [\"--no-index\", r\"--find-links=file:///%s\" % args.wheel_dir]\n\n    subprocess.check_call([self.virtualenv_python64] + cmd +\n                          [\"--upgrade\", \"pip>=8.1.1\"])\n    if args.build_32:\n      subprocess.check_call([self.virtualenv_python32] + cmd +\n                            [\"--upgrade\", \"pip>=8.1.1\"])\n    os.environ[\"PROTOC\"] = args.protoc", "response": "Clean the build environment."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef BuildTemplates(self):\n    if args.config:\n      build_args = [\n          \"--verbose\", \"--config\", args.config, \"build\", \"--output\",\n          args.output_dir\n      ]\n    else:\n      build_args = [\"--verbose\", \"build\", \"--output\", args.output_dir]\n    subprocess.check_call([self.grr_client_build64] + build_args)\n    if args.build_32:\n      subprocess.check_call([self.grr_client_build32] + build_args)", "response": "Builds the client templates."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _CleanupInstall(self):\n    if os.path.exists(self.install_path):\n      shutil.rmtree(self.install_path)\n      if os.path.exists(self.install_path):\n        raise RuntimeError(\"Install path still exists: %s\" % self.install_path)\n\n    # Deliberately don't check return code, since service may not be installed.\n    subprocess.call([\"sc\", \"stop\", self.service_name])", "response": "Cleanup from any previous installer enough for _CheckInstallSuccess."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _CheckInstallSuccess(self):\n    if not os.path.exists(self.install_path):\n      raise RuntimeError(\"Install failed, no files at: %s\" % self.install_path)\n\n    try:\n      output = subprocess.check_output([\"sc\", \"query\", self.service_name])\n      service_running = \"RUNNING\" in output\n    except subprocess.CalledProcessError as e:\n      if e.returncode == 1060:\n        # 1060 means: The specified service does not exist as an installed\n        # service.\n        service_running = False\n      else:\n        raise\n\n    if self.expect_service_running:\n      if not service_running:\n        raise RuntimeError(\n            \"GRR service not running after install, sc query output: %s\" %\n            output)\n    else:\n      if service_running:\n        raise RuntimeError(\n            \"GRR service running after install with expect_service_running == \"\n            \"False, sc query output: %s\" % output)", "response": "Checks if the installer installed correctly."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _InstallInstallers(self):\n    # 32 bit binary will refuse to install on a 64bit system so we only install\n    # the 64 bit version\n    installer_amd64 = glob.glob(\n        os.path.join(args.output_dir, \"dbg_*_amd64.exe\")).pop()\n    self._CleanupInstall()\n    # The exit code is always 0, test to see if install was actually successful.\n    subprocess.check_call([installer_amd64])\n    self._CheckInstallSuccess()", "response": "Install the installer built by RepackTemplates."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nensure that given value has the expected type.", "response": "def AssertType(value, expected_type):\n  \"\"\"Ensures that given value has certain type.\n\n  Args:\n    value: A value to assert the type for.\n    expected_type: An expected type for the given value.\n\n  Raises:\n    TypeError: If given value does not have the expected type.\n  \"\"\"\n  if not isinstance(value, expected_type):\n    message = \"Expected type `%r`, but got value `%r` of type `%s`\"\n    message %= (expected_type, value, type(value))\n    raise TypeError(message)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef AssertIterableType(iterable, expected_item_type):\n  # We do not consider iterators to be iterables even though Python does. An\n  # \"iterable\" should be a type that can be iterated (that is: an iterator can\n  # be constructed for them). Iterators should not be considered to be iterable\n  # because it makes no sense to construct an iterator for iterator. The most\n  # important practical implication is that act of iterating an iterator drains\n  # it whereas act of iterating the iterable does not.\n  if isinstance(iterable, collections.Iterator):\n    message = \"Expected iterable container but got iterator `%s` instead\"\n    message %= iterable\n    raise TypeError(message)\n\n  AssertType(iterable, collections.Iterable)\n  for item in iterable:\n    AssertType(item, expected_item_type)", "response": "Ensures that given iterable container has certain type."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef AssertDictType(dct, expected_key_type, expected_value_type):\n  AssertType(dct, dict)\n  for key, value in iteritems(dct):\n    AssertType(key, expected_key_type)\n    AssertType(value, expected_value_type)", "response": "Ensures that given dictionary is actually a dictionary of specified type."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nrun the single hook specified by resolving all its prerequisites.", "response": "def _RunSingleHook(self, hook_cls, executed_set, required=None):\n    \"\"\"Run the single hook specified by resolving all its prerequisites.\"\"\"\n    # If we already ran do nothing.\n    if hook_cls in executed_set:\n      return\n\n    # Ensure all the pre execution hooks are run.\n    for pre_hook in hook_cls.pre:\n      self._RunSingleHook(pre_hook, executed_set, required=hook_cls.__name__)\n\n    # Now run this hook.\n    cls_instance = hook_cls()\n    if required:\n      logging.debug(\"Initializing %s, required by %s\", hook_cls.__name__,\n                    required)\n    else:\n      logging.debug(\"Initializing %s\", hook_cls.__name__)\n\n    # Always call the Run hook.\n    cls_instance.Run()\n    executed_set.add(hook_cls)\n\n    # Only call the RunOnce() hook if not already called.\n    if hook_cls not in self.already_run_once:\n      cls_instance.RunOnce()\n      self.already_run_once.add(hook_cls)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ninitialize the object from a database hunt object.", "response": "def InitFromHuntObject(self,\n                         hunt_obj,\n                         hunt_counters=None,\n                         with_full_summary=False):\n    \"\"\"Initialize API hunt object from a database hunt object.\n\n    Args:\n      hunt_obj: rdf_hunt_objects.Hunt to read the data from.\n      hunt_counters: Optional db.HuntCounters object with counters information.\n      with_full_summary: if True, hunt_runner_args, completion counts and a few\n        other fields will be filled in. The way to think about it is that with\n        with_full_summary==True ApiHunt will have the data to render \"Hunt\n        Overview\" page and with with_full_summary==False it will have enough\n        data to be rendered as a hunts list row.\n\n    Returns:\n      Self.\n    \"\"\"\n\n    self.urn = rdfvalue.RDFURN(\"hunts\").Add(str(hunt_obj.hunt_id))\n    self.hunt_id = hunt_obj.hunt_id\n    if (hunt_obj.args.hunt_type ==\n        rdf_hunt_objects.HuntArguments.HuntType.STANDARD):\n      self.name = \"GenericHunt\"\n    else:\n      self.name = \"VariableGenericHunt\"\n    self.state = str(hunt_obj.hunt_state)\n    self.crash_limit = hunt_obj.crash_limit\n    self.client_limit = hunt_obj.client_limit\n    self.client_rate = hunt_obj.client_rate\n    self.created = hunt_obj.create_time\n    self.duration = hunt_obj.duration\n    self.creator = hunt_obj.creator\n    self.description = hunt_obj.description\n    self.is_robot = hunt_obj.creator in [\"GRRWorker\", \"Cron\"]\n    if hunt_counters is not None:\n      self.results_count = hunt_counters.num_results\n      self.clients_with_results_count = hunt_counters.num_clients_with_results\n      self.clients_queued_count = (\n          hunt_counters.num_clients - hunt_counters.num_successful_clients -\n          hunt_counters.num_failed_clients - hunt_counters.num_crashed_clients)\n      # TODO(user): remove this hack when AFF4 is gone. For regression tests\n      # compatibility only.\n      self.total_cpu_usage = hunt_counters.total_cpu_seconds or 0\n      self.total_net_usage = hunt_counters.total_network_bytes_sent\n\n      if with_full_summary:\n        self.all_clients_count = hunt_counters.num_clients\n        self.completed_clients_count = (\n            hunt_counters.num_successful_clients +\n            hunt_counters.num_failed_clients)\n        self.remaining_clients_count = (\n            self.all_clients_count - self.completed_clients_count)\n    else:\n      self.results_count = 0\n      self.clients_with_results_count = 0\n      self.clients_queued_count = 0\n      self.total_cpu_usage = 0\n      self.total_net_usage = 0\n\n      if with_full_summary:\n        self.all_clients_count = 0\n        self.completed_clients_count = 0\n        self.remaining_clients_count = 0\n\n    if hunt_obj.original_object.object_type != \"UNKNOWN\":\n      ref = ApiFlowLikeObjectReference()\n      self.original_object = ref.FromFlowLikeObjectReference(\n          hunt_obj.original_object)\n\n    if with_full_summary:\n      hra = self.hunt_runner_args = rdf_hunts.HuntRunnerArgs(\n          hunt_name=self.name,\n          description=hunt_obj.description,\n          client_rule_set=hunt_obj.client_rule_set,\n          crash_limit=hunt_obj.crash_limit,\n          avg_results_per_client_limit=hunt_obj.avg_results_per_client_limit,\n          avg_cpu_seconds_per_client_limit=hunt_obj\n          .avg_cpu_seconds_per_client_limit,\n          avg_network_bytes_per_client_limit=hunt_obj\n          .avg_network_bytes_per_client_limit,\n          client_rate=hunt_obj.client_rate,\n          original_object=hunt_obj.original_object)\n\n      if hunt_obj.HasField(\"output_plugins\"):\n        hra.output_plugins = hunt_obj.output_plugins\n\n      # TODO(user): This is a backwards compatibility code. Remove\n      # HuntRunnerArgs from ApiHunt.\n      if hunt_obj.client_limit != 100:\n        hra.client_limit = hunt_obj.client_limit\n\n      if hunt_obj.HasField(\"per_client_cpu_limit\"):\n        hra.per_client_cpu_limit = hunt_obj.per_client_cpu_limit\n\n      if hunt_obj.HasField(\"per_client_network_limit_bytes\"):\n        hra.per_client_network_limit_bytes = (\n            hunt_obj.per_client_network_bytes_limit)\n\n      if hunt_obj.HasField(\"total_network_bytes_limit\"):\n        hra.network_bytes_limit = hunt_obj.total_network_bytes_limit\n\n      self.client_rule_set = hunt_obj.client_rule_set\n\n      if (hunt_obj.args.hunt_type ==\n          rdf_hunt_objects.HuntArguments.HuntType.STANDARD):\n        self.flow_name = hunt_obj.args.standard.flow_name\n        self.flow_args = hunt_obj.args.standard.flow_args\n\n    return self"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef InitFromFlowObject(self, fo):\n\n    self.client_id = fo.client_id\n    if fo.HasField(\"backtrace\"):\n      self.backtrace = fo.backtrace\n    self.log_message = fo.error_message\n    self.timestamp = fo.last_update_time\n\n    return self", "response": "Initialize from rdf_flow_objects. Flow object."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef InitFromDataPoints(self, start_stats, complete_stats):\n    self.start_points = self._ConvertToResultList(start_stats)\n    self.complete_points = self._ConvertToResultList(complete_stats)\n    return self", "response": "Initializes the approval from the given data points."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _Resample(self, stats, target_size):\n    t_first = stats[0][0]\n    t_last = stats[-1][0]\n    interval = (t_last - t_first) / target_size\n\n    result = []\n\n    current_t = t_first\n    current_v = 0\n    i = 0\n    while i < len(stats):\n      stat_t = stats[i][0]\n      stat_v = stats[i][1]\n      if stat_t <= (current_t + interval):\n        # Always add the last value in an interval to the result.\n        current_v = stat_v\n        i += 1\n      else:\n        result.append([current_t + interval, current_v])\n        current_t += interval\n\n    result.append([current_t + interval, current_v])\n    return result", "response": "Resamples the stats to have a specific number of data points."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nretrieve the stats for a hunt.", "response": "def _HandleLegacy(self, args, token=None):\n    \"\"\"Retrieves the stats for a hunt.\"\"\"\n    hunt_obj = aff4.FACTORY.Open(\n        args.hunt_id.ToURN(), aff4_type=implementation.GRRHunt, token=token)\n\n    stats = hunt_obj.GetRunner().context.usage_stats\n\n    return ApiGetHuntStatsResult(stats=stats)"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nretrieve the clients for a given hunt.", "response": "def _HandleLegacy(self, args, token=None):\n    \"\"\"Retrieves the clients for a hunt.\"\"\"\n    hunt_urn = args.hunt_id.ToURN()\n    hunt_obj = aff4.FACTORY.Open(\n        hunt_urn, aff4_type=implementation.GRRHunt, token=token)\n\n    clients_by_status = hunt_obj.GetClientsByStatus()\n    hunt_clients = clients_by_status[args.client_status.name]\n    total_count = len(hunt_clients)\n\n    if args.count:\n      hunt_clients = sorted(hunt_clients)[args.offset:args.offset + args.count]\n    else:\n      hunt_clients = sorted(hunt_clients)[args.offset:]\n\n    flow_id = \"%s:hunt\" % hunt_urn.Basename()\n    results = [\n        ApiHuntClient(client_id=c.Basename(), flow_id=flow_id)\n        for c in hunt_clients\n    ]\n    return ApiListHuntClientsResult(items=results, total_count=total_count)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _HandleLegacy(self, args, token=None):\n\n    hunt_obj = aff4.FACTORY.Open(\n        args.hunt_id.ToURN(), aff4_type=implementation.GRRHunt, token=token)\n\n    if isinstance(hunt_obj.context, rdf_hunts.HuntContext):  # New style hunt.\n      # TODO(amoser): Hunt state will go away soon, we don't render it anymore.\n      state = api_call_handler_utils.ApiDataObject()\n      result = ApiGetHuntContextResult(context=hunt_obj.context, state=state)\n      # Assign args last since it needs the other fields set to\n      # determine the args protobuf.\n      result.args = hunt_obj.args\n      return result\n\n    else:\n      # Just pack the whole context data object in the state\n      # field. This contains everything for old style hunts so we at\n      # least show the data somehow.\n      context = api_call_handler_utils.ApiDataObject().InitFromDataObject(\n          hunt_obj.context)\n\n      return ApiGetHuntContextResult(state=context)", "response": "Retrieves the context for a hunt."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _HandleLegacy(self, args, token=None):\n\n    # We only create generic hunts with /hunts/create requests.\n    generic_hunt_args = rdf_hunts.GenericHuntArgs()\n    generic_hunt_args.flow_runner_args.flow_name = args.flow_name\n    generic_hunt_args.flow_args = args.flow_args\n\n    # Clear all fields marked with HIDDEN, except for output_plugins - they are\n    # marked HIDDEN, because we have a separate UI for them, not because they\n    # shouldn't be shown to the user at all.\n    #\n    # TODO(user): Refactor the code to remove the HIDDEN label from\n    # HuntRunnerArgs.output_plugins.\n    args.hunt_runner_args.ClearFieldsWithLabel(\n        rdf_structs.SemanticDescriptor.Labels.HIDDEN,\n        exceptions=\"output_plugins\")\n    args.hunt_runner_args.hunt_name = standard.GenericHunt.__name__\n\n    if args.original_hunt and args.original_flow:\n      raise ValueError(\n          \"A hunt can't be a copy of a flow and a hunt at the same time.\")\n    if args.original_hunt:\n      ref = rdf_hunts.FlowLikeObjectReference.FromHuntId(\n          utils.SmartStr(args.original_hunt.hunt_id))\n      args.hunt_runner_args.original_object = ref\n    elif args.original_flow:\n      ref = rdf_hunts.FlowLikeObjectReference.FromFlowIdAndClientId(\n          utils.SmartStr(args.original_flow.flow_id),\n          utils.SmartStr(args.original_flow.client_id))\n      args.hunt_runner_args.original_object = ref\n\n    # Anyone can create the hunt but it will be created in the paused\n    # state. Permissions are required to actually start it.\n    with implementation.StartHunt(\n        runner_args=args.hunt_runner_args, args=generic_hunt_args,\n        token=token) as hunt_obj:\n\n      # Nothing really to do here - hunts are always created in the paused\n      # state.\n      logging.info(\"User %s created a new %s hunt (%s)\", token.username,\n                   hunt_obj.args.flow_runner_args.flow_name, hunt_obj.urn)\n\n      return ApiHunt().InitFromAff4Object(hunt_obj, with_full_summary=True)", "response": "Handles the legacy hunt creation and creation of a new hunt."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nregister a new constructor in the internal list of constructors.", "response": "def Register(self, name, constructor):\n    \"\"\"Registers a new constructor in the factory.\n\n    Args:\n      name: A name associated with given constructor.\n      constructor: A constructor function that creates instances.\n\n    Raises:\n      ValueError: If there already is a constructor associated with given name.\n    \"\"\"\n    precondition.AssertType(name, Text)\n\n    if name in self._constructors:\n      message = \"Duplicated constructors %r and %r for name '%s'\"\n      message %= (constructor, self._constructors[name], name)\n      raise ValueError(message)\n\n    self._constructors[name] = constructor"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef Unregister(self, name):\n    precondition.AssertType(name, Text)\n\n    try:\n      del self._constructors[name]\n    except KeyError:\n      raise ValueError(\"Constructor with name '%s' is not registered\" % name)", "response": "Unregisters a constructor with the specified name."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ncreates a new instance of the type that the factory supports.", "response": "def Create(self, name):\n    \"\"\"Creates a new instance.\n\n    Args:\n      name: A name identifying the constructor to use for instantiation.\n\n    Returns:\n      An instance of the type that the factory supports.\n    \"\"\"\n    precondition.AssertType(name, Text)\n\n    try:\n      constructor = self._constructors[name]\n    except KeyError:\n      message = \"No constructor for name '%s' has been registered\"\n      message %= name\n      raise ValueError(message)\n\n    instance = constructor()\n    if not isinstance(instance, self._cls):\n      message = (\"Constructor %r for name '%s' returned instance of type %r \"\n                 \"(expected %r)\")\n      message %= (constructor, name, type(instance), self._cls)\n      raise TypeError(message)\n\n    return instance"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nregisters a stat entry at a given timestamp.", "response": "def AddStatEntry(self, stat_entry, timestamp):\n    \"\"\"Registers stat entry at a given timestamp.\"\"\"\n\n    if timestamp in self._stat_entries:\n      message = (\"Duplicated stat entry write for path '%s' of type '%s' at \"\n                 \"timestamp '%s'. Old: %s. New: %s.\")\n      message %= (\"/\".join(self._components), self._path_type, timestamp,\n                  self._stat_entries[timestamp], stat_entry)\n      raise db.Error(message)\n\n    if timestamp not in self._path_infos:\n      path_info = rdf_objects.PathInfo(\n          path_type=self._path_type,\n          components=self._components,\n          timestamp=timestamp,\n          stat_entry=stat_entry)\n      self.AddPathInfo(path_info)\n    else:\n      self._path_infos[timestamp].stat_entry = stat_entry"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nregisters a hash entry at a given timestamp.", "response": "def AddHashEntry(self, hash_entry, timestamp):\n    \"\"\"Registers hash entry at a given timestamp.\"\"\"\n\n    if timestamp in self._hash_entries:\n      message = (\"Duplicated hash entry write for path '%s' of type '%s' at \"\n                 \"timestamp '%s'. Old: %s. New: %s.\")\n      message %= (\"/\".join(self._components), self._path_type, timestamp,\n                  self._hash_entries[timestamp], hash_entry)\n      raise db.Error(message)\n\n    if timestamp not in self._path_infos:\n      path_info = rdf_objects.PathInfo(\n          path_type=self._path_type,\n          components=self._components,\n          timestamp=timestamp,\n          hash_entry=hash_entry)\n      self.AddPathInfo(path_info)\n    else:\n      self._path_infos[timestamp].hash_entry = hash_entry"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef AddPathInfo(self, path_info):\n\n    if self._path_type != path_info.path_type:\n      message = \"Incompatible path types: `%s` and `%s`\"\n      raise ValueError(message % (self._path_type, path_info.path_type))\n    if self._components != path_info.components:\n      message = \"Incompatible path components: `%s` and `%s`\"\n      raise ValueError(message % (self._components, path_info.components))\n\n    if path_info.timestamp in self._path_infos:\n      raise ValueError(\"PathInfo with timestamp %r was added before.\" %\n                       path_info.timestamp)\n\n    new_path_info = path_info.Copy()\n    if new_path_info.timestamp is None:\n      new_path_info.timestamp = rdfvalue.RDFDatetime.Now()\n    self._path_infos[new_path_info.timestamp] = new_path_info", "response": "Adds a path info to the path record."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef AddChild(self, path_info):\n\n    if self._path_type != path_info.path_type:\n      message = \"Incompatible path types: `%s` and `%s`\"\n      raise ValueError(message % (self._path_type, path_info.path_type))\n    if self._components != path_info.components[:-1]:\n      message = \"Incompatible path components, expected `%s` but got `%s`\"\n      raise ValueError(message % (self._components, path_info.components[:-1]))\n\n    self._children.add(path_info.GetPathID())", "response": "Makes the path aware of some child."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ngenerate a summary about the path record.", "response": "def GetPathInfo(self, timestamp=None):\n    \"\"\"Generates a summary about the path record.\n\n    Args:\n      timestamp: A point in time from which the data should be retrieved.\n\n    Returns:\n      A `rdf_objects.PathInfo` instance.\n    \"\"\"\n    path_info_timestamp = self._LastEntryTimestamp(self._path_infos, timestamp)\n    try:\n      result = self._path_infos[path_info_timestamp].Copy()\n    except KeyError:\n      result = rdf_objects.PathInfo(\n          path_type=self._path_type, components=self._components)\n\n    stat_entry_timestamp = self._LastEntryTimestamp(self._stat_entries,\n                                                    timestamp)\n    result.last_stat_entry_timestamp = stat_entry_timestamp\n    result.stat_entry = self._stat_entries.get(stat_entry_timestamp)\n\n    hash_entry_timestamp = self._LastEntryTimestamp(self._hash_entries,\n                                                    timestamp)\n    result.last_hash_entry_timestamp = hash_entry_timestamp\n    result.hash_entry = self._hash_entries.get(hash_entry_timestamp)\n\n    return result"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _LastEntryTimestamp(dct, upper_bound_timestamp):\n    if upper_bound_timestamp is None:\n      upper_bound = lambda _: True\n    else:\n      upper_bound = lambda key: key <= upper_bound_timestamp\n\n    try:\n      return max(filter(upper_bound, iterkeys(dct)))\n    except ValueError:  # Thrown if `max` input (result of filtering) is empty.\n      return None", "response": "Searches for the highest timestamp lower than the specified one."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef ReadPathInfo(self, client_id, path_type, components, timestamp=None):\n    try:\n      path_record = self.path_records[(client_id, path_type, components)]\n      return path_record.GetPathInfo(timestamp=timestamp)\n    except KeyError:\n      raise db.UnknownPathError(\n          client_id=client_id, path_type=path_type, components=components)", "response": "Reads a path info record for a given path."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nread path info records for given paths.", "response": "def ReadPathInfos(self, client_id, path_type, components_list):\n    \"\"\"Retrieves path info records for given paths.\"\"\"\n    result = {}\n\n    for components in components_list:\n      try:\n        path_record = self.path_records[(client_id, path_type, components)]\n        result[components] = path_record.GetPathInfo()\n      except KeyError:\n        result[components] = None\n\n    return result"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef ListDescendentPathInfos(self,\n                              client_id,\n                              path_type,\n                              components,\n                              timestamp=None,\n                              max_depth=None):\n    \"\"\"Lists path info records that correspond to children of given path.\"\"\"\n    result = []\n\n    for path_idx, path_record in iteritems(self.path_records):\n      other_client_id, other_path_type, other_components = path_idx\n      if client_id != other_client_id or path_type != other_path_type:\n        continue\n      if len(other_components) == len(components):\n        continue\n      if not collection.StartsWith(other_components, components):\n        continue\n      if (max_depth is not None and\n          len(other_components) - len(components) > max_depth):\n        continue\n\n      result.append(path_record.GetPathInfo(timestamp=timestamp))\n\n    if timestamp is None:\n      return sorted(result, key=lambda _: tuple(_.components))\n\n    # We need to filter implicit path infos if specific timestamp is given.\n\n    # TODO(hanuszczak): If we were to switch to use path trie instead of storing\n    # records by path id, everything would be much easier.\n\n    class TrieNode(object):\n      \"\"\"A trie of path components with path infos as values.\"\"\"\n\n      def __init__(self):\n        self.path_info = None\n        self.children = {}\n        self.explicit = False\n\n      def Add(self, path_info, idx=0):\n        \"\"\"Adds given path info to the trie (or one of its subtrees).\"\"\"\n        components = path_info.components\n        if idx == len(components):\n          self.path_info = path_info\n          self.explicit |= (\n              path_info.HasField(\"stat_entry\") or\n              path_info.HasField(\"hash_entry\"))\n        else:\n          child = self.children.setdefault(components[idx], TrieNode())\n          child.Add(path_info, idx=idx + 1)\n          self.explicit |= child.explicit\n\n      def Collect(self, path_infos):\n        if self.path_info is not None and self.explicit:\n          path_infos.append(self.path_info)\n\n        for component in sorted(iterkeys(self.children)):\n          self.children[component].Collect(path_infos)\n\n    trie = TrieNode()\n    for path_info in result:\n      trie.Add(path_info)\n\n    explicit_path_infos = []\n    trie.Collect(explicit_path_infos)\n    return explicit_path_infos", "response": "Lists all path info records that correspond to children of given path."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _WritePathInfo(self, client_id, path_info):\n    if client_id not in self.metadatas:\n      raise db.UnknownClientError(client_id)\n\n    path_record = self._GetPathRecord(client_id, path_info)\n    path_record.AddPathInfo(path_info)\n\n    parent_path_info = path_info.GetParent()\n    if parent_path_info is not None:\n      parent_path_record = self._GetPathRecord(client_id, parent_path_info)\n      parent_path_record.AddChild(path_info)", "response": "Writes a single path info record for given client."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nclears the path history for the given paths of given clients.", "response": "def MultiClearPathHistory(self, path_infos):\n    \"\"\"Clears path history for specified paths of given clients.\"\"\"\n    for client_id, client_path_infos in iteritems(path_infos):\n      self.ClearPathHistory(client_id, client_path_infos)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef ClearPathHistory(self, client_id, path_infos):\n    for path_info in path_infos:\n      path_record = self._GetPathRecord(client_id, path_info)\n      path_record.ClearHistory()", "response": "Clears the path history for the given paths of given client."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef MultiWritePathHistory(self, client_path_histories):\n    for client_path, client_path_history in iteritems(client_path_histories):\n      if client_path.client_id not in self.metadatas:\n        raise db.UnknownClientError(client_path.client_id)\n\n      path_info = rdf_objects.PathInfo(\n          path_type=client_path.path_type, components=client_path.components)\n\n      for timestamp, stat_entry in iteritems(client_path_history.stat_entries):\n        path_record = self._GetPathRecord(\n            client_path.client_id, path_info, set_default=False)\n        if path_record is None:\n          # TODO(hanuszczak): Provide more details about paths that caused that.\n          raise db.AtLeastOneUnknownPathError([])\n\n        path_record.AddStatEntry(stat_entry, timestamp)\n\n      for timestamp, hash_entry in iteritems(client_path_history.hash_entries):\n        path_record = self._GetPathRecord(\n            client_path.client_id, path_info, set_default=False)\n        if path_record is None:\n          # TODO(hanuszczak): Provide more details about paths that caused that.\n          raise db.AtLeastOneUnknownPathError([])\n\n        path_record.AddHashEntry(hash_entry, timestamp)", "response": "Writes a collection of hash and stat entries observed for given paths."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nread a collection of hash and stat entries for given paths.", "response": "def ReadPathInfosHistories(self, client_id, path_type, components_list):\n    \"\"\"Reads a collection of hash and stat entries for given paths.\"\"\"\n    results = {}\n\n    for components in components_list:\n      try:\n        path_record = self.path_records[(client_id, path_type, components)]\n      except KeyError:\n        results[components] = []\n        continue\n\n      entries_by_ts = {}\n      for ts, stat_entry in path_record.GetStatEntries():\n        pi = rdf_objects.PathInfo(\n            path_type=path_type,\n            components=components,\n            timestamp=ts,\n            stat_entry=stat_entry)\n        entries_by_ts[ts] = pi\n\n      for ts, hash_entry in path_record.GetHashEntries():\n        try:\n          pi = entries_by_ts[ts]\n        except KeyError:\n          pi = rdf_objects.PathInfo(\n              path_type=path_type, components=components, timestamp=ts)\n          entries_by_ts[ts] = pi\n\n        pi.hash_entry = hash_entry\n\n      results[components] = [\n          entries_by_ts[k] for k in sorted(iterkeys(entries_by_ts))\n      ]\n\n    return results"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nconverts the supported_os property into a condition.", "response": "def ConvertSupportedOSToConditions(src_object):\n  \"\"\"Turn supported_os into a condition.\"\"\"\n  if src_object.supported_os:\n    conditions = \" OR \".join(\"os == '%s'\" % o for o in src_object.supported_os)\n    return conditions"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef GetArtifactCollectorArgs(flow_args, knowledge_base):\n  args = rdf_artifacts.ClientArtifactCollectorArgs()\n  args.knowledge_base = knowledge_base\n\n  args.apply_parsers = flow_args.apply_parsers\n  args.ignore_interpolation_errors = flow_args.ignore_interpolation_errors\n  args.max_file_size = flow_args.max_file_size\n  args.use_tsk = flow_args.use_tsk\n\n  if not flow_args.recollect_knowledge_base:\n    artifact_names = flow_args.artifact_list\n  else:\n    artifact_names = GetArtifactsForCollection(knowledge_base.os,\n                                               flow_args.artifact_list)\n\n  expander = ArtifactExpander(knowledge_base, flow_args.path_type,\n                              flow_args.max_file_size)\n  for artifact_name in artifact_names:\n    rdf_artifact = artifact_registry.REGISTRY.GetArtifact(artifact_name)\n    if not MeetsConditions(knowledge_base, rdf_artifact):\n      continue\n    if artifact_name in expander.processed_artifacts:\n      continue\n    requested_by_user = artifact_name in flow_args.artifact_list\n    for expanded_artifact in expander.Expand(rdf_artifact, requested_by_user):\n      args.artifacts.append(expanded_artifact)\n  return args", "response": "Prepare bundle of artifacts and their dependencies for the client."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nchecks conditions on the source.", "response": "def MeetsConditions(knowledge_base, source):\n  \"\"\"Check conditions on the source.\"\"\"\n  source_conditions_met = True\n  os_conditions = ConvertSupportedOSToConditions(source)\n  if os_conditions:\n    source.conditions.append(os_conditions)\n  for condition in source.conditions:\n    source_conditions_met &= artifact_utils.CheckCondition(\n        condition, knowledge_base)\n\n  return source_conditions_met"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef GetArtifactsForCollection(os_name, artifact_list):\n  artifact_arranger = ArtifactArranger(os_name, artifact_list)\n  artifact_names = artifact_arranger.GetArtifactsInProperOrder()\n  return artifact_names", "response": "Wrapper for ArtifactArranger. GetArtifactsForCollection."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef Expand(self, rdf_artifact, requested):\n\n    source_type = rdf_artifacts.ArtifactSource.SourceType\n\n    expanded_artifact = rdf_artifacts.ExpandedArtifact(\n        name=rdf_artifact.name,\n        provides=rdf_artifact.provides,\n        requested_by_user=requested)\n\n    for source in rdf_artifact.sources:\n      if MeetsConditions(self._knowledge_base, source):\n        type_name = source.type\n\n        if type_name == source_type.ARTIFACT_GROUP:\n          for subartifact in self._ExpandArtifactGroupSource(source, requested):\n            yield subartifact\n          continue\n\n        elif type_name == source_type.ARTIFACT_FILES:\n          expanded_sources = self._ExpandArtifactFilesSource(source, requested)\n\n        else:\n          expanded_sources = self._ExpandBasicSource(source)\n\n        expanded_artifact.sources.Extend(expanded_sources)\n    self.processed_artifacts.add(rdf_artifact.name)\n    if expanded_artifact.sources:\n      yield expanded_artifact", "response": "Expand an artifact by extending its sources."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _ExpandArtifactGroupSource(self, source, requested):\n    artifact_list = []\n    if \"names\" in source.attributes:\n      artifact_list = source.attributes[\"names\"]\n    for artifact_name in artifact_list:\n      if artifact_name in self.processed_artifacts:\n        continue\n      artifact_obj = artifact_registry.REGISTRY.GetArtifact(artifact_name)\n      for expanded_artifact in self.Expand(artifact_obj, requested):\n        yield expanded_artifact", "response": "Recursively expands an artifact group source."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _ExpandArtifactFilesSource(self, source, requested):\n    expanded_source = rdf_artifacts.ExpandedSource(base_source=source)\n    sub_sources = []\n    artifact_list = []\n    if \"artifact_list\" in source.attributes:\n      artifact_list = source.attributes[\"artifact_list\"]\n    for artifact_name in artifact_list:\n      if artifact_name in self.processed_artifacts:\n        continue\n      artifact_obj = artifact_registry.REGISTRY.GetArtifact(artifact_name)\n      for expanded_artifact in self.Expand(artifact_obj, requested):\n        sub_sources.extend(expanded_artifact.sources)\n    expanded_source.artifact_sources = sub_sources\n    expanded_source.path_type = self._path_type\n    return [expanded_source]", "response": "Recursively expands an artifact files source."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ninitializing the dependency graph.", "response": "def _InitializeGraph(self, os_name, artifact_list):\n    \"\"\"Creates the nodes and directed edges of the dependency graph.\n\n    Args:\n      os_name: String specifying the OS name.\n      artifact_list: List of requested artifact names.\n    \"\"\"\n    dependencies = artifact_registry.REGISTRY.SearchDependencies(\n        os_name, artifact_list)\n    artifact_names, attribute_names = dependencies\n\n    self._AddAttributeNodes(attribute_names)\n    self._AddArtifactNodesAndEdges(artifact_names)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _AddAttributeNodes(self, attribute_names):\n    for attribute_name in attribute_names:\n      self.graph[attribute_name] = self.Node(is_artifact=False)", "response": "Adds the attribute nodes to the dependency graph."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nadding the artifact nodes and edges to the graph.", "response": "def _AddArtifactNodesAndEdges(self, artifact_names):\n    \"\"\"Add the artifact nodes to the graph.\n\n    For every artifact that has to be collected, add a node to the dependency\n    graph.\n\n    The edges represent the dependencies. An artifact has outgoing edges to the\n    attributes it provides and incoming edges from attributes it depends on.\n    Initially, only artifacts without incoming edges are reachable. An artifact\n    becomes reachable if all of its dependencies are reachable.\n\n    Args:\n      artifact_names: List of names of the artifacts to collect.\n    \"\"\"\n    for artifact_name in artifact_names:\n      self.graph[artifact_name] = self.Node(is_artifact=True)\n      rdf_artifact = artifact_registry.REGISTRY.GetArtifact(artifact_name)\n      self._AddDependencyEdges(rdf_artifact)\n      self._AddProvidesEdges(rdf_artifact)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nadd an edge for every dependency of the given artifact.", "response": "def _AddDependencyEdges(self, rdf_artifact):\n    \"\"\"Add an edge for every dependency of the given artifact.\n\n    This method gets the attribute names for a given artifact and for every\n    attribute it adds a directed edge from the attribute node to the artifact\n    node. If an artifact does not have any dependencies it is added to the set\n    of reachable nodes.\n\n    Args:\n      rdf_artifact: The artifact object.\n    \"\"\"\n    artifact_dependencies = artifact_registry.GetArtifactPathDependencies(\n        rdf_artifact)\n    if artifact_dependencies:\n      for attribute in artifact_dependencies:\n        self._AddEdge(attribute, rdf_artifact.name)\n    else:\n      self.reachable_nodes.add(rdf_artifact.name)\n      self.graph[rdf_artifact.name].is_provided = True"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nadd an edge for every attribute in the given artifact provides.", "response": "def _AddProvidesEdges(self, rdf_artifact):\n    \"\"\"Add an edge for every attribute the given artifact provides.\n\n    This method adds a directed edge from the artifact node to every attribute\n    this artifact provides.\n\n    Args:\n      rdf_artifact: The artifact object.\n    \"\"\"\n    for attribute in rdf_artifact.provides:\n      self._AddEdge(rdf_artifact.name, attribute)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _AddEdge(self, start_node, end_node):\n\n    self.graph[start_node].outgoing.append(end_node)\n\n    # This check is necessary because an artifact can provide attributes that\n    # are not covered by the graph because they are not relevant for the\n    # requested artifacts.\n    if end_node in self.graph:\n      self.graph[end_node].incoming.append(start_node)", "response": "Adds a directed edge to the graph."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nbring the artifacts in a linear order that resolves dependencies.", "response": "def GetArtifactsInProperOrder(self):\n    \"\"\"Bring the artifacts in a linear order that resolves dependencies.\n\n    This method obtains a linear ordering of the nodes and then returns the list\n    of artifact names.\n\n    Returns:\n      A list of `ArtifactName` instances such that if they are collected in the\n      given order their dependencies are resolved.\n    \"\"\"\n    artifact_list = []\n    while self.reachable_nodes:\n      node_name = self.reachable_nodes.pop()\n      node = self.graph[node_name]\n      if node.is_artifact:\n        artifact_list.append(node_name)\n      for next_node_name in node.outgoing:\n        if next_node_name not in self.graph:\n          continue\n        next_node = self.graph[next_node_name]\n        if next_node.is_provided:\n          continue\n        next_node.incoming.remove(node_name)\n        if not (next_node.is_artifact and next_node.incoming):\n          next_node.is_provided = True\n          self.reachable_nodes.add(next_node_name)\n    return artifact_list"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nconsume an entire range or part thereof.", "response": "def ConsumeRange(self, start, end):\n    \"\"\"Consumes an entire range, or part thereof.\n\n    If the finger has no ranges left, or the curent range start is higher\n    than the end of the consumed block, nothing happens. Otherwise,\n    the current range is adjusted for the consumed block, or removed,\n    if the entire block is consumed. For things to work, the consumed\n    range and the current finger starts must be equal, and the length\n    of the consumed range may not exceed the length of the current range.\n\n    Args:\n      start: Beginning of range to be consumed.\n      end: First offset after the consumed range (end + 1).\n\n    Raises:\n      RuntimeError: if the start position of the consumed range is\n          higher than the start of the current range in the finger, or if\n          the consumed range cuts accross block boundaries.\n    \"\"\"\n    old = self.CurrentRange()\n    if old is None:\n      return\n    if old.start > start:\n      if old.start < end:\n        raise RuntimeError('Block end too high.')\n      return\n    if old.start < start:\n      raise RuntimeError('Block start too high.')\n    if old.end == end:\n      del self.ranges[0]\n    elif old.end > end:\n      self.ranges[0] = Range(end, old.end)\n    else:\n      raise RuntimeError('Block length exceeds range.')"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn the next range of interest in a namedtuple.", "response": "def _GetNextInterval(self):\n    \"\"\"Returns the next Range of the file that is to be hashed.\n\n    For all fingers, inspect their next expected range, and return the\n    lowest uninterrupted range of interest. If the range is larger than\n    BLOCK_SIZE, truncate it.\n\n    Returns:\n      Next range of interest in a Range namedtuple.\n    \"\"\"\n    ranges = [x.CurrentRange() for x in self.fingers]\n    starts = set([r.start for r in ranges if r])\n    ends = set([r.end for r in ranges if r])\n    if not starts:\n      return None\n    min_start = min(starts)\n    starts.remove(min_start)\n    ends |= starts\n    min_end = min(ends)\n    if min_end - min_start > self.BLOCK_SIZE:\n      min_end = min_start + self.BLOCK_SIZE\n    return Range(min_start, min_end)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _HashBlock(self, block, start, end):\n    for finger in self.fingers:\n      expected_range = finger.CurrentRange()\n      if expected_range is None:\n        continue\n      if (start > expected_range.start or\n          (start == expected_range.start and end > expected_range.end) or\n          (start < expected_range.start and end > expected_range.start)):\n        raise RuntimeError('Cutting across fingers.')\n      if start == expected_range.start:\n        finger.HashBlock(block)", "response": "Hashes the given block into the hashers of the fingers."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef HashIt(self):\n    while True:\n      interval = self._GetNextInterval()\n      if interval is None:\n        break\n      self.file.seek(interval.start, os.SEEK_SET)\n      block = self.file.read(interval.end - interval.start)\n      if len(block) != interval.end - interval.start:\n        raise RuntimeError('Short read on file.')\n      self._HashBlock(block, interval.start, interval.end)\n      self._AdjustIntervals(interval.start, interval.end)\n\n    results = []\n    for finger in self.fingers:\n      res = {}\n      leftover = finger.CurrentRange()\n      if leftover:\n        if (len(finger.ranges) > 1 or leftover.start != self.filelength or\n            leftover.end != self.filelength):\n          raise RuntimeError('Non-empty range remains.')\n      res.update(finger.metadata)\n      for hasher in finger.hashers:\n        res[hasher.name] = hasher.digest()\n      results.append(res)\n\n    # Clean out things for a fresh start (on the same file object).\n    self.fingers = []\n\n    # Make sure the results come back in 'standard' order, regardless of the\n    # order in which fingers were added. Helps with reproducing test results.\n    return sorted(results, key=lambda r: r['name'])", "response": "Finalizing function for the Fingerprint class. This method applies all the different hash functions over the input file and computes the resulting hashes."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef EvalGeneric(self, hashers=None):\n    if hashers is None:\n      hashers = Fingerprinter.GENERIC_HASH_CLASSES\n    hashfuncs = [x() for x in hashers]\n    finger = Finger(hashfuncs, [Range(0, self.filelength)], {'name': 'generic'})\n    self.fingers.append(finger)\n    return True", "response": "Evaluates the entire file with the given hash functions."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nparsing PECOFF headers. Reads header magic and some data structures in a file to determine if it is a valid PECOFF header, and figure out the offsets at which relevant data is stored. While this code contains multiple seeks and small reads, that is compensated by the underlying libc buffering mechanism. Returns: None if the parsed file is not PECOFF. A dict with offsets and lengths for CheckSum, CertTable, and SignedData fields in the PECOFF binary, for those that are present.", "response": "def _PecoffHeaderParser(self):\n    \"\"\"Parses PECOFF headers.\n\n    Reads header magic and some data structures in a file to determine if\n    it is a valid PECOFF header, and figure out the offsets at which\n    relevant data is stored.\n    While this code contains multiple seeks and small reads, that is\n    compensated by the underlying libc buffering mechanism.\n\n    Returns:\n      None if the parsed file is not PECOFF.\n      A dict with offsets and lengths for CheckSum, CertTable, and SignedData\n      fields in the PECOFF binary, for those that are present.\n    \"\"\"\n    extents = {}\n    self.file.seek(0, os.SEEK_SET)\n    buf = self.file.read(2)\n    if buf != b'MZ':\n      return None\n    self.file.seek(0x3C, os.SEEK_SET)\n    buf = self.file.read(4)\n    pecoff_sig_offset = struct.unpack('<I', buf)[0]\n    if pecoff_sig_offset >= self.filelength:\n      return None\n    self.file.seek(pecoff_sig_offset, os.SEEK_SET)\n    buf = self.file.read(4)\n    if buf != b'PE\\0\\0':\n      return None\n    self.file.seek(pecoff_sig_offset + 20, os.SEEK_SET)\n    buf = self.file.read(2)\n    optional_header_size = struct.unpack('<H', buf)[0]\n    optional_header_offset = pecoff_sig_offset + 4 + 20\n    if optional_header_size + optional_header_offset > self.filelength:\n      # This is not strictly a failure for windows, but such files better\n      # be treated as generic files. They can not be carrying SignedData.\n      return None\n    if optional_header_size < 68:\n      # We can't do authenticode-style hashing. If this is a valid binary,\n      # which it can be, the header still does not even contain a checksum.\n      return None\n    self.file.seek(optional_header_offset, os.SEEK_SET)\n    buf = self.file.read(2)\n    image_magic = struct.unpack('<H', buf)[0]\n    if image_magic == 0x10b:\n      # 32 bit\n      rva_base = optional_header_offset + 92\n      cert_base = optional_header_offset + 128\n    elif image_magic == 0x20b:\n      # 64 bit\n      rva_base = optional_header_offset + 108\n      cert_base = optional_header_offset + 144\n    else:\n      # A ROM image or such, not in the PE/COFF specs. Not sure what to do.\n      return None\n    extents['CheckSum'] = RelRange(optional_header_offset + 64, 4)\n    self.file.seek(rva_base, os.SEEK_SET)\n    buf = self.file.read(4)\n    number_of_rva = struct.unpack('<I', buf)[0]\n    if (number_of_rva < 5 or\n        optional_header_offset + optional_header_size < cert_base + 8):\n      return extents\n    extents['CertTable'] = RelRange(cert_base, 8)\n\n    self.file.seek(cert_base, os.SEEK_SET)\n    buf = self.file.read(8)\n    start, length = struct.unpack('<II', buf)\n    if (length == 0 or start < optional_header_offset + optional_header_size or\n        start + length > self.filelength):\n      # The location of the SignedData blob is just wrong (or there is none).\n      # Ignore it -- everything else we did still makes sense.\n      return extents\n    extents['SignedData'] = RelRange(start, length)\n    return extents"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _CollectSignedData(self, extent):\n    start, length = extent\n\n    self.file.seek(start, os.SEEK_SET)\n    buf = self.file.read(length)\n    signed_data = []\n    # This loop ignores trailing cruft, or too-short signedData chunks.\n    while len(buf) >= 8:\n      dw_length, w_revision, w_cert_type = struct.unpack('<IHH', buf[:8])\n      if dw_length < 8:\n        # If the entire blob is smaller than its header, bail out.\n        return signed_data\n      b_cert = buf[8:dw_length]\n      buf = buf[(dw_length + 7) & 0x7ffffff8:]\n      signed_data.append((w_revision, w_cert_type, b_cert))\n    return signed_data", "response": "Extracts signedData blob from PECOFF binary and parses first layer."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef EvalPecoff(self, hashers=None):\n    try:\n      extents = self._PecoffHeaderParser()\n    except struct.error:\n      # Parsing the header failed. Just ignore this, and claim\n      # that the file is not a valid PE/COFF image file.\n      extents = None\n    if extents is None:\n      return False\n\n    signed_data = None\n    ranges = []\n    start = 0\n    # Ordering of these conditions matches expected order in file.\n    # If a condition holds true, the matching range is skipped for hashing.\n    if 'CheckSum' in extents:\n      ranges.append(Range(start, end=extents['CheckSum'].start))\n      start = sum(extents['CheckSum'])\n      # New start now points past CheckSum area.\n    if 'CertTable' in extents:\n      ranges.append(Range(start, end=extents['CertTable'].start))\n      start = sum(extents['CertTable'])\n      # New start now points past CertTable area.\n    if 'SignedData' in extents:\n      # Exclude the range even if the blob itself can't be parsed correctly.\n      ranges.append(Range(start, end=extents['SignedData'].start))\n      start = sum(extents['SignedData'])\n      # New start now points past SignedData area.\n      signed_data = self._CollectSignedData(extents['SignedData'])\n    ranges.append(Range(start, end=self.filelength))\n\n    if hashers is None:\n      hashers = Fingerprinter.AUTHENTICODE_HASH_CLASSES\n    hashfuncs = [x() for x in hashers]\n    metadata = {'name': 'pecoff'}\n    if signed_data:\n      metadata['SignedData'] = signed_data\n    finger = Finger(hashfuncs, ranges, metadata)\n    self.fingers.append(finger)\n    return True", "response": "Evaluates the PE or COFF file."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nparse the History file.", "response": "def Parse(self, stat, file_object, knowledge_base):\n    \"\"\"Parse the History file.\"\"\"\n    _, _ = stat, knowledge_base\n    # TODO(user): Convert this to use the far more intelligent plaso parser.\n    ff = Firefox3History(file_object)\n    for timestamp, unused_entry_type, url, title in ff.Parse():\n      yield rdf_webhistory.BrowserHistoryItem(\n          url=url,\n          domain=urlparse.urlparse(url).netloc,\n          access_time=timestamp,\n          program_name=\"Firefox\",\n          source_path=file_object.Path(),\n          title=title)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef Parse(self):\n    for timestamp, url, title in self.Query(self.VISITS_QUERY):\n      if not isinstance(timestamp, (long, int)):\n        timestamp = 0\n\n      yield [timestamp, \"FIREFOX3_VISIT\", url, title]", "response": "Iterator returning dict for each entry in history."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nread a tag from the buffer and return a tuple.", "response": "def ReadTag(buf, pos):\n  \"\"\"Read a tag from the buffer, and return a (tag_bytes, new_pos) tuple.\"\"\"\n  try:\n    start = pos\n    while ORD_MAP_AND_0X80[buf[pos]]:\n      pos += 1\n    pos += 1\n    return (buf[start:pos], pos)\n  except IndexError:\n    raise ValueError(\"Invalid tag\")"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef VarintEncode(value):\n  result = b\"\"\n  if value < 0:\n    raise ValueError(\"Varint can not encode a negative number.\")\n\n  bits = value & 0x7f\n  value >>= 7\n\n  while value:\n    result += HIGH_CHR_MAP[bits]\n    bits = value & 0x7f\n    value >>= 7\n\n  result += CHR_MAP[bits]\n\n  return result", "response": "Convert an integer to a varint and write it using the write function."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nencode a signed integer as a zigzag encoded signed integer.", "response": "def SignedVarintEncode(value):\n  \"\"\"Encode a signed integer as a zigzag encoded signed integer.\"\"\"\n  result = b\"\"\n  if value < 0:\n    value += (1 << 64)\n\n  bits = value & 0x7f\n  value >>= 7\n  while value:\n    result += HIGH_CHR_MAP[bits]\n    bits = value & 0x7f\n    value >>= 7\n  result += CHR_MAP[bits]\n\n  return result"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef VarintReader(buf, pos=0):\n  result = 0\n  shift = 0\n  while 1:\n    b = buf[pos]\n\n    result |= (ORD_MAP_AND_0X7F[b] << shift)\n    pos += 1\n    if not ORD_MAP_AND_0X80[b]:\n      return (result, pos)\n    shift += 7\n    if shift >= 64:\n      raise rdfvalue.DecodeError(\"Too many bytes when decoding varint.\")", "response": "A 64 bit decoder from google. protobuf. internal. decoder."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nsplits the buffer into a list of strings.", "response": "def SplitBuffer(buff, index=0, length=None):\n  \"\"\"Parses the buffer as a prototypes.\n\n  Args:\n    buff: The buffer to parse.\n    index: The position to start parsing.\n    length: Optional length to parse until.\n\n  Yields:\n    Splits the buffer into tuples of strings:\n        (encoded_tag, encoded_length, wire_format).\n  \"\"\"\n  buffer_len = length or len(buff)\n  while index < buffer_len:\n    # data_index is the index where the data begins (i.e. after the tag).\n    encoded_tag, data_index = ReadTag(buff, index)\n\n    tag_type = ORD_MAP[encoded_tag[0]] & TAG_TYPE_MASK\n    if tag_type == WIRETYPE_VARINT:\n      # new_index is the index of the next tag.\n      _, new_index = VarintReader(buff, data_index)\n      yield (encoded_tag, b\"\", buff[data_index:new_index])\n      index = new_index\n\n    elif tag_type == WIRETYPE_FIXED64:\n      yield (encoded_tag, b\"\", buff[data_index:data_index + 8])\n      index = 8 + data_index\n\n    elif tag_type == WIRETYPE_FIXED32:\n      yield (encoded_tag, b\"\", buff[data_index:data_index + 4])\n      index = 4 + data_index\n\n    elif tag_type == WIRETYPE_LENGTH_DELIMITED:\n      # Start index of the string.\n      length, start = VarintReader(buff, data_index)\n      yield (\n          encoded_tag,\n          buff[data_index:start],  # Encoded length.\n          buff[start:start + length])  # Raw data of element.\n      index = start + length\n\n    else:\n      raise rdfvalue.DecodeError(\"Unexpected Tag.\")"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _GetOrderedEntries(data):\n\n  # The raw data dictionary has two kinds of keys: strings (which correspond to\n  # field name) or integers (if the name is unknown). In Python 3 it is not\n  # possible to compare integers and strings to each other, so we first tag each\n  # with either a 0 or 1 (so named fields are going to be serialized first) and\n  # let the lexicographical ordering of the tuples take care of the rest.\n  def Tag(field):\n    \"\"\"Tags field name with a number to make comparison possible.\"\"\"\n\n    # TODO: We use `string_types` here because in Python 2\n    # attribute names (which are passed e.g. through keyword arguments) are\n    # represented as `bytes` whereas in Python 3 it is `unicode`. This should\n    # be replaced with `str` once support for Python 2 is dropped.\n    if isinstance(field, string_types):\n      return 0, field\n    if isinstance(field, int):\n      return 1, field\n\n    message = \"Unexpected field '{}' of type '{}'\".format(field, type(field))\n    raise TypeError(message)\n\n  for field in sorted(iterkeys(data), key=Tag):\n    yield data[field]", "response": "Returns an iterator over the entries of the RDFProtoStruct in a well - defined order."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nserializes given triplets of python and wire values and a descriptor.", "response": "def _SerializeEntries(entries):\n  \"\"\"Serializes given triplets of python and wire values and a descriptor.\"\"\"\n\n  output = []\n  for python_format, wire_format, type_descriptor in entries:\n\n    if wire_format is None or (python_format and\n                               type_descriptor.IsDirty(python_format)):\n      wire_format = type_descriptor.ConvertToWireFormat(python_format)\n\n    precondition.AssertIterableType(wire_format, bytes)\n    output.extend(wire_format)\n\n  return b\"\".join(output)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreading all tags until the next end group and store in the value_obj.", "response": "def ReadIntoObject(buff, index, value_obj, length=0):\n  \"\"\"Reads all tags until the next end group and store in the value_obj.\"\"\"\n  raw_data = value_obj.GetRawData()\n  count = 0\n\n  # Split the buffer into tags and wire_format representations, then collect\n  # these into the raw data cache.\n  for (encoded_tag, encoded_length, encoded_field) in SplitBuffer(\n      buff, index=index, length=length):\n\n    type_info_obj = value_obj.type_infos_by_encoded_tag.get(encoded_tag)\n\n    # Internal format to store parsed fields.\n    wire_format = (encoded_tag, encoded_length, encoded_field)\n\n    # If the tag is not found we need to skip it. Skipped fields are\n    # inaccessible to this actual object, because they have no type info\n    # describing them, however they are still stored in the raw data\n    # representation because they will be re-serialized back. This way\n    # programs which simply read protobufs and write them back do not need to\n    # know all the fields, some of which were defined in a later version of\n    # the application. In order to avoid having to worry about repeated fields\n    # here, we just insert them into the raw data dict with a key which should\n    # be unique.\n    if type_info_obj is None:\n      # Record an unknown field. The key is unique and ensures we do not collide\n      # the dict on repeated fields of the encoded tag. Note that this field is\n      # not really accessible using Get() and does not have a python format\n      # representation. It will be written back using the same wire format it\n      # was read with, therefore does not require a type descriptor at all.\n      raw_data[count] = (None, wire_format, None)\n\n      count += 1\n\n    # Repeated fields are handled especially.\n    elif type_info_obj.__class__ is ProtoList:\n      value_obj.Get(type_info_obj.name).wrapped_list.append((None, wire_format))\n\n    else:\n      # Set the python_format as None so it gets converted lazily on access.\n      raw_data[type_info_obj.name] = (None, wire_format, type_info_obj)\n\n  value_obj.SetRawData(raw_data)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef Validate(self, value, **_):\n    if isinstance(value, rdfvalue.RDFString):\n      # TODO(hanuszczak): Use `str` here.\n      return Text(value)\n\n    if isinstance(value, Text):\n      return value\n\n    if isinstance(value, bytes):\n      return value.decode(\"utf-8\")\n\n    raise type_info.TypeValueError(\"Not a valid unicode string: %r\" % value)", "response": "Validates a python format representation of the value."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef ConvertToWireFormat(self, value):\n    value = value.encode(\"utf8\")\n    return (self.encoded_tag, VarintEncode(len(value)), value)", "response": "Converts the value to wire format."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nchecks that value is a valid enum value.", "response": "def Validate(self, value, **_):\n    \"\"\"Check that value is a valid enum.\"\"\"\n    # None is a valid value - it means the field is not set.\n    if value is None:\n      return\n\n    # If the value is a string we need to try to convert it to an integer.\n    checked_value = value\n    if isinstance(value, string_types):\n      # NOTE: that when initializing from string, enum values are\n      # case-insensitive.\n      checked_value = self.enum.get(value.upper())\n      if checked_value is None and value.isdigit():\n        checked_value = int(value)\n      if checked_value is None:\n        raise type_info.TypeValueError(\n            \"Value %s is not a valid enum value for field %s\" %\n            (value, self.name))\n\n    return EnumNamedValue(checked_value, name=self.reverse_enum.get(value))"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef Definition(self):\n    result = self._FormatDescriptionComment()\n\n    result += \"  enum %s {\\n\" % self.enum_name\n    for k, v in sorted(iteritems(self.reverse_enum)):\n      result += \"    %s = %s;\\n\" % (v, k)\n\n    result += \"  }\\n\"\n\n    result += self._FormatField()\n    return result", "response": "Return a string with the definition of this field."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncheck that value is a valid enum.", "response": "def Validate(self, value, **_):\n    \"\"\"Check that value is a valid enum.\"\"\"\n    if value is None:\n      return\n\n    return rdfvalue.RDFBool(super(ProtoBoolean, self).Validate(value))"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef ConvertFromWireFormat(self, value, container=None):\n    result = self.type()\n    ReadIntoObject(value[2], 0, result)\n\n    return result", "response": "Converts a wire format string to a string."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef ConvertToWireFormat(self, value):\n    output = _SerializeEntries(_GetOrderedEntries(value.GetRawData()))\n    return (self.encoded_tag, VarintEncode(len(output)), output)", "response": "Encode the nested protobuf into wire format."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef LateBind(self, target=None):\n    if not issubclass(target, RDFProtoStruct):\n      raise TypeError(\"Field %s expects a protobuf, but target is %s\" %\n                      (self, target))\n\n    self.late_bound = False\n\n    # The target type is now resolved.\n    self.type = target\n\n    # Register us in our owner.\n    self.owner.AddDescriptor(self)", "response": "Late binding callback.\n\n    This method is called on this field descriptor when the target RDFValue\n    class is finally defined. It gives the field descriptor an opportunity to\n    initialize after the point of definition.\n\n    Args:\n      target: The target nested class.\n\n    Raises:\n      TypeError: If the target class is not of the expected type."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn and clear the dirty state of the python object.", "response": "def IsDirty(self, proto):\n    \"\"\"Return and clear the dirty state of the python object.\"\"\"\n    if proto.dirty:\n      return True\n\n    for python_format, _, type_descriptor in itervalues(proto.GetRawData()):\n      if python_format is not None and type_descriptor.IsDirty(python_format):\n        proto.dirty = True\n        return True\n\n    return False"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nencode the nested protobuf into wire format.", "response": "def ConvertToWireFormat(self, value):\n    \"\"\"Encode the nested protobuf into wire format.\"\"\"\n    data = value.SerializeToString()\n    return (self.encoded_tag, VarintEncode(len(data)), data)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nconvert the wire format message into a value.", "response": "def ConvertFromWireFormat(self, value, container=None):\n    \"\"\"The wire format is an AnyValue message.\"\"\"\n    result = AnyValue()\n    ReadIntoObject(value[2], 0, result)\n    if self._type is not None:\n      converted_value = self._type(container)\n    else:\n      converted_value = self._TypeFromAnyValue(result)\n\n    # If one of the protobuf library wrapper classes is used, unwrap the value.\n    if result.type_url.startswith(\"type.googleapis.com/google.protobuf.\"):\n      wrapper_cls = self.__class__.WRAPPER_BY_TYPE[\n          converted_value.data_store_type]\n      wrapper_value = wrapper_cls()\n      wrapper_value.ParseFromString(result.value)\n      return converted_value.FromDatastoreValue(wrapper_value.value)\n    else:\n      # TODO(user): Type stored in type_url is currently ignored when value\n      # is decoded. We should use it to deserialize the value and then check\n      # that value type and dynamic type are compatible.\n      return converted_value.FromSerializedString(result.value)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef ConvertToWireFormat(self, value):\n    # Is it a protobuf-based value?\n    if hasattr(value.__class__, \"protobuf\"):\n      if value.__class__.protobuf:\n        type_name = (\"type.googleapis.com/%s\" %\n                     value.__class__.protobuf.__name__)\n      else:\n        type_name = value.__class__.__name__\n      data = value.SerializeToString()\n    # Is it a primitive value?\n    elif hasattr(value.__class__, \"data_store_type\"):\n      wrapper_cls = self.__class__.WRAPPER_BY_TYPE[\n          value.__class__.data_store_type]\n      wrapped_data = wrapper_cls()\n      wrapped_data.value = value.SerializeToDataStore()\n\n      type_name = (\"type.googleapis.com/google.protobuf.%s\" %\n                   wrapper_cls.__name__)\n      data = wrapped_data.SerializeToString()\n    else:\n      raise ValueError(\"Can't convert value %s to an protobuf.Any value.\" %\n                       value)\n\n    any_value = AnyValue(type_url=type_name, value=data)\n    output = _SerializeEntries(_GetOrderedEntries(any_value.GetRawData()))\n\n    return (self.encoded_tag, VarintEncode(len(output)), output)", "response": "Encode the nested protobuf into wire format."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef Validate(self, value, **_):\n    # Assigning from same kind can allow us to skip verification since all\n    # elements in a RepeatedFieldHelper already are coerced to the delegate\n    # type. In that case we just make a copy. This only works when the value\n    # wraps the same type as us.\n    if (value.__class__ is RepeatedFieldHelper and\n        value.type_descriptor is self.delegate):\n      result = value.Copy()\n\n    # Make sure the base class finds the value valid.\n    else:\n      # The value may be a generator here, so we just iterate over it.\n      result = RepeatedFieldHelper(type_descriptor=self.delegate)\n      result.Extend(value)\n\n    return result", "response": "Check that value is a list of the required type."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nconverts the value to the wire format.", "response": "def ConvertToWireFormat(self, value):\n    \"\"\"Convert to the wire format.\n\n    Args:\n      value: is of type RepeatedFieldHelper.\n\n    Returns:\n      A wire format representation of the value.\n    \"\"\"\n    output = _SerializeEntries(\n        (python_format, wire_format, value.type_descriptor)\n        for (python_format, wire_format) in value.wrapped_list)\n    return b\"\", b\"\", output"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nadds a descriptor to the current descriptor.", "response": "def AddDescriptor(self, field_desc):\n    \"\"\"This method will be called by our delegate during late binding.\"\"\"\n    # Just relay it up to our owner.\n    self.late_bound = False\n    self.delegate = field_desc\n    self.wire_type = self.delegate.wire_type\n    self.owner.AddDescriptor(self)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef LateBind(self, target=None):\n    self.type = target\n    self._GetPrimitiveEncoder()\n\n    # Now re-add the descriptor to the owner protobuf.\n    self.late_bound = False\n    self.owner.AddDescriptor(self)", "response": "Bind the field descriptor to the owner once the target is defined."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _GetPrimitiveEncoder(self):\n    # Decide what should the primitive type be for packing the target rdfvalue\n    # into the protobuf and create a delegate descriptor to control that.\n    primitive_cls = self._PROTO_DATA_STORE_LOOKUP[self.type.data_store_type]\n    self.primitive_desc = primitive_cls(**self._kwargs)\n\n    # Our wiretype is the same as the delegate's.\n    self.wire_type = self.primitive_desc.wire_type\n    self.proto_type_name = self.primitive_desc.proto_type_name\n\n    # Recalculate our tags.\n    self.CalculateTags()", "response": "Finds the primitive encoder according to the type s data_store_type."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef Copy(self, field_number=None):\n    new_args = self._kwargs.copy()\n    if field_number is not None:\n      new_args[\"field_number\"] = field_number\n\n    return ProtoRDFValue(\n        rdf_type=self.original_proto_type_name,\n        default=getattr(self, \"default\", None),\n        **new_args)", "response": "Returns a copy of this descriptor."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef AsPrimitiveProto(self):\n    if self.protobuf:\n      result = self.protobuf()\n      result.ParseFromString(self.SerializeToString())\n      return result", "response": "Return an old style protocol buffer object."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ninitializing the internal structure from a given dictionary.", "response": "def FromDict(self, dictionary):\n    \"\"\"Initializes itself from a given dictionary.\"\"\"\n    dynamic_fields = []\n\n    for key, value in iteritems(dictionary):\n      field_type_info = self.type_infos.get(key)\n      if isinstance(field_type_info, ProtoEmbedded):\n        nested_value = field_type_info.GetDefault(container=self)\n        nested_value.FromDict(value)\n        self.Set(key, nested_value)\n      elif isinstance(field_type_info, ProtoList):\n        if isinstance(field_type_info.delegate, ProtoEmbedded):\n          nested_values = []\n          for v in value:\n            nested_value = field_type_info.delegate.GetDefault(container=self)\n            nested_value.FromDict(v)\n            nested_values.append(nested_value)\n\n          self.Set(key, nested_values)\n        else:\n          self.Set(key, value)\n      elif isinstance(field_type_info, ProtoDynamicEmbedded):\n        dynamic_fields.append(field_type_info)\n      elif field_type_info.proto_type_name == \"bytes\":\n        self.Set(key, base64.decodestring(value or \"\"))\n      else:\n        self.Set(key, value)\n\n    # Process dynamic fields after all other fields, because most probably\n    # their class is determined by one of the previously set fields.\n    for dynamic_field in dynamic_fields:\n      nested_value = dynamic_field.GetDefault(container=self)\n      if nested_value is None:\n        raise RuntimeError(\"Can't initialize dynamic field %s, probably some \"\n                           \"necessary fields weren't supplied.\" %\n                           dynamic_field.name)\n      nested_value.FromDict(dictionary[dynamic_field.name])\n      self.Set(dynamic_field.name, nested_value)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nemit .proto file definitions.", "response": "def EmitProto(cls):\n    \"\"\"Emits .proto file definitions.\"\"\"\n    result = \"message %s {\\n\" % cls.__name__\n    for _, desc in sorted(iteritems(cls.type_infos_by_field_number)):\n      result += desc.Definition()\n\n    result += \"}\\n\"\n    return result"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef FromTextFormat(cls, text):\n    tmp = cls.protobuf()  # pylint: disable=not-callable\n    text_format.Merge(text, tmp)\n\n    return cls.FromSerializedString(tmp.SerializeToString())", "response": "Parse this object from a text representation."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef AddDescriptor(cls, field_desc):\n    if not isinstance(field_desc, ProtoType):\n      raise type_info.TypeValueError(\n          \"%s field '%s' should be of type ProtoType\" %\n          (cls.__name__, field_desc.name))\n\n    # Ensure the field descriptor knows the class that owns it.\n    field_desc.SetOwner(cls)\n\n    # If the field is late bound we do not really add it to the descriptor set\n    # yet. We must wait for the LateBindingPlaceHolder() to add it later.\n    if field_desc.late_bound:\n      # Keep track of unbound fields.\n      cls.late_bound_type_infos[field_desc.name] = field_desc\n      return\n\n    # Ensure this field number is unique:\n    if field_desc.field_number in cls.type_infos_by_field_number:\n      raise type_info.TypeValueError(\n          \"Field number %s for field %s is not unique in %s\" %\n          (field_desc.field_number, field_desc.name, cls.__name__))\n\n    # We store an index of the type info by tag values to speed up parsing.\n    cls.type_infos_by_field_number[field_desc.field_number] = field_desc\n    cls.type_infos_by_encoded_tag[field_desc.encoded_tag] = field_desc\n\n    cls.type_infos.Append(field_desc)\n    cls.late_bound_type_infos.pop(field_desc.name, None)\n\n    # Add direct accessors only if the class does not already have them.\n    if not hasattr(cls, field_desc.name):\n      # This lambda is a class method so pylint: disable=protected-access\n      # This is much faster than __setattr__/__getattr__\n      setattr(\n          cls, field_desc.name,\n          property(lambda self: self.Get(field_desc.name), lambda self, x: self.\n                   _Set(x, field_desc), None, field_desc.description))", "response": "Adds a descriptor to the Proto Struct."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef RunInstaller():\n\n  try:\n    os.makedirs(os.path.dirname(config.CONFIG[\"Installer.logfile\"]))\n  except OSError:\n    pass\n\n  # Always log to the installer logfile at debug level. This way if our\n  # installer fails we can send detailed diagnostics.\n  handler = logging.FileHandler(config.CONFIG[\"Installer.logfile\"], mode=\"wb\")\n\n  handler.setLevel(logging.DEBUG)\n\n  # Add this to the root logger.\n  logging.getLogger().addHandler(handler)\n\n  # Ordinarily when the client starts up, the local volatile\n  # configuration is read. Howevwer, when running the installer, we\n  # need to ensure that only the installer configuration is used so\n  # nothing gets overridden by local settings. We there must reload\n  # the configuration from the flag and ignore the Config.writeback\n  # location.\n  config.CONFIG.Initialize(filename=flags.FLAGS.config, reset=True)\n  config.CONFIG.AddContext(contexts.INSTALLER_CONTEXT,\n                           \"Context applied when we run the client installer.\")\n\n  logging.warning(\"Starting installation procedure for GRR client.\")\n  try:\n    Installer().Init()\n  except Exception as e:  # pylint: disable=broad-except\n    # Ouch! we failed to install... Not a lot we can do\n    # here - just log the error and give up.\n    logging.exception(\"Installation failed: %s\", e)\n\n    # Error return status.\n    sys.exit(-1)\n\n  # Exit successfully.\n  sys.exit(0)", "response": "Run all the current installers and exit the process."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef MapItemsIterator(function, items):\n  return ItemsIterator(\n      items=map(function, items), total_count=items.total_count)", "response": "Maps ItemsIterator via given function."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef Poll(generator=None, condition=None, interval=None, timeout=None):\n\n  if not generator:\n    raise ValueError(\"generator has to be a lambda\")\n\n  if not condition:\n    raise ValueError(\"condition has to be a lambda\")\n\n  if interval is None:\n    interval = DEFAULT_POLL_INTERVAL\n\n  if timeout is None:\n    timeout = DEFAULT_POLL_TIMEOUT\n\n  started = time.time()\n  while True:\n    obj = generator()\n    check_result = condition(obj)\n    if check_result:\n      return obj\n\n    if timeout and (time.time() - started) > timeout:\n      raise errors.PollTimeoutError(\n          \"Polling on %s timed out after %ds.\" % (obj, timeout))\n    time.sleep(interval)", "response": "Periodically calls generator function until a condition is satisfied."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nconverts given URN string to a client id string.", "response": "def UrnStringToClientId(urn):\n  \"\"\"Converts given URN string to a client id string.\"\"\"\n  if urn.startswith(AFF4_PREFIX):\n    urn = urn[len(AFF4_PREFIX):]\n\n  components = urn.split(\"/\")\n  return components[0]"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef UrnStringToHuntId(urn):\n  if urn.startswith(AFF4_PREFIX):\n    urn = urn[len(AFF4_PREFIX):]\n\n  components = urn.split(\"/\")\n  if len(components) != 2 or components[0] != \"hunts\":\n    raise ValueError(\"Invalid hunt URN: %s\" % urn)\n\n  return components[-1]", "response": "Converts given URN string to a flow id string."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns a message instance corresponding to a given type URL.", "response": "def TypeUrlToMessage(type_url):\n  \"\"\"Returns a message instance corresponding to a given type URL.\"\"\"\n\n  if not type_url.startswith(TYPE_URL_PREFIX):\n    raise ValueError(\"Type URL has to start with a prefix %s: %s\" %\n                     (TYPE_URL_PREFIX, type_url))\n\n  full_name = type_url[len(TYPE_URL_PREFIX):]\n  try:\n    return symbol_database.Default().GetSymbol(full_name)()\n  except KeyError as e:\n    raise ProtobufTypeNotFound(str(e))"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nregister all API - releated descriptors in a given symbol DB.", "response": "def RegisterProtoDescriptors(db, *additional_descriptors):\n  \"\"\"Registers all API-releated descriptors in a given symbol DB.\"\"\"\n  db.RegisterFileDescriptor(artifact_pb2.DESCRIPTOR)\n  db.RegisterFileDescriptor(client_pb2.DESCRIPTOR)\n  db.RegisterFileDescriptor(config_pb2.DESCRIPTOR)\n  db.RegisterFileDescriptor(cron_pb2.DESCRIPTOR)\n  db.RegisterFileDescriptor(flow_pb2.DESCRIPTOR)\n  db.RegisterFileDescriptor(hunt_pb2.DESCRIPTOR)\n  db.RegisterFileDescriptor(output_plugin_pb2.DESCRIPTOR)\n  db.RegisterFileDescriptor(reflection_pb2.DESCRIPTOR)\n  db.RegisterFileDescriptor(stats_pb2.DESCRIPTOR)\n  db.RegisterFileDescriptor(user_pb2.DESCRIPTOR)\n  db.RegisterFileDescriptor(vfs_pb2.DESCRIPTOR)\n\n  db.RegisterFileDescriptor(checks_pb2.DESCRIPTOR)\n  db.RegisterFileDescriptor(deprecated_pb2.DESCRIPTOR)\n  db.RegisterFileDescriptor(flows_pb2.DESCRIPTOR)\n  db.RegisterFileDescriptor(jobs_pb2.DESCRIPTOR)\n  db.RegisterFileDescriptor(osquery_pb2.DESCRIPTOR)\n  db.RegisterFileDescriptor(wrappers_pb2.DESCRIPTOR)\n\n  for d in additional_descriptors:\n    db.RegisterFileDescriptor(d)"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nconverts python datetime to epoch microseconds.", "response": "def _DateToEpoch(date):\n  \"\"\"Converts python datetime to epoch microseconds.\"\"\"\n  tz_zero = datetime.datetime.utcfromtimestamp(0)\n  diff_sec = int((date - tz_zero).total_seconds())\n  return diff_sec * 1000000"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nparsing the stat entries.", "response": "def ParseMultiple(self, stat_entries, knowledge_base):\n    \"\"\"Parse the StatEntry objects.\"\"\"\n    _ = knowledge_base\n\n    for stat_entry in stat_entries:\n      # TODO: `st_mode` has to be an `int`, not `StatMode`.\n      if stat.S_ISDIR(int(stat_entry.st_mode)):\n        homedir = stat_entry.pathspec.path\n        username = os.path.basename(homedir)\n        if username not in self.blacklist:\n          yield rdf_client.User(username=username, homedir=homedir)"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nparses the system profiler output.", "response": "def Parse(self, cmd, args, stdout, stderr, return_val, time_taken,\n            knowledge_base):\n    \"\"\"Parse the system profiler output. We get it in the form of a plist.\"\"\"\n    _ = stderr, time_taken, args, knowledge_base  # Unused\n    self.CheckReturn(cmd, return_val)\n\n    plist = biplist.readPlist(io.BytesIO(stdout))\n\n    if len(plist) > 1:\n      raise parser.ParseError(\"SPHardwareDataType plist has too many items.\")\n\n    hardware_list = plist[0][\"_items\"][0]\n    serial_number = hardware_list.get(\"serial_number\", None)\n    system_product_name = hardware_list.get(\"machine_model\", None)\n    bios_version = hardware_list.get(\"boot_rom_version\", None)\n\n    yield rdf_client.HardwareInfo(\n        serial_number=serial_number,\n        bios_version=bios_version,\n        system_product_name=system_product_name)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef Parse(self, statentry, file_object, knowledge_base):\n    _ = knowledge_base\n    kwargs = {}\n    try:\n      kwargs[\"aff4path\"] = file_object.urn\n    except AttributeError:\n      pass\n\n    direct_copy_items = [\n        \"Label\", \"Disabled\", \"UserName\", \"GroupName\", \"Program\",\n        \"StandardInPath\", \"StandardOutPath\", \"StandardErrorPath\",\n        \"LimitLoadToSessionType\", \"EnableGlobbing\", \"EnableTransactions\",\n        \"OnDemand\", \"RunAtLoad\", \"RootDirectory\", \"WorkingDirectory\", \"Umask\",\n        \"TimeOut\", \"ExitTimeOut\", \"ThrottleInterval\", \"InitGroups\",\n        \"StartOnMount\", \"StartInterval\", \"Debug\", \"WaitForDebugger\", \"Nice\",\n        \"ProcessType\", \"AbandonProcessGroup\", \"LowPriorityIO\", \"LaunchOnlyOnce\"\n    ]\n\n    string_array_items = [\n        \"LimitLoadToHosts\", \"LimitLoadFromHosts\", \"LimitLoadToSessionType\",\n        \"ProgramArguments\", \"WatchPaths\", \"QueueDirectories\"\n    ]\n\n    flag_only_items = [\"SoftResourceLimits\", \"HardResourceLimits\", \"Sockets\"]\n\n    plist = {}\n\n    try:\n      plist = biplist.readPlist(file_object)\n    except (biplist.InvalidPlistException, ValueError, IOError) as e:\n      plist[\"Label\"] = \"Could not parse plist: %s\" % e\n\n    # These are items that can be directly copied\n    for key in direct_copy_items:\n      kwargs[key] = plist.get(key)\n\n    # These could be a string, they could be an array, we don't know and neither\n    # does Apple so we check.\n    for key in string_array_items:\n      elements = plist.get(key)\n      if isinstance(elements, string_types):\n        kwargs[key] = [elements]\n      else:\n        kwargs[key] = elements\n\n    # These are complex items that can appear in multiple data structures\n    # so we only flag on their existence\n    for key in flag_only_items:\n      if plist.get(key):\n        kwargs[key] = True\n\n    if plist.get(\"inetdCompatability\") is not None:\n      kwargs[\"inetdCompatabilityWait\"] = plist.get(\"inetdCompatability\").get(\n          \"Wait\")\n\n    keepalive = plist.get(\"KeepAlive\")\n    if isinstance(keepalive, bool) or keepalive is None:\n      kwargs[\"KeepAlive\"] = keepalive\n    else:\n      keepalivedict = {}\n      keepalivedict[\"SuccessfulExit\"] = keepalive.get(\"SuccessfulExit\")\n      keepalivedict[\"NetworkState\"] = keepalive.get(\"NetworkState\")\n\n      pathstates = keepalive.get(\"PathState\")\n      if pathstates is not None:\n        keepalivedict[\"PathState\"] = []\n        for pathstate in pathstates:\n          keepalivedict[\"PathState\"].append(\n              rdf_plist.PlistBoolDictEntry(\n                  name=pathstate, value=pathstates[pathstate]))\n\n      otherjobs = keepalive.get(\"OtherJobEnabled\")\n      if otherjobs is not None:\n        keepalivedict[\"OtherJobEnabled\"] = []\n        for otherjob in otherjobs:\n          keepalivedict[\"OtherJobEnabled\"].append(\n              rdf_plist.PlistBoolDictEntry(\n                  name=otherjob, value=otherjobs[otherjob]))\n      kwargs[\"KeepAliveDict\"] = rdf_plist.LaunchdKeepAlive(**keepalivedict)\n\n    envvars = plist.get(\"EnvironmentVariables\")\n    if envvars is not None:\n      kwargs[\"EnvironmentVariables\"] = []\n      for envvar in envvars:\n        kwargs[\"EnvironmentVariables\"].append(\n            rdf_plist.PlistStringDictEntry(name=envvar, value=envvars[envvar]))\n\n    startcalendarinterval = plist.get(\"StartCalendarInterval\")\n    if startcalendarinterval is not None:\n      if isinstance(startcalendarinterval, dict):\n        kwargs[\"StartCalendarInterval\"] = [\n            rdf_plist.LaunchdStartCalendarIntervalEntry(\n                Minute=startcalendarinterval.get(\"Minute\"),\n                Hour=startcalendarinterval.get(\"Hour\"),\n                Day=startcalendarinterval.get(\"Day\"),\n                Weekday=startcalendarinterval.get(\"Weekday\"),\n                Month=startcalendarinterval.get(\"Month\"))\n        ]\n      else:\n        kwargs[\"StartCalendarInterval\"] = []\n        for entry in startcalendarinterval:\n          kwargs[\"StartCalendarInterval\"].append(\n              rdf_plist.LaunchdStartCalendarIntervalEntry(\n                  Minute=entry.get(\"Minute\"),\n                  Hour=entry.get(\"Hour\"),\n                  Day=entry.get(\"Day\"),\n                  Weekday=entry.get(\"Weekday\"),\n                  Month=entry.get(\"Month\")))\n\n    yield rdf_plist.LaunchdPlist(**kwargs)", "response": "Parse the Plist file."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nparsing the install history plist file.", "response": "def Parse(self, statentry, file_object, knowledge_base):\n    \"\"\"Parse the Plist file.\"\"\"\n\n    plist = biplist.readPlist(file_object)\n\n    if not isinstance(plist, list):\n      raise parser.ParseError(\n          \"InstallHistory plist is a '%s', expecting a list\" % type(plist))\n\n    packages = []\n    for sw in plist:\n      packages.append(\n          rdf_client.SoftwarePackage(\n              name=sw.get(\"displayName\"),\n              version=sw.get(\"displayVersion\"),\n              description=\",\".join(sw.get(\"packageIdentifiers\")),\n              # TODO(hanuszczak): make installed_on an RDFDatetime\n              installed_on=_DateToEpoch(sw.get(\"date\")),\n              install_state=rdf_client.SoftwarePackage.InstallState.INSTALLED))\n\n    if packages:\n      yield rdf_client.SoftwarePackages(packages=packages)"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nadds the running contexts to the config system.", "response": "def SetPlatformArchContext():\n  \"\"\"Add the running contexts to the config system.\"\"\"\n\n  # Initialize the running platform context:\n  _CONFIG.AddContext(\"Platform:%s\" % platform.system().title())\n\n  machine = platform.uname()[4]\n  if machine in [\"x86_64\", \"AMD64\", \"i686\"]:\n    # 32 bit binaries running on AMD64 will still have a i386 arch.\n    if platform.architecture()[0] == \"32bit\":\n      arch = \"i386\"\n    else:\n      arch = \"amd64\"\n  elif machine == \"x86\":\n    arch = \"i386\"\n  else:\n    arch = machine\n\n  _CONFIG.AddContext(\"Arch:%s\" % arch)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nparsing given YAML file.", "response": "def _ParseYamlFromFile(filedesc):\n  \"\"\"Parses given YAML file.\"\"\"\n  content = filedesc.read()\n  return yaml.Parse(content) or collections.OrderedDict()"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef DEFINE_choice(name, default, choices, help):\n  _CONFIG.DEFINE_choice(name, default, choices, help)", "response": "A helper for defining choice string options."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef DEFINE_multichoice(name, default, choices, help):\n  _CONFIG.DEFINE_multichoice(name, default, choices, help)", "response": "Defines a new option for a list of choices."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ninitialize a ConfigManager object with the specified options.", "response": "def LoadConfig(config_obj,\n               config_file=None,\n               config_fd=None,\n               secondary_configs=None,\n               contexts=None,\n               reset=False,\n               parser=ConfigFileParser):\n  \"\"\"Initialize a ConfigManager with the specified options.\n\n  Args:\n    config_obj: The ConfigManager object to use and update. If None, one will be\n      created.\n    config_file: Filename to read the config from.\n    config_fd: A file-like object to read config data from.\n    secondary_configs: A list of secondary config URLs to load.\n    contexts: Add these contexts to the config object.\n    reset: Completely wipe previous config before doing the load.\n    parser: Specify which parser to use.\n\n  Returns:\n    The resulting config object. The one passed in, unless None was specified.\n  \"\"\"\n  if config_obj is None or reset:\n    # Create a new config object.\n    config_obj = _CONFIG.MakeNewConfig()\n\n  # Initialize the config with a filename or file like object.\n  if config_file is not None:\n    config_obj.Initialize(filename=config_file, must_exist=True, parser=parser)\n  elif config_fd is not None:\n    config_obj.Initialize(fd=config_fd, parser=parser)\n\n  # Load all secondary files.\n  if secondary_configs:\n    for config_file in secondary_configs:\n      config_obj.LoadSecondaryConfig(config_file)\n\n  if contexts:\n    for context in contexts:\n      config_obj.AddContext(context)\n\n  return config_obj"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nparsing all the command line options which control the config system.", "response": "def ParseConfigCommandLine():\n  \"\"\"Parse all the command line options which control the config system.\"\"\"\n  # The user may specify the primary config file on the command line.\n  if flags.FLAGS.config:\n    _CONFIG.Initialize(filename=flags.FLAGS.config, must_exist=True)\n  else:\n    raise RuntimeError(\"A config file is not specified.\")\n\n  # Allow secondary configuration files to be specified.\n  if flags.FLAGS.secondary_configs:\n    for config_file in flags.FLAGS.secondary_configs:\n      _CONFIG.LoadSecondaryConfig(config_file)\n\n  # Allow individual options to be specified as global overrides.\n  for statement in flags.FLAGS.parameter:\n    if \"=\" not in statement:\n      raise RuntimeError(\"statement %s on command line not valid.\" % statement)\n\n    name, value = statement.split(\"=\", 1)\n    _CONFIG.global_override[name] = value\n\n  # Load additional contexts from the command line.\n  for context in flags.FLAGS.context:\n    if context:\n      _CONFIG.AddContext(context)\n\n  if _CONFIG[\"Config.writeback\"]:\n    _CONFIG.SetWriteBack(_CONFIG[\"Config.writeback\"])\n\n  # Does the user want to dump help? We do this after the config system is\n  # initialized so the user can examine what we think the value of all the\n  # parameters are.\n  if flags.FLAGS.config_help:\n    print(\"Configuration overview.\")\n\n    _CONFIG.PrintHelp()\n    sys.exit(0)"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nuses pkg_resources to find the path to the required resource.", "response": "def Filter(self, filename_spec):\n    \"\"\"Use pkg_resources to find the path to the required resource.\"\"\"\n    if \"@\" in filename_spec:\n      file_path, package_name = filename_spec.split(\"@\")\n    else:\n      file_path, package_name = filename_spec, Resource.default_package\n\n    resource_path = package.ResourcePath(package_name, file_path)\n    if resource_path is not None:\n      return resource_path\n\n\n    # pylint: disable=unreachable\n    raise FilterError(\n        \"Unable to find resource %s while interpolating: \" % filename_spec)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nsaving the raw data to the config file.", "response": "def SaveDataToFD(self, raw_data, fd):\n    \"\"\"Merge the raw data with the config file and store it.\"\"\"\n    for key, value in iteritems(raw_data):\n      # TODO(hanuszczak): Incorrect type specification for `set`.\n      # pytype: disable=wrong-arg-types\n      self.set(\"\", key, value=value)\n      # pytype: enable=wrong-arg-types\n\n    self.write(fd)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nsave the raw data as our configuration.", "response": "def SaveData(self, raw_data):\n    \"\"\"Store the raw data as our configuration.\"\"\"\n    if self.filename is None:\n      raise IOError(\"Unknown filename\")\n\n    logging.info(\"Writing back configuration to file %s\", self.filename)\n    # Ensure intermediate directories exist\n    try:\n      os.makedirs(os.path.dirname(self.filename))\n    except (IOError, OSError):\n      pass\n\n    try:\n      # We can not use the standard open() call because we need to\n      # enforce restrictive file permissions on the created file.\n      mode = os.O_WRONLY | os.O_CREAT | os.O_TRUNC\n      fd = os.open(self.filename, mode, 0o600)\n      with os.fdopen(fd, \"wb\") as config_file:\n        self.SaveDataToFD(raw_data, config_file)\n\n    except OSError as e:\n      logging.warning(\"Unable to write config file %s: %s.\", self.filename, e)"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef SaveDataToFD(self, raw_data, fd):\n    fd.write(yaml.Dump(raw_data).encode(\"utf-8\"))", "response": "Save the raw data to the config file."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _RawData(self, data):\n    if not isinstance(data, dict):\n      return data\n\n    result = collections.OrderedDict()\n    for k, v in iteritems(data):\n      result[k] = self._RawData(v)\n\n    return result", "response": "Convert data to common format."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef Escape(self, string=\"\", **_):\n    # Translate special escapes:\n    self.stack[-1] += self.STRING_ESCAPES.get(string, string)", "response": "Support standard string escaping."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef Filter(self, match=None, **_):\n    arg = self.stack.pop(-1)\n\n    # Filters can be specified as a comma separated list.\n    for filter_name in match.group(1).split(\",\"):\n      filter_object = ConfigFilter.classes_by_name.get(filter_name)\n      if filter_object is None:\n        raise FilterError(\"Unknown filter function %r\" % filter_name)\n\n      if not filter_object.sensitive_arg:\n        logging.debug(\"Applying filter %s for %s.\", filter_name, arg)\n      arg = filter_object().Filter(arg)\n      precondition.AssertType(arg, Text)\n\n    self.stack[-1] += arg", "response": "Filter the current expression."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef ExpandArg(self, **_):\n    # This function is called when we see close ) and the stack depth has to\n    # exactly match the number of (.\n    if len(self.stack) <= 1:\n      raise lexer.ParseError(\n          \"Unbalanced parenthesis: Can not expand '%s'\" % self.processed_buffer)\n\n    # This is the full parameter name: e.g. Logging.path\n    parameter_name = self.stack.pop(-1)\n    if \".\" not in parameter_name:\n      parameter_name = \"%s.%s\" % (self.default_section, parameter_name)\n\n    final_value = self.config.Get(parameter_name, context=self.context)\n    if final_value is None:\n      final_value = \"\"\n\n    type_info_obj = (\n        self.config.FindTypeInfo(parameter_name) or type_info.String())\n\n    # Encode the interpolated string according to its type.\n    self.stack[-1] += type_info_obj.ToString(final_value)", "response": "Expand the args as a section. parameter from the config."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef MakeNewConfig(self):\n    result = self.__class__()\n\n    # We do not need to copy here since these never change.\n    result.type_infos = self.type_infos\n    result.defaults = self.defaults\n    result.context = self.context\n    result.valid_contexts = self.valid_contexts\n\n    return result", "response": "Creates a new configuration object based on this one."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef CopyConfig(self):\n    newconf = self.MakeNewConfig()\n    newconf.raw_data = copy.deepcopy(self.raw_data)\n    newconf.files = copy.deepcopy(self.files)\n    newconf.secondary_config_parsers = copy.deepcopy(\n        self.secondary_config_parsers)\n    newconf.writeback = copy.deepcopy(self.writeback)\n    newconf.writeback_data = copy.deepcopy(self.writeback_data)\n    newconf.global_override = copy.deepcopy(self.global_override)\n    newconf.context_descriptions = copy.deepcopy(self.context_descriptions)\n    newconf.constants = copy.deepcopy(self.constants)\n    newconf.initialized = copy.deepcopy(self.initialized)\n    return newconf", "response": "Make a complete copy of the current config."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nsets the main config file which will receive any modifications.", "response": "def SetWriteBack(self, filename):\n    \"\"\"Sets the config file which will receive any modifications.\n\n    The main config file can be made writable, but directing all Set()\n    operations into a secondary location. This secondary location will\n    receive any updates and will override the options for this file.\n\n    Args:\n      filename: A filename which will receive updates. The file is parsed first\n        and merged into the raw data from this object.\n    \"\"\"\n    try:\n      self.writeback = self.LoadSecondaryConfig(filename)\n      self.MergeData(self.writeback.RawData(), self.writeback_data)\n    except IOError as e:\n      # This means that we probably aren't installed correctly.\n      logging.error(\"Unable to read writeback file: %s\", e)\n      return\n    except Exception as we:  # pylint: disable=broad-except\n      # Could be yaml parse error, could be some malformed parameter. Move the\n      # writeback file so that we start in a clean state next run\n      if os.path.exists(filename):\n        try:\n          b = filename + \".bak\"\n          os.rename(filename, b)\n          logging.warning(\"Broken writeback (%s) renamed to: %s\", we, b)\n        except Exception as e:  # pylint: disable=broad-except\n          logging.error(\"Unable to rename broken writeback: %s\", e)\n      raise we\n    logging.debug(\"Configuration writeback is set to %s\", filename)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef Validate(self, sections=None, parameters=None):\n    if isinstance(sections, string_types):\n      sections = [sections]\n\n    if sections is None:\n      sections = []\n\n    if parameters is None:\n      parameters = []\n\n    validation_errors = {}\n    for section in sections:\n      for descriptor in self.type_infos:\n        if descriptor.name.startswith(section + \".\"):\n          try:\n            self.Get(descriptor.name)\n          except (Error, ValueError) as e:\n            validation_errors[descriptor.name] = e\n\n    for parameter in parameters:\n      for descriptor in self.type_infos:\n        if parameter == descriptor.name:\n          try:\n            self.Get(descriptor.name)\n          except (Error, ValueError) as e:\n            validation_errors[descriptor.name] = e\n\n    return validation_errors", "response": "Validates the GRR configuration file."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef AddContext(self, context_string, description=None):\n    if context_string not in self.context:\n      if context_string not in self.valid_contexts:\n        raise InvalidContextError(\n            \"Invalid context specified: %s\" % context_string)\n\n      self.context.append(context_string)\n      self.context_descriptions[context_string] = description\n\n    self.FlushCache()", "response": "Adds a context string to the global configuration."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef SetRaw(self, name, value):\n    if self.writeback is None:\n      logging.warning(\"Attempting to modify a read only config object.\")\n    if name in self.constants:\n      raise ConstModificationError(\n          \"Attempting to modify constant value %s\" % name)\n\n    self.writeback_data[name] = value\n    self.FlushCache()", "response": "Set the raw string without verification or escaping."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef Set(self, name, value):\n    # If the configuration system has a write back location we use it,\n    # otherwise we use the primary configuration object.\n    if self.writeback is None:\n      logging.warning(\"Attempting to modify a read only config object for %s.\",\n                      name)\n    if name in self.constants:\n      raise ConstModificationError(\n          \"Attempting to modify constant value %s\" % name)\n\n    writeback_data = self.writeback_data\n\n    # Check if the new value conforms with the type_info.\n    if value is not None:\n      if isinstance(value, Text):\n        value = self.EscapeString(value)\n\n    writeback_data[name] = value\n    self.FlushCache()", "response": "Updates the value of a configuration option."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef Write(self):\n    if self.writeback:\n      self.writeback.SaveData(self.writeback_data)\n    else:\n      raise RuntimeError(\"Attempting to write a configuration without a \"\n                         \"writeback location.\")", "response": "Write out the updated configuration to the fd."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef WriteToFD(self, fd):\n    if self.writeback:\n      self.writeback.SaveDataToFD(self.writeback_data, fd)\n    else:\n      raise RuntimeError(\"Attempting to write a configuration without a \"\n                         \"writeback location.\")", "response": "Write out the updated configuration to the fd."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef Persist(self, config_option):\n    if not self.writeback:\n      raise RuntimeError(\"Attempting to write a configuration without a \"\n                         \"writeback location.\")\n\n    writeback_raw_value = dict(self.writeback.RawData()).get(config_option)\n    raw_value = None\n\n    for parser in [self.parser] + self.secondary_config_parsers:\n      if parser == self.writeback:\n        continue\n\n      config_raw_data = dict(parser.RawData())\n      raw_value = config_raw_data.get(config_option)\n      if raw_value is None:\n        continue\n      break\n\n    if writeback_raw_value == raw_value:\n      return\n\n    if raw_value is None:\n      return\n\n    self.SetRaw(config_option, raw_value)\n    self.Write()", "response": "Stores <config_option > in the writeback."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nadd an option to the configuration system.", "response": "def AddOption(self, descriptor, constant=False):\n    \"\"\"Registers an option with the configuration system.\n\n    Args:\n      descriptor: A TypeInfoObject instance describing the option.\n      constant: If this is set, the option is treated as a constant - it can be\n        read at any time (before parsing the configuration) and it's an error to\n        try to override it in a config file.\n\n    Raises:\n      RuntimeError: The descriptor's name must contain a . to denote the section\n         name, otherwise we raise.\n      AlreadyInitializedError: If the config has already been read it's too late\n         to define new options.\n    \"\"\"\n    if self.initialized:\n      raise AlreadyInitializedError(\n          \"Config was already initialized when defining %s\" % descriptor.name)\n\n    descriptor.section = descriptor.name.split(\".\")[0]\n    if descriptor.name in self.type_infos:\n      logging.warning(\"Config Option %s multiply defined!\", descriptor.name)\n\n    self.type_infos.Append(descriptor)\n    if constant:\n      self.constants.add(descriptor.name)\n\n    # Register this option's default value.\n    self.defaults[descriptor.name] = descriptor.GetDefault()\n    self.FlushCache()"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef MergeData(self, merge_data, raw_data=None):\n    self.FlushCache()\n    if raw_data is None:\n      raw_data = self.raw_data\n\n    for k, v in iteritems(merge_data):\n      # A context clause.\n      if isinstance(v, dict) and k not in self.type_infos:\n        if k not in self.valid_contexts:\n          raise InvalidContextError(\"Invalid context specified: %s\" % k)\n        context_data = raw_data.setdefault(k, collections.OrderedDict())\n        self.MergeData(v, context_data)\n\n      else:\n        # Find the descriptor for this field.\n        descriptor = self.type_infos.get(k)\n        if descriptor is None:\n          msg = (\"Missing config definition for %s. This option is likely \"\n                 \"deprecated or renamed. Check the release notes.\" % k)\n          if flags.FLAGS.disallow_missing_config_definitions:\n            raise MissingConfigDefinitionError(msg)\n\n        if isinstance(v, string_types):\n          v = v.strip()\n\n        # If we are already initialized and someone tries to modify a constant\n        # value (e.g. via Set()), break loudly.\n        if self.initialized and k in self.constants:\n          raise ConstModificationError(\n              \"Attempting to modify constant value %s\" % k)\n\n        raw_data[k] = v", "response": "Merges data read from a config file into the current config."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef GetParserFromFilename(self, path):\n    # Find the configuration parser.\n    handler_name = path.split(\"://\")[0]\n    for parser_cls in itervalues(GRRConfigParser.classes):\n      if parser_cls.name == handler_name:\n        return parser_cls\n\n    # Handle the filename.\n    extension = os.path.splitext(path)[1]\n    if extension in [\".yaml\", \".yml\"]:\n      return YamlParser\n\n    return ConfigFileParser", "response": "Returns the appropriate parser class from the filename."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef LoadSecondaryConfig(self, filename=None, parser=None):\n    if filename:\n      # Maintain a stack of config file locations in loaded order.\n      self.files.append(filename)\n\n      parser_cls = self.GetParserFromFilename(filename)\n      parser = parser_cls(filename=filename)\n      logging.debug(\"Loading configuration from %s\", filename)\n      self.secondary_config_parsers.append(parser)\n    elif parser is None:\n      raise ValueError(\"Must provide either a filename or a parser.\")\n\n    clone = self.MakeNewConfig()\n    clone.MergeData(parser.RawData())\n    clone.initialized = True\n\n    for file_to_load in clone[\"Config.includes\"]:\n      # We can not include a relative file from a config which does not have\n      # path.\n      if not os.path.isabs(file_to_load):\n        if not filename:\n          raise ConfigFileNotFound(\n              \"While loading %s: Unable to include a relative path (%s) \"\n              \"from a config without a filename\" % (filename, file_to_load))\n\n        # If the included path is relative, we take it as relative to the\n        # current path of the config.\n        file_to_load = os.path.join(os.path.dirname(filename), file_to_load)\n\n      clone_parser = clone.LoadSecondaryConfig(file_to_load)\n      # If an include file is specified but it was not found, raise an error.\n      if not clone_parser.parsed:\n        raise ConfigFileNotFound(\n            \"Unable to load include file %s\" % file_to_load)\n\n    self.MergeData(clone.raw_data)\n    self.files.extend(clone.files)\n\n    return parser", "response": "Loads an additional configuration file from a file."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef Initialize(self,\n                 filename=None,\n                 data=None,\n                 fd=None,\n                 reset=True,\n                 must_exist=False,\n                 parser=ConfigFileParser):\n    \"\"\"Initializes the config manager.\n\n    This method is used to add more config options to the manager. The config\n    can be given as one of the parameters as described in the Args section.\n\n    Args:\n      filename: The name of the configuration file to use.\n      data: The configuration given directly as a long string of data.\n      fd: A file descriptor of a configuration file.\n      reset: If true, the previous configuration will be erased.\n      must_exist: If true the data source must exist and be a valid\n        configuration file, or we raise an exception.\n      parser: The parser class to use (i.e. the format of the file). If not\n        specified guess from the filename.\n\n    Raises:\n      RuntimeError: No configuration was passed in any of the parameters.\n\n      ConfigFormatError: Raised when the configuration file is invalid or does\n        not exist..\n    \"\"\"\n    self.FlushCache()\n    if reset:\n      # Clear previous configuration.\n      self.raw_data = collections.OrderedDict()\n      self.writeback_data = collections.OrderedDict()\n      self.writeback = None\n      self.initialized = False\n\n    if fd is not None:\n      self.parser = self.LoadSecondaryConfig(parser=parser(fd=fd))\n\n    elif filename is not None:\n      self.parser = self.LoadSecondaryConfig(filename)\n      if must_exist and not self.parser.parsed:\n        raise ConfigFormatError(\"Unable to parse config file %s\" % filename)\n\n    elif data is not None:\n      self.parser = self.LoadSecondaryConfig(parser=parser(data=data))\n\n    elif must_exist:\n      raise RuntimeError(\"Registry path not provided.\")\n\n    self.initialized = True", "response": "Initializes the config manager."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nget the raw value without interpolations.", "response": "def GetRaw(self, name, context=None, default=utils.NotAValue):\n    \"\"\"Get the raw value without interpolations.\"\"\"\n    if context is None:\n      context = self.context\n\n    # Getting a raw value is pretty cheap so we wont bother with the cache here.\n    _, value = self._GetValue(name, context, default=default)\n    return value"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nretrieving the value of the named parameter from the section.", "response": "def Get(self, name, default=utils.NotAValue, context=None):\n    \"\"\"Get the value contained  by the named parameter.\n\n    This method applies interpolation/escaping of the named parameter and\n    retrieves the interpolated value.\n\n    Args:\n      name: The name of the parameter to retrieve. This should be in the format\n        of \"Section.name\"\n      default: If retrieving the value results in an error, return this default.\n      context: A list of context strings to resolve the configuration. This is a\n        set of roles the caller is current executing with. For example (client,\n        windows). If not specified we take the context from the current thread's\n        TLS stack.\n\n    Returns:\n      The value of the parameter.\n    Raises:\n      ConfigFormatError: if verify=True and the config doesn't validate.\n      RuntimeError: if a value is retrieved before the config is initialized.\n      ValueError: if a bad context is passed.\n    \"\"\"\n    if not self.initialized:\n      if name not in self.constants:\n        raise RuntimeError(\"Error while retrieving %s: \"\n                           \"Configuration hasn't been initialized yet.\" % name)\n    if context:\n      # Make sure it's not just a string and is iterable.\n      if (isinstance(context, string_types) or\n          not isinstance(context, collections.Iterable)):\n        raise ValueError(\"context should be a list, got %r\" % context)\n\n    calc_context = context\n\n    # Only use the cache if possible.\n    cache_key = (name, tuple(context or ()))\n    if default is utils.NotAValue and cache_key in self.cache:\n      return self.cache[cache_key]\n\n    # Use a default global context if context is not provided.\n    if context is None:\n      calc_context = self.context\n\n    type_info_obj = self.FindTypeInfo(name)\n    _, return_value = self._GetValue(\n        name, context=calc_context, default=default)\n\n    # If we returned the specified default, we just return it here.\n    if return_value is default:\n      return default\n\n    try:\n      return_value = self.InterpolateValue(\n          return_value,\n          default_section=name.split(\".\")[0],\n          type_info_obj=type_info_obj,\n          context=calc_context)\n    except (lexer.ParseError, ValueError) as e:\n      # We failed to parse the value, but a default was specified, so we just\n      # return that.\n      if default is not utils.NotAValue:\n        return default\n\n      raise ConfigFormatError(\"While parsing %s: %s\" % (name, e))\n\n    try:\n      new_value = type_info_obj.Validate(return_value)\n      if new_value is not None:\n        # Update the stored value with the valid data.\n        return_value = new_value\n    except ValueError:\n      if default is not utils.NotAValue:\n        return default\n\n      raise\n\n    # Cache the value for next time.\n    if default is utils.NotAValue:\n      self.cache[cache_key] = return_value\n\n    return return_value"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nresolves the given context and returns the config options active under this context.", "response": "def _ResolveContext(self, context, name, raw_data, path=None):\n    \"\"\"Returns the config options active under this context.\"\"\"\n    if path is None:\n      path = []\n\n    for element in context:\n      if element not in self.valid_contexts:\n        raise InvalidContextError(\"Invalid context specified: %s\" % element)\n\n      if element in raw_data:\n        context_raw_data = raw_data[element]\n\n        value = context_raw_data.get(name)\n        if value is not None:\n          if isinstance(value, string_types):\n            value = value.strip()\n\n          yield context_raw_data, value, path + [element]\n\n        # Recurse into the new context configuration.\n        for context_raw_data, value, new_path in self._ResolveContext(\n            context, name, context_raw_data, path=path + [element]):\n          yield context_raw_data, value, new_path"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nsearch for the value based on the context.", "response": "def _GetValue(self, name, context, default=utils.NotAValue):\n    \"\"\"Search for the value based on the context.\"\"\"\n    container = self.defaults\n\n    # The caller provided a default value.\n    if default is not utils.NotAValue:\n      value = default\n\n    # Take the default from the definition.\n    elif name in self.defaults:\n      value = self.defaults[name]\n\n    else:\n      raise UnknownOption(\"Option %s not defined.\" % name)\n\n    # We resolve the required key with the default raw data, and then iterate\n    # over all elements in the context to see if there are overriding context\n    # configurations.\n    new_value = self.raw_data.get(name)\n    if new_value is not None:\n      value = new_value\n      container = self.raw_data\n\n    # Now check for any contexts. We enumerate all the possible resolutions and\n    # sort by their path length. The last one will be the match with the deepest\n    # path (i.e .the most specific match).\n    matches = list(self._ResolveContext(context, name, self.raw_data))\n\n    if matches:\n      # Sort by the length of the path - longest match path will be at the end.\n      matches.sort(key=lambda x: len(x[2]))\n      value = matches[-1][1]\n      container = matches[-1][0]\n\n      if (len(matches) >= 2 and len(matches[-1][2]) == len(matches[-2][2]) and\n          matches[-1][2] != matches[-2][2] and\n          matches[-1][1] != matches[-2][1]):\n        # This warning specifies that there is an ambiguous match, the config\n        # attempts to find the most specific value e.g. if you have a value\n        # for X.y in context A,B,C, and a value for X.y in D,B it should choose\n        # the one in A,B,C. This warning is for if you have a value in context\n        # A,B and one in A,C. The config doesn't know which one to pick so picks\n        # one and displays this warning.\n        logging.warning(\n            \"Ambiguous configuration for key %s: \"\n            \"Contexts of equal length: %s (%s) and %s (%s)\", name,\n            matches[-1][2], matches[-1][1], matches[-2][2], matches[-2][1])\n\n    # If there is a writeback location this overrides any previous\n    # values.\n    if self.writeback_data:\n      new_value = self.writeback_data.get(name)\n      if new_value is not None:\n        value = new_value\n        container = self.writeback_data\n\n    # Allow the global override to force an option value.\n    if name in self.global_override:\n      return self.global_override, self.global_override[name]\n\n    return container, value"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef FindTypeInfo(self, name):\n    result = self.type_infos.get(name)\n    if result is None:\n      # Not found, assume string.\n      result = type_info.String(name=name, default=\"\")\n\n    return result", "response": "Search for a type_info instance which describes this key."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef InterpolateValue(self,\n                       value,\n                       type_info_obj=type_info.String(),\n                       default_section=None,\n                       context=None):\n    \"\"\"Interpolate the value and parse it with the appropriate type.\"\"\"\n    # It is only possible to interpolate strings...\n    if isinstance(value, Text):\n      try:\n        value = StringInterpolator(\n            value,\n            self,\n            default_section=default_section,\n            parameter=type_info_obj.name,\n            context=context).Parse()\n      except InterpolationError as e:\n        # TODO(hanuszczak): This is a quick hack to not refactor too much while\n        # working on Python 3 compatibility. But this is bad and exceptions\n        # should not be used like this.\n        message = \"{cause}: {detail}\".format(cause=e, detail=value)\n        raise type(e)(message)\n\n      # Parse the data from the string.\n      value = type_info_obj.FromString(value)\n\n    # ... and lists of strings.\n    if isinstance(value, list):\n      value = [\n          self.InterpolateValue(\n              v, default_section=default_section, context=context)\n          for v in value\n      ]\n\n    return value", "response": "Interpolate the value and parse it with the appropriate type."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn True if target_platforms matches the supplied parameters.", "response": "def MatchBuildContext(self,\n                        target_os,\n                        target_arch,\n                        target_package,\n                        context=None):\n    \"\"\"Return true if target_platforms matches the supplied parameters.\n\n    Used by buildanddeploy to determine what clients need to be built.\n\n    Args:\n      target_os: which os we are building for in this run (linux, windows,\n        darwin)\n      target_arch: which arch we are building for in this run (i386, amd64)\n      target_package: which package type we are building (exe, dmg, deb, rpm)\n      context: config_lib context\n\n    Returns:\n      bool: True if target_platforms spec matches parameters.\n    \"\"\"\n    for spec in self.Get(\"ClientBuilder.target_platforms\", context=context):\n      spec_os, arch, package_name = spec.split(\"_\")\n      if (spec_os == target_os and arch == target_arch and\n          package_name == target_package):\n        return True\n    return False"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef DEFINE_bool(self, name, default, help, constant=False):\n    self.AddOption(\n        type_info.Bool(name=name, default=default, description=help),\n        constant=constant)", "response": "A helper for defining boolean options."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef DEFINE_float(self, name, default, help, constant=False):\n    self.AddOption(\n        type_info.Float(name=name, default=default, description=help),\n        constant=constant)", "response": "A helper for defining float options."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef DEFINE_string(self, name, default, help, constant=False):\n    self.AddOption(\n        type_info.String(name=name, default=default or \"\", description=help),\n        constant=constant)", "response": "A helper for defining string options."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ndefining a new multichoice option.", "response": "def DEFINE_multichoice(self, name, default, choices, help, constant=False):\n    \"\"\"Choose multiple options from a list.\"\"\"\n    self.AddOption(\n        type_info.MultiChoice(\n            name=name, default=default, choices=choices, description=help),\n        constant=constant)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef DEFINE_integer_list(self, name, default, help, constant=False):\n    self.AddOption(\n        type_info.List(\n            name=name,\n            default=default,\n            description=help,\n            validator=type_info.Integer()),\n        constant=constant)", "response": "A helper for defining lists of integer options."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef DEFINE_constant_string(self, name, default, help):\n    self.AddOption(\n        type_info.String(name=name, default=default or \"\", description=help),\n        constant=True)", "response": "A helper for defining constant strings."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef RawData(self):\n    result = collections.OrderedDict()\n\n    i = 0\n    while True:\n      try:\n        name, value, value_type = winreg.EnumValue(self._AccessRootKey(), i)\n        # Only support strings here.\n        if value_type == winreg.REG_SZ:\n          precondition.AssertType(value, Text)\n          result[name] = value\n      except OSError:\n        break\n\n      i += 1\n\n    return result", "response": "Yields the valus in each section."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreading audit entries from the database.", "response": "def ReadAPIAuditEntries(self,\n                          username=None,\n                          router_method_names=None,\n                          min_timestamp=None,\n                          max_timestamp=None,\n                          cursor=None):\n    \"\"\"Returns audit entries stored in the database.\"\"\"\n\n    query = \"\"\"SELECT details, timestamp\n        FROM api_audit_entry\n        FORCE INDEX (api_audit_entry_by_username_timestamp)\n        {WHERE_PLACEHOLDER}\n        ORDER BY timestamp ASC\n    \"\"\"\n\n    conditions = []\n    values = []\n    where = \"\"\n\n    if username is not None:\n      conditions.append(\"username = %s\")\n      values.append(username)\n\n    if router_method_names:\n      placeholders = [\"%s\"] * len(router_method_names)\n      placeholders = \", \".join(placeholders)\n      conditions.append(\"router_method_name IN (%s)\" % placeholders)\n      values.extend(router_method_names)\n\n    if min_timestamp is not None:\n      conditions.append(\"timestamp >= FROM_UNIXTIME(%s)\")\n      values.append(mysql_utils.RDFDatetimeToTimestamp(min_timestamp))\n\n    if max_timestamp is not None:\n      conditions.append(\"timestamp <= FROM_UNIXTIME(%s)\")\n      values.append(mysql_utils.RDFDatetimeToTimestamp(max_timestamp))\n\n    if conditions:\n      where = \"WHERE \" + \" AND \".join(conditions)\n\n    query = query.replace(\"{WHERE_PLACEHOLDER}\", where)\n    cursor.execute(query, values)\n\n    return [\n        _AuditEntryFromRow(details, timestamp)\n        for details, timestamp in cursor.fetchall()\n    ]"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ncounts the number of API audit entries for a user and calendar day.", "response": "def CountAPIAuditEntriesByUserAndDay(self,\n                                       min_timestamp=None,\n                                       max_timestamp=None,\n                                       cursor=None):\n    \"\"\"Returns audit entry counts grouped by user and calendar day.\"\"\"\n    query = \"\"\"\n        -- Timestamps are timezone-agnostic whereas dates are not. Hence, we are\n        -- forced to pick some timezone, in order to extract the day. We choose\n        -- UTC ('+00:00') as it is the most sane default.\n        SELECT username,\n               CAST(CONVERT_TZ(timestamp, @@SESSION.time_zone, '+00:00')\n                    AS DATE) AS day,\n               COUNT(*)\n        FROM api_audit_entry\n        FORCE INDEX (api_audit_entry_by_username_timestamp)\n        {WHERE_PLACEHOLDER}\n        GROUP BY username, day\n    \"\"\"\n    conditions = []\n    values = []\n    where = \"\"\n\n    if min_timestamp is not None:\n      conditions.append(\"timestamp >= FROM_UNIXTIME(%s)\")\n      values.append(mysql_utils.RDFDatetimeToTimestamp(min_timestamp))\n\n    if max_timestamp is not None:\n      conditions.append(\"timestamp <= FROM_UNIXTIME(%s)\")\n      values.append(mysql_utils.RDFDatetimeToTimestamp(max_timestamp))\n\n    if conditions:\n      where = \"WHERE \" + \" AND \".join(conditions)\n\n    query = query.replace(\"{WHERE_PLACEHOLDER}\", where)\n    cursor.execute(query, values)\n\n    return {(username, rdfvalue.RDFDatetime.FromDate(day)): count\n            for (username, day, count) in cursor.fetchall()}"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nwrites an API audit entry to the database.", "response": "def WriteAPIAuditEntry(self, entry, cursor=None):\n    \"\"\"Writes an audit entry to the database.\"\"\"\n    args = {\n        \"username\":\n            entry.username,\n        \"router_method_name\":\n            entry.router_method_name,\n        \"details\":\n            entry.SerializeToString(),\n        \"timestamp\":\n            mysql_utils.RDFDatetimeToTimestamp(rdfvalue.RDFDatetime.Now()),\n    }\n    query = \"\"\"\n    INSERT INTO api_audit_entry (username, router_method_name, details,\n        timestamp)\n    VALUES (%(username)s, %(router_method_name)s, %(details)s,\n        FROM_UNIXTIME(%(timestamp)s))\n    \"\"\"\n    cursor.execute(query, args)"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ngets the last active breakdown evolved over time.", "response": "def GetReportData(self, get_report_args, token):\n    \"\"\"Show how the last active breakdown evolved over time.\"\"\"\n    report = rdf_report_plugins.ApiReportData(\n        representation_type=rdf_report_plugins.ApiReportData.RepresentationType\n        .LINE_CHART)\n\n    series_with_timestamps = client_report_utils.FetchAllGraphSeries(\n        get_report_args.client_label,\n        rdf_stats.ClientGraphSeries.ReportType.N_DAY_ACTIVE,\n        period=rdfvalue.Duration(\"180d\"))\n\n    categories = {}\n    for timestamp, graph_series in sorted(iteritems(series_with_timestamps)):\n      self._ProcessGraphSeries(graph_series, timestamp, categories)\n\n    graphs = []\n    for k, v in iteritems(categories):\n      graph = dict(label=k, data=v)\n      graphs.append(graph)\n\n    report.line_chart.data = sorted(\n        (rdf_report_plugins.ApiReportDataSeries2D(\n            label=label,\n            points=(rdf_report_plugins.ApiReportDataPoint2D(x=x, y=y)\n                    for x, y in points))\n         for label, points in iteritems(categories)),\n        key=lambda series: int(series.label.split()[0]),\n        reverse=True)\n\n    return report"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nextracting only the operating system type from the active histogram.", "response": "def GetReportData(self, get_report_args, token):\n    \"\"\"Extract only the operating system type from the active histogram.\"\"\"\n    report = rdf_report_plugins.ApiReportData(\n        representation_type=rdf_report_plugins.ApiReportData.RepresentationType\n        .PIE_CHART)\n\n    graph_series = client_report_utils.FetchMostRecentGraphSeries(\n        get_report_args.client_label,\n        rdf_stats.ClientGraphSeries.ReportType.OS_TYPE,\n        token=token)\n    if graph_series is not None:\n      for graph in graph_series.graphs:\n        # Find the correct graph and merge the OS categories together\n        if \"%s day\" % self.__class__.ACTIVE_DAYS in graph.title:\n          for sample in graph:\n            report.pie_chart.data.Append(\n                rdf_report_plugins.ApiReportDataPoint1D(\n                    label=sample.label, x=sample.y_value))\n          break\n\n    report.pie_chart.data = sorted(\n        report.pie_chart.data, key=lambda point: point.label)\n\n    return report"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef StartService(service_name):\n  try:\n    win32serviceutil.StartService(service_name)\n    logging.info(\"Service '%s' started.\", service_name)\n  except pywintypes.error as e:\n    if getattr(e, \"winerror\", None) == winerror.ERROR_SERVICE_DOES_NOT_EXIST:\n      logging.debug(\"Tried to start '%s', but the service is not installed.\",\n                    service_name)\n    else:\n      logging.exception(\"Encountered error trying to start '%s':\", service_name)", "response": "Starts a Windows service with the given name."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nstop a Windows service with the given name.", "response": "def StopService(service_name, service_binary_name=None):\n  \"\"\"Stop a Windows service with the given name.\n\n  Args:\n    service_name: string The name of the service to be stopped.\n    service_binary_name: string If given, also kill this binary as a best effort\n        fallback solution.\n  \"\"\"\n  # QueryServiceStatus returns: scvType, svcState, svcControls, err,\n  # svcErr, svcCP, svcWH\n  try:\n    status = win32serviceutil.QueryServiceStatus(service_name)[1]\n  except pywintypes.error as e:\n    if getattr(e, \"winerror\", None) == winerror.ERROR_SERVICE_DOES_NOT_EXIST:\n      logging.debug(\"Tried to stop '%s', but the service is not installed.\",\n                    service_name)\n    else:\n      logging.exception(\"Unable to query status of service '%s':\", service_name)\n    return\n\n  for _ in range(20):\n    if status == win32service.SERVICE_STOPPED:\n      break\n    elif status != win32service.SERVICE_STOP_PENDING:\n      try:\n        win32serviceutil.StopService(service_name)\n      except pywintypes.error:\n        logging.exception(\"Unable to stop service '%s':\", service_name)\n    time.sleep(1)\n    status = win32serviceutil.QueryServiceStatus(service_name)[1]\n\n  if status == win32service.SERVICE_STOPPED:\n    logging.info(\"Service '%s' stopped.\", service_name)\n    return\n  elif not service_binary_name:\n    return\n\n  # Taskkill will fail on systems predating Windows XP, this is a best\n  # effort fallback solution.\n  output = subprocess.check_output(\n      [\"taskkill\", \"/im\", \"%s*\" % service_binary_name, \"/f\"],\n      shell=True,\n      stdin=subprocess.PIPE,\n      stderr=subprocess.PIPE)\n\n  logging.debug(\"%s\", output)\n\n  # Sleep a bit to ensure that process really quits.\n  time.sleep(2)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef StopPreviousService(self):\n    StopService(\n        service_name=config.CONFIG[\"Nanny.service_name\"],\n        service_binary_name=config.CONFIG[\"Nanny.service_binary_name\"])\n\n    if not config.CONFIG[\"Client.fleetspeak_enabled\"]:\n      return\n\n    StopService(service_name=config.CONFIG[\"Client.fleetspeak_service_name\"])\n\n    # Delete GRR's Fleetspeak config from the registry so Fleetspeak\n    # doesn't try to restart GRR unless/until installation completes\n    # successfully.\n    key_path = config.CONFIG[\"Client.fleetspeak_unsigned_services_regkey\"]\n    regkey = OpenRegkey(key_path)\n    try:\n      winreg.DeleteValue(regkey, config.CONFIG[\"Client.name\"])\n      logging.info(\"Deleted value '%s' of key '%s'.\",\n                   config.CONFIG[\"Client.name\"], key_path)\n    except OSError as e:\n      # Windows will raise a no-such-file-or-directory error if\n      # GRR's config hasn't been written to the registry yet.\n      if e.errno != errno.ENOENT:\n        raise", "response": "Stops the previous GRR process."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef RunOnce(self):\n    self.StopPreviousService()\n\n    executable_directory = os.path.dirname(sys.executable)\n    install_path = config.CONFIG[\"Client.install_path\"]\n    logging.info(\"Installing binaries %s -> %s\", executable_directory,\n                 config.CONFIG[\"Client.install_path\"])\n    if os.path.exists(install_path):\n      attempts = 0\n      while True:\n        try:\n          shutil.rmtree(install_path)\n          break\n        except OSError as e:\n          attempts += 1\n          if e.errno == errno.EACCES and attempts < 10:\n            # The currently installed GRR process may stick around for a few\n            # seconds after the service is terminated (keeping the contents of\n            # the installation directory locked).\n            logging.warning(\n                \"Encountered permission-denied error while trying to empty out \"\n                \"'%s'. Retrying...\", install_path)\n            time.sleep(3)\n          else:\n            raise e\n    os.makedirs(install_path)\n\n    # Recursively copy the temp directory to the installation directory.\n    for root, dirs, files in os.walk(executable_directory):\n      for name in dirs:\n        src_path = os.path.join(root, name)\n        relative_path = os.path.relpath(src_path, executable_directory)\n        dest_path = os.path.join(install_path, relative_path)\n\n        try:\n          os.mkdir(dest_path)\n        except OSError as e:\n          # Ignore already-exists exceptions.\n          if e.errno != errno.EEXIST:\n            raise\n\n      for name in files:\n        src_path = os.path.join(root, name)\n        relative_path = os.path.relpath(src_path, executable_directory)\n        dest_path = os.path.join(install_path, relative_path)\n\n        shutil.copy(src_path, dest_path)", "response": "This method is called by the main thread. It will recursively copy the binaries from the temporary unpack location to the installation directory."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef InstallNanny(self):\n    # We need to copy the nanny sections to the registry to ensure the\n    # service is correctly configured.\n    new_config = config.CONFIG.MakeNewConfig()\n    new_config.SetWriteBack(config.CONFIG[\"Config.writeback\"])\n\n    for option in self.nanny_options:\n      new_config.Set(option, config.CONFIG.Get(option))\n\n    new_config.Write()\n\n    args = [\n        config.CONFIG[\"Nanny.binary\"], \"--service_key\",\n        config.CONFIG[\"Client.config_key\"], \"install\"\n    ]\n\n    logging.debug(\"Calling %s\", (args,))\n    output = subprocess.check_output(\n        args, shell=True, stdin=subprocess.PIPE, stderr=subprocess.PIPE)\n    logging.debug(\"%s\", output)", "response": "Install the nanny program."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nbuild a legacy AFF4 urn string for a given subject and approval type.", "response": "def BuildLegacySubject(subject_id, approval_type):\n  \"\"\"Builds a legacy AFF4 urn string for a given subject and approval type.\"\"\"\n  at = rdf_objects.ApprovalRequest.ApprovalType\n\n  if approval_type == at.APPROVAL_TYPE_CLIENT:\n    return \"aff4:/%s\" % subject_id\n  elif approval_type == at.APPROVAL_TYPE_HUNT:\n    return \"aff4:/hunts/%s\" % subject_id\n  elif approval_type == at.APPROVAL_TYPE_CRON_JOB:\n    return \"aff4:/cron/%s\" % subject_id\n\n  raise ValueError(\"Invalid approval type.\")"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef CheckClientApprovalRequest(approval_request):\n\n  _CheckExpired(approval_request)\n  _CheckHasEnoughGrants(approval_request)\n\n  if not client_approval_auth.CLIENT_APPROVAL_AUTH_MGR.IsActive():\n    return True\n\n  token = access_control.ACLToken(username=approval_request.requestor_username)\n  approvers = set(g.grantor_username for g in approval_request.grants)\n\n  labels = sorted(\n      data_store.REL_DB.ReadClientLabels(approval_request.subject_id),\n      key=lambda l: l.name)\n  for label in labels:\n    client_approval_auth.CLIENT_APPROVAL_AUTH_MGR.CheckApproversForLabel(\n        token, rdfvalue.RDFURN(approval_request.subject_id),\n        approval_request.requestor_username, approvers, label.name)\n\n  return True", "response": "Checks if a client approval request is granted."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef CheckApprovalRequest(approval_request):\n\n  at = rdf_objects.ApprovalRequest.ApprovalType\n\n  if approval_request.approval_type == at.APPROVAL_TYPE_CLIENT:\n    return CheckClientApprovalRequest(approval_request)\n  elif approval_request.approval_type == at.APPROVAL_TYPE_HUNT:\n    return CheckHuntApprovalRequest(approval_request)\n  elif approval_request.approval_type == at.APPROVAL_TYPE_CRON_JOB:\n    return CheckCronJobApprovalRequest(approval_request)\n  else:\n    raise ValueError(\n        \"Invalid approval type: %s\" % approval_request.approval_type)", "response": "Checks if an approval request is granted."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nparse YAML data into a list of APIAuthorization objects.", "response": "def ParseYAMLAuthorizationsList(yaml_data):\n    \"\"\"Parses YAML data into a list of APIAuthorization objects.\"\"\"\n    try:\n      raw_list = yaml.ParseMany(yaml_data)\n    except (ValueError, pyyaml.YAMLError) as e:\n      raise InvalidAPIAuthorization(\"Invalid YAML: %s\" % e)\n\n    result = []\n    for auth_src in raw_list:\n      auth = APIAuthorization()\n      auth.router_cls = _GetRouterClass(auth_src[\"router\"])\n      auth.users = auth_src.get(\"users\", [])\n      auth.groups = auth_src.get(\"groups\", [])\n      auth.router_params = auth_src.get(\"router_params\", {})\n\n      result.append(auth)\n\n    return result"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncreates a router with a given name and params.", "response": "def _CreateRouter(self, router_cls, params=None):\n    \"\"\"Creates a router with a given name and params.\"\"\"\n    if not router_cls.params_type and params:\n      raise ApiCallRouterDoesNotExpectParameters(\n          \"%s is not configurable\" % router_cls)\n\n    rdf_params = None\n    if router_cls.params_type:\n      rdf_params = router_cls.params_type()\n      if params:\n        rdf_params.FromDict(params)\n\n    return router_cls(params=rdf_params)"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns a router corresponding to a given username.", "response": "def GetRouterForUser(self, username):\n    \"\"\"Returns a router corresponding to a given username.\"\"\"\n\n    for index, router in enumerate(self.routers):\n      router_id = str(index)\n\n      if self.auth_manager.CheckPermissions(username, router_id):\n        logging.debug(\"Matched router %s to user %s\", router.__class__.__name__,\n                      username)\n        return router\n\n    logging.debug(\"No router ACL rule match for user %s. Using default \"\n                  \"router %s\", username, self.default_router.__class__.__name__)\n    return self.default_router"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef RetryUpload(self, job, job_id, error):\n    if self.IsErrorRetryable(error):\n      retry_count = 0\n      sleep_interval = config.CONFIG[\"BigQuery.retry_interval\"]\n      while retry_count < config.CONFIG[\"BigQuery.retry_max_attempts\"]:\n\n        time.sleep(sleep_interval.seconds)\n        logging.info(\"Retrying job_id: %s\", job_id)\n        retry_count += 1\n\n        try:\n          response = job.execute()\n          return response\n        except errors.HttpError as e:\n          if self.IsErrorRetryable(e):\n            sleep_interval *= config.CONFIG[\"BigQuery.retry_multiplier\"]\n            logging.exception(\"Error with job: %s, will retry in %s\", job_id,\n                              sleep_interval)\n          else:\n            raise BigQueryJobUploadError(\n                \"Can't retry error code %s. Giving up\"\n                \" on job: %s.\" % (e.resp.status, job_id))\n    else:\n      raise BigQueryJobUploadError(\"Can't retry error code %s. Giving up on \"\n                                   \"job: %s.\" % (error.resp.status, job_id))\n\n    raise BigQueryJobUploadError(\n        \"Giving up on job:%s after %s retries.\" % (job_id, retry_count))", "response": "Retry the BigQuery upload job."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef InsertData(self, table_id, fd, schema, job_id):\n    configuration = {\n        \"schema\": {\n            \"fields\": schema\n        },\n        \"destinationTable\": {\n            \"projectId\": self.project_id,\n            \"tableId\": table_id,\n            \"datasetId\": self.dataset_id\n        },\n        \"sourceFormat\": \"NEWLINE_DELIMITED_JSON\",\n    }\n\n    body = {\n        \"configuration\": {\n            \"load\": configuration\n        },\n        \"jobReference\": {\n            \"projectId\": self.project_id,\n            \"jobId\": job_id\n        }\n    }\n\n    # File content can be gzipped for bandwidth efficiency. The server handles\n    # it correctly without any changes to the request.\n    mediafile = http.MediaFileUpload(\n        fd.name, mimetype=\"application/octet-stream\")\n    job = self.service.jobs().insert(\n        projectId=self.project_id, body=body, media_body=mediafile)\n    try:\n      response = job.execute()\n      return response\n    except errors.HttpError as e:\n      if self.GetDataset(self.dataset_id):\n        logging.exception(\"Error with job: %s\", job_id)\n      else:\n        # If this is our first export ever, we need to create the dataset.\n        logging.info(\"Attempting to create dataset: %s\", self.dataset_id)\n        self.CreateDataset()\n      return self.RetryUpload(job, job_id, e)", "response": "Insert data into a bigquery table."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef CreateFlowArgs(self, flow_name=None):\n    if not self._flow_descriptors:\n      self._flow_descriptors = {}\n\n      result = self._context.SendRequest(\"ListFlowDescriptors\", None)\n      for item in result.items:\n        self._flow_descriptors[item.name] = item\n\n    try:\n      flow_descriptor = self._flow_descriptors[flow_name]\n    except KeyError:\n      raise UnknownFlowName(flow_name)\n\n    return utils.CopyProto(utils.UnpackAny(flow_descriptor.default_args))", "response": "Creates flow arguments object for a given flow with a given name."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nmake a stat response for a given TSK info object.", "response": "def MakeStatResponse(self, tsk_file, tsk_attribute=None, append_name=None):\n    \"\"\"Given a TSK info object make a StatEntry.\n\n    Note that tsk uses two things to uniquely identify a data stream - the inode\n    object given in tsk_file and the attribute object which may correspond to an\n    ADS of this file for filesystems which support ADS. We store both of these\n    in the stat response.\n\n    Args:\n      tsk_file: A TSK File object for the specified inode.\n      tsk_attribute: A TSK Attribute object for the ADS. If None we use the main\n        stream.\n      append_name: If specified we append this name to the last element of the\n        pathspec.\n\n    Returns:\n      A StatEntry which can be used to re-open this exact VFS node.\n    \"\"\"\n    precondition.AssertOptionalType(append_name, Text)\n\n    info = tsk_file.info\n    response = rdf_client_fs.StatEntry()\n    meta = info.meta\n    if meta:\n      response.st_ino = meta.addr\n      for attribute in [\n          \"mode\", \"nlink\", \"uid\", \"gid\", \"size\", \"atime\", \"mtime\", \"ctime\",\n          \"crtime\"\n      ]:\n        try:\n          value = int(getattr(meta, attribute))\n          if value < 0:\n            value &= 0xFFFFFFFF\n\n          setattr(response, \"st_%s\" % attribute, value)\n        except AttributeError:\n          pass\n\n    name = info.name\n    child_pathspec = self.pathspec.Copy()\n\n    if append_name is not None:\n      # Append the name to the most inner pathspec\n      child_pathspec.last.path = utils.JoinPath(child_pathspec.last.path,\n                                                append_name)\n\n    child_pathspec.last.inode = meta.addr\n    if tsk_attribute is not None:\n      child_pathspec.last.ntfs_type = int(tsk_attribute.info.type)\n      child_pathspec.last.ntfs_id = int(tsk_attribute.info.id)\n      child_pathspec.last.stream_name = tsk_attribute.info.name\n\n      # Update the size with the attribute size.\n      response.st_size = tsk_attribute.info.size\n\n      default = rdf_paths.PathSpec.tsk_fs_attr_type.TSK_FS_ATTR_TYPE_DEFAULT\n      last = child_pathspec.last\n      if last.ntfs_type != default or last.ntfs_id:\n        # This is an ads and should be treated as a file.\n        # Clear all file type bits.\n        response.st_mode &= ~self.stat_type_mask\n        response.st_mode |= stat.S_IFREG\n\n    else:\n      child_pathspec.last.ntfs_type = None\n      child_pathspec.last.ntfs_id = None\n      child_pathspec.last.stream_name = None\n\n    if name:\n      # Encode the type onto the st_mode response\n      response.st_mode |= self.FILE_TYPE_LOOKUP.get(int(name.type), 0)\n\n    if meta:\n      # What if the types are different? What to do here?\n      response.st_mode |= self.META_TYPE_LOOKUP.get(int(meta.type), 0)\n\n    # Write the pathspec on the response.\n    response.pathspec = child_pathspec\n    return response"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef Read(self, length):\n    if not self.IsFile():\n      raise IOError(\"%s is not a file.\" % self.pathspec.last.path)\n\n    available = min(self.size - self.offset, length)\n    if available > 0:\n      # This raises a RuntimeError in some situations.\n      try:\n        data = self.fd.read_random(self.offset, available,\n                                   self.pathspec.last.ntfs_type,\n                                   self.pathspec.last.ntfs_id)\n      except RuntimeError as e:\n        raise IOError(e)\n\n      self.offset += len(data)\n\n      return data\n    return b\"\"", "response": "Reads a random number of bytes from the file."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef Stat(self, ext_attrs=None):\n    del ext_attrs  # Unused.\n    return self.MakeStatResponse(self.fd, tsk_attribute=self.tsk_attribute)", "response": "Return a stat of the file."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nlisting all the files in the directory.", "response": "def ListFiles(self, ext_attrs=None):\n    \"\"\"List all the files in the directory.\"\"\"\n    del ext_attrs  # Unused.\n\n    if not self.IsDirectory():\n      raise IOError(\"%s is not a directory\" % self.pathspec.CollapsePath())\n\n    for f in self.fd.as_directory():\n      try:\n        name = _DecodeUTF8WithWarning(f.info.name.name)\n        # Drop these useless entries.\n        if name in [\".\", \"..\"] or name in self.BLACKLIST_FILES:\n          continue\n\n        # First we yield a standard response using the default attributes.\n        yield self.MakeStatResponse(f, tsk_attribute=None, append_name=name)\n\n        # Now send back additional named attributes for the ADS.\n        for attribute in f:\n          if attribute.info.type in [\n              pytsk3.TSK_FS_ATTR_TYPE_NTFS_DATA, pytsk3.TSK_FS_ATTR_TYPE_DEFAULT\n          ]:\n            if attribute.info.name:\n              yield self.MakeStatResponse(\n                  f, append_name=name, tsk_attribute=attribute)\n      except AttributeError:\n        pass"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef main(args):\n\n  if args.subparser_name == \"version\":\n    version = config_server.VERSION[\"packageversion\"]\n    print(\"GRR configuration updater {}\".format(version))\n    return\n\n  token = config_updater_util.GetToken()\n  grr_config.CONFIG.AddContext(contexts.COMMAND_LINE_CONTEXT)\n  grr_config.CONFIG.AddContext(contexts.CONFIG_UPDATER_CONTEXT)\n\n  if args.subparser_name == \"initialize\":\n    config_lib.ParseConfigCommandLine()\n    if args.noprompt:\n      config_updater_util.InitializeNoPrompt(\n          grr_config.CONFIG,\n          external_hostname=args.external_hostname,\n          admin_password=args.admin_password,\n          mysql_hostname=args.mysql_hostname,\n          mysql_port=args.mysql_port,\n          mysql_username=args.mysql_username,\n          mysql_password=args.mysql_password,\n          mysql_db=args.mysql_db,\n          mysql_client_key_path=args.mysql_client_key_path,\n          mysql_client_cert_path=args.mysql_client_cert_path,\n          mysql_ca_cert_path=args.mysql_ca_cert_path,\n          redownload_templates=args.redownload_templates,\n          repack_templates=not args.norepack_templates,\n          token=token)\n    else:\n      config_updater_util.Initialize(\n          grr_config.CONFIG,\n          external_hostname=args.external_hostname,\n          admin_password=args.admin_password,\n          redownload_templates=args.redownload_templates,\n          repack_templates=not args.norepack_templates,\n          token=token)\n    return\n\n  server_startup.Init()\n\n  try:\n    print(\"Using configuration %s\" % grr_config.CONFIG)\n  except AttributeError:\n    raise RuntimeError(\"No valid config specified.\")\n\n  if args.subparser_name == \"generate_keys\":\n    try:\n      config_updater_keys_util.GenerateKeys(\n          grr_config.CONFIG, overwrite_keys=args.overwrite_keys)\n    except RuntimeError as e:\n      # GenerateKeys will raise if keys exist and overwrite_keys is not set.\n      print(\"ERROR: %s\" % e)\n      sys.exit(1)\n    grr_config.CONFIG.Write()\n\n  elif args.subparser_name == \"repack_clients\":\n    upload = not args.noupload\n    repacking.TemplateRepacker().RepackAllTemplates(upload=upload, token=token)\n\n  elif args.subparser_name == \"show_user\":\n    if args.username:\n      print(config_updater_util.GetUserSummary(args.username))\n    else:\n      print(config_updater_util.GetAllUserSummaries())\n\n  elif args.subparser_name == \"update_user\":\n    config_updater_util.UpdateUser(\n        args.username, password=args.password, is_admin=args.admin)\n\n  elif args.subparser_name == \"delete_user\":\n    config_updater_util.DeleteUser(args.username)\n\n  elif args.subparser_name == \"add_user\":\n    config_updater_util.CreateUser(\n        args.username, password=args.password, is_admin=args.admin)\n\n  elif args.subparser_name == \"upload_python\":\n    config_updater_util.UploadSignedBinary(\n        args.file,\n        rdf_objects.SignedBinaryID.BinaryType.PYTHON_HACK,\n        args.platform,\n        upload_subdirectory=args.upload_subdirectory)\n\n  elif args.subparser_name == \"upload_exe\":\n    config_updater_util.UploadSignedBinary(\n        args.file,\n        rdf_objects.SignedBinaryID.BinaryType.EXECUTABLE,\n        args.platform,\n        upload_subdirectory=args.upload_subdirectory)\n\n  elif args.subparser_name == \"set_var\":\n    var = args.var\n    val = args.val\n\n    config = grr_config.CONFIG\n    print(\"Setting %s to %s\" % (var, val))\n    if val.startswith(\"[\"):  # Allow setting of basic lists.\n      val = val[1:-1].split(\",\")\n    config.Set(var, val)\n    config.Write()\n\n  elif args.subparser_name == \"upload_artifact\":\n    with io.open(args.file, \"r\") as filedesc:\n      source = filedesc.read()\n    try:\n      artifact.UploadArtifactYamlFile(source, overwrite=args.overwrite_artifact)\n    except rdf_artifacts.ArtifactDefinitionError as e:\n      print(\"Error %s. You may need to set --overwrite_artifact.\" % e)\n\n  elif args.subparser_name == \"delete_artifacts\":\n    artifact_list = args.artifact\n    if not artifact_list:\n      raise ValueError(\"No artifact to delete given.\")\n    artifact_registry.DeleteArtifactsFromDatastore(artifact_list, token=token)\n    print(\"Artifacts %s deleted.\" % artifact_list)\n\n  elif args.subparser_name == \"rotate_server_key\":\n    print(\"\"\"\nYou are about to rotate the server key. Note that:\n\n  - Clients might experience intermittent connection problems after\n    the server keys rotated.\n\n  - It's not possible to go back to an earlier key. Clients that see a\n    new certificate will remember the cert's serial number and refuse\n    to accept any certificate with a smaller serial number from that\n    point on.\n    \"\"\")\n\n    if input(\"Continue? [yN]: \").upper() == \"Y\":\n      if args.keylength:\n        keylength = int(args.keylength)\n      else:\n        keylength = grr_config.CONFIG[\"Server.rsa_key_length\"]\n\n      maintenance_utils.RotateServerKey(\n          cn=args.common_name, keylength=keylength)", "response": "Main function of the GRR configuration updater."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef ProcessResponse(self, state, response):\n    emails_left = self.args.emails_limit - self.IncrementCounter()\n    if emails_left < 0:\n      return\n\n    if data_store.RelationalDBEnabled():\n      client_id = response.source.Basename()\n      client = data_store.REL_DB.ReadClientSnapshot(client_id)\n      hostname = client.knowledge_base.fqdn or \"unknown hostname\"\n      client_fragment_id = \"/clients/%s\" % client_id\n    else:\n      client_id = response.source\n      client = aff4.FACTORY.Open(client_id, token=self.token)\n      hostname = client.Get(client.Schema.HOSTNAME) or \"unknown hostname\"\n      client_fragment_id = \"/clients/%s\" % client_id.Basename()\n\n    if emails_left == 0:\n      additional_message = (self.too_many_mails_msg % self.args.emails_limit)\n    else:\n      additional_message = \"\"\n\n    subject = self.__class__.subject_template.render(\n        source_urn=utils.SmartUnicode(self.source_urn))\n    body = self.__class__.template.render(\n        client_id=client_id,\n        client_fragment_id=client_fragment_id,\n        admin_ui_url=config.CONFIG[\"AdminUI.url\"],\n        source_urn=self.source_urn,\n        additional_message=additional_message,\n        signature=config.CONFIG[\"Email.signature\"],\n        hostname=utils.SmartUnicode(hostname),\n        creator=utils.SmartUnicode(self.token.username))\n\n    email_alerts.EMAIL_ALERTER.SendEmail(\n        self.args.email_address,\n        \"grr-noreply\",\n        utils.SmartStr(subject),\n        utils.SmartStr(body),\n        is_html=True)", "response": "Sends an email for each response."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nwrite a new artifact to the database.", "response": "def WriteArtifact(self, artifact):\n    \"\"\"Writes new artifact to the database.\"\"\"\n    name = str(artifact.name)\n\n    if name in self.artifacts:\n      raise db.DuplicatedArtifactError(name)\n\n    self.artifacts[name] = artifact.Copy()"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreads an artifact from the database.", "response": "def ReadArtifact(self, name):\n    \"\"\"Looks up an artifact with given name from the database.\"\"\"\n    try:\n      artifact = self.artifacts[name]\n    except KeyError:\n      raise db.UnknownArtifactError(name)\n\n    return artifact.Copy()"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nlist all artifacts that are stored in the database.", "response": "def ReadAllArtifacts(self):\n    \"\"\"Lists all artifacts that are stored in the database.\"\"\"\n    artifacts = []\n\n    for artifact in itervalues(self.artifacts):\n      artifacts.append(artifact.Copy())\n\n    return artifacts"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef DeleteArtifact(self, name):\n    try:\n      del self.artifacts[name]\n    except KeyError:\n      raise db.UnknownArtifactError(name)", "response": "Deletes an artifact with given name from the database."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef StaticAdd(cls, queue_urn, rdf_value, mutation_pool=None):\n    if not isinstance(rdf_value, cls.rdf_type):\n      raise ValueError(\"This collection only accepts values of type %s.\" %\n                       cls.rdf_type.__name__)\n    if mutation_pool is None:\n      raise ValueError(\"Mutation pool can't be none.\")\n\n    timestamp = rdfvalue.RDFDatetime.Now().AsMicrosecondsSinceEpoch()\n\n    if not isinstance(queue_urn, rdfvalue.RDFURN):\n      queue_urn = rdfvalue.RDFURN(queue_urn)\n\n    mutation_pool.QueueAddItem(queue_urn, rdf_value, timestamp)", "response": "Adds an rdf value to a queue."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef Add(self, rdf_value, mutation_pool=None):\n    self.StaticAdd(self.urn, rdf_value, mutation_pool=mutation_pool)", "response": "Adds an rdf value to the queue."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns and claims up to limit unclaimed records for timeout seconds.", "response": "def ClaimRecords(self,\n                   limit=10000,\n                   timeout=\"30m\",\n                   start_time=None,\n                   record_filter=lambda x: False,\n                   max_filtered=1000):\n    \"\"\"Returns and claims up to limit unclaimed records for timeout seconds.\n\n    Returns a list of records which are now \"claimed\", a claimed record will\n    generally be unavailable to be claimed until the claim times out. Note\n    however that in case of an unexpected timeout or other error a record might\n    be claimed twice at the same time. For this reason it should be considered\n    weaker than a true lock.\n\n    Args:\n      limit: The number of records to claim.\n\n      timeout: The duration of the claim.\n\n      start_time: The time to start claiming records at. Only records with a\n        timestamp after this point will be claimed.\n\n      record_filter: A filter method to determine if the record should be\n        returned. It will be called serially on each record and the record will\n        be filtered (not returned or locked) if it returns True.\n\n      max_filtered: If non-zero, limits the number of results read when\n        filtered. Specifically, if max_filtered filtered results are read\n        sequentially without any unfiltered results, we stop looking for\n        results.\n\n    Returns:\n      A list (id, record) where record is a self.rdf_type and id is a record\n      identifier which can be used to delete or release the record.\n\n    Raises:\n      LockError: If the queue is not locked.\n\n    \"\"\"\n    if not self.locked:\n      raise aff4.LockError(\"Queue must be locked to claim records.\")\n\n    with data_store.DB.GetMutationPool() as mutation_pool:\n      return mutation_pool.QueueClaimRecords(\n          self.urn,\n          self.rdf_type,\n          limit=limit,\n          timeout=timeout,\n          start_time=start_time,\n          record_filter=record_filter,\n          max_filtered=max_filtered)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef RefreshClaims(self, ids, timeout=\"30m\"):\n    with data_store.DB.GetMutationPool() as mutation_pool:\n      mutation_pool.QueueRefreshClaims(ids, timeout=timeout)", "response": "Refreshes claims on records identified by ids."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef DeleteRecords(cls, ids, token):\n    with data_store.DB.GetMutationPool() as mutation_pool:\n      mutation_pool.QueueDeleteRecords(ids)", "response": "Deletes the ClaimRecords identified by ids."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef ReleaseRecords(cls, ids, token):\n    with data_store.DB.GetMutationPool() as mutation_pool:\n      mutation_pool.QueueReleaseRecords(ids)", "response": "Releases any claim on the records identified by ids."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nparses the History file.", "response": "def Parse(self, stat, file_object, knowledge_base):\n    \"\"\"Parse the History file.\"\"\"\n    _ = knowledge_base\n    # TODO(user): Convert this to use the far more intelligent plaso parser.\n    chrome = ChromeParser(file_object)\n    for timestamp, entry_type, url, data1, _, _ in chrome.Parse():\n      if entry_type == \"CHROME_DOWNLOAD\":\n        yield rdf_webhistory.BrowserHistoryItem(\n            url=url,\n            domain=urlparse.urlparse(url).netloc,\n            access_time=timestamp,\n            program_name=\"Chrome\",\n            source_path=file_object.Path(),\n            download_path=data1)\n      elif entry_type == \"CHROME_VISIT\":\n        yield rdf_webhistory.BrowserHistoryItem(\n            url=url,\n            domain=urlparse.urlparse(url).netloc,\n            access_time=timestamp,\n            program_name=\"Chrome\",\n            source_path=file_object.Path(),\n            title=data1)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nparsing the history and return a list of attributes.", "response": "def Parse(self):\n    \"\"\"Iterator returning a list for each entry in history.\n\n    We store all the download events in an array (choosing this over visits\n    since there are likely to be less of them). We later interleave them with\n    visit events to get an overall correct time order.\n\n    Yields:\n      a list of attributes for each entry\n    \"\"\"\n    # Query for old style and newstyle downloads storage.\n    query_iter = itertools.chain(\n        self.Query(self.DOWNLOADS_QUERY), self.Query(self.DOWNLOADS_QUERY_2))\n\n    results = []\n    for timestamp, url, path, received_bytes, total_bytes in query_iter:\n      timestamp = self.ConvertTimestamp(timestamp)\n      results.append((timestamp, \"CHROME_DOWNLOAD\", url, path, received_bytes,\n                      total_bytes))\n\n    for timestamp, url, title, typed_count in self.Query(self.VISITS_QUERY):\n      timestamp = self.ConvertTimestamp(timestamp)\n      results.append((timestamp, \"CHROME_VISIT\", url, title, typed_count, \"\"))\n\n    results.sort(key=lambda it: it[0])\n    for it in results:\n      yield it"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef ParseFromUnicode(self, value):\n    precondition.AssertType(value, Text)\n    value = value.strip()\n\n    super(ClientURN, self).ParseFromUnicode(value)\n\n    match = self.CLIENT_ID_RE.match(self._string_urn)\n    if not match:\n      raise type_info.TypeValueError(\"Client urn malformed: %s\" % value)\n\n    clientid = match.group(\"clientid\")\n    clientid_correctcase = \"\".join((clientid[0].upper(), clientid[1:].lower()))\n\n    self._string_urn = self._string_urn.replace(clientid, clientid_correctcase,\n                                                1)", "response": "Parses a string into a client URN."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef Add(self, path, age=None):\n    if not isinstance(path, string_types):\n      raise ValueError(\"Only strings should be added to a URN.\")\n\n    result = rdfvalue.RDFURN(self.Copy(age))\n    result.Update(path=utils.JoinPath(self._string_urn, path))\n\n    return result", "response": "Adds a relative stem to the current value and returns a new RDFURN."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef MergeOrAddUser(self, kb_user):\n\n    user = self.GetUser(\n        sid=kb_user.sid, uid=kb_user.uid, username=kb_user.username)\n    new_attrs = []\n    merge_conflicts = []  # Record when we overwrite a value.\n    if not user:\n      new_attrs = self._CreateNewUser(kb_user)\n    else:\n      for key, val in iteritems(kb_user.AsDict()):\n        if user.Get(key) and user.Get(key) != val:\n          merge_conflicts.append((key, user.Get(key), val))\n        user.Set(key, val)\n        new_attrs.append(\"users.%s\" % key)\n\n    return new_attrs, merge_conflicts", "response": "Merge a user into existing users or add new if it doesn t exist."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef GetUser(self, sid=None, uid=None, username=None):\n    if sid:\n      for user in self.users:\n        if user.sid == sid:\n          return user\n      return None\n    if uid:\n      for user in self.users:\n        if user.uid == uid:\n          return user\n    if username:\n      for user in self.users:\n        if user.username == username:\n          # Make sure we aren't combining different uids if we know them\n          # user.uid = 0 is the default, which makes this more complicated.\n          if uid and user.uid and user.uid != uid:\n            return None\n          else:\n            return user", "response": "Retrieve a User based on sid uid or username."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef arch(self):\n    if self.machine in [\"x86_64\", \"AMD64\", \"i686\"]:\n      # 32 bit binaries running on AMD64 will still have a i386 arch.\n      if self.architecture == \"32bit\":\n        return \"i386\"\n\n      return \"amd64\"\n    elif self.machine == \"x86\":\n      return \"i386\"\n\n    return self.machine", "response": "Return a more standard representation of the architecture."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nfill a Uname from the currently running platform.", "response": "def FromCurrentSystem(cls):\n    \"\"\"Fill a Uname from the currently running platform.\"\"\"\n    uname = platform.uname()\n    fqdn = socket.getfqdn()\n    system = uname[0]\n    architecture, _ = platform.architecture()\n    if system == \"Windows\":\n      service_pack = platform.win32_ver()[2]\n      kernel = uname[3]  # 5.1.2600\n      release = uname[2]  # XP, 2000, 7\n      version = uname[3] + service_pack  # 5.1.2600 SP3, 6.1.7601 SP1\n    elif system == \"Darwin\":\n      kernel = uname[2]  # 12.2.0\n      release = \"OSX\"  # OSX\n      version = platform.mac_ver()[0]  # 10.8.2\n    elif system == \"Linux\":\n      kernel = uname[2]  # 3.2.5\n      release = platform.linux_distribution()[0]  # Ubuntu\n      version = platform.linux_distribution()[1]  # 12.04\n\n    # Emulate PEP 425 naming conventions - e.g. cp27-cp27mu-linux_x86_64.\n    if pep425tags:\n      pep425tag = \"%s%s-%s-%s\" % (\n          pep425tags.get_abbr_impl(), pep425tags.get_impl_ver(),\n          str(pep425tags.get_abi_tag()).lower(), pep425tags.get_platform())\n    else:\n      # For example: windows_7_amd64\n      pep425tag = \"%s_%s_%s\" % (system, release, architecture)\n\n    return cls(\n        system=system,\n        architecture=architecture,\n        release=release,\n        version=version,\n        machine=uname[4],  # x86, x86_64\n        kernel=kernel,\n        fqdn=fqdn,\n        pep425tag=pep425tag,\n    )"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nparsing the History file.", "response": "def Parse(self, stat, file_object, knowledge_base):\n    \"\"\"Parse the History file.\"\"\"\n    _, _ = stat, knowledge_base\n    # TODO(user): Convert this to use the far more intelligent plaso parser.\n    ie = IEParser(file_object)\n    for dat in ie.Parse():\n      yield rdf_webhistory.BrowserHistoryItem(\n          url=dat[\"url\"],\n          domain=urlparse.urlparse(dat[\"url\"]).netloc,\n          access_time=dat.get(\"mtime\"),\n          program_name=\"Internet Explorer\",\n          source_urn=file_object.urn)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef Parse(self):\n    if not self._file:\n      logging.error(\"Couldn't open file\")\n      return\n\n    # Limit read size to 5MB.\n    self.input_dat = self._file.read(1024 * 1024 * 5)\n    if not self.input_dat.startswith(self.FILE_HEADER):\n      logging.error(\"Invalid index.dat file %s\", self._file)\n      return\n\n    # Events aren't time ordered in the history file, so we collect them all\n    # then sort.\n    events = []\n    for event in self._DoParse():\n      events.append(event)\n\n    for event in sorted(events, key=operator.itemgetter(\"mtime\")):\n      yield event", "response": "Parse the file and return events."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nretrieve a single browser history record from the file.", "response": "def _GetRecord(self, offset, record_size):\n    \"\"\"Retrieve a single record from the file.\n\n    Args:\n      offset: offset from start of input_dat where header starts\n      record_size: length of the header according to file (untrusted)\n\n    Returns:\n      A dict containing a single browser history record.\n    \"\"\"\n    record_header = \"<4sLQQL\"\n    get4 = lambda x: struct.unpack(\"<L\", self.input_dat[x:x + 4])[0]\n    url_offset = struct.unpack(\"B\", self.input_dat[offset + 52:offset + 53])[0]\n    if url_offset in [0xFF, 0xFE]:\n      return None\n    data_offset = get4(offset + 68)\n    data_size = get4(offset + 72)\n    start_pos = offset + data_offset\n    data = struct.unpack(\"{0}s\".format(data_size),\n                         self.input_dat[start_pos:start_pos + data_size])[0]\n    fmt = record_header\n    unknown_size = url_offset - struct.calcsize(fmt)\n    fmt += \"{0}s\".format(unknown_size)\n    fmt += \"{0}s\".format(record_size - struct.calcsize(fmt))\n    dat = struct.unpack(fmt, self.input_dat[offset:offset + record_size])\n    header, blocks, mtime, ctime, ftime, _, url = dat\n    url = url.split(b\"\\x00\")[0].decode(\"utf-8\")\n    if mtime:\n      mtime = mtime // 10 - WIN_UNIX_DIFF_MSECS\n    if ctime:\n      ctime = ctime // 10 - WIN_UNIX_DIFF_MSECS\n    return {\n        \"header\": header,  # the header\n        \"blocks\": blocks,  # number of blocks\n        \"urloffset\": url_offset,  # offset of URL in file\n        \"data_offset\": data_offset,  # offset for start of data\n        \"data_size\": data_size,  # size of data\n        \"data\": data,  # actual data\n        \"mtime\": mtime,  # modified time\n        \"ctime\": ctime,  # created time\n        \"ftime\": ftime,  # file time\n        \"url\": url  # the url visited\n    }"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _DoParse(self):\n    get4 = lambda x: struct.unpack(\"<L\", self.input_dat[x:x + 4])[0]\n    filesize = get4(0x1c)\n    offset = get4(0x20)\n    coffset = offset\n    while coffset < filesize:\n      etype = struct.unpack(\"4s\", self.input_dat[coffset:coffset + 4])[0]\n      if etype == b\"REDR\":\n        pass\n      elif etype in [b\"URL \"]:\n        # Found a valid record\n        reclen = get4(coffset + 4) * self.BLOCK_SIZE\n        yield self._GetRecord(coffset, reclen)\n      coffset += self.BLOCK_SIZE", "response": "Parse a file for browser history records yielding dicts."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nupdates a config with a random csrf key.", "response": "def _GenerateCSRFKey(config):\n  \"\"\"Update a config with a random csrf key.\"\"\"\n  secret_key = config.Get(\"AdminUI.csrf_secret_key\", None)\n  if not secret_key:\n    # TODO(amoser): Remove support for django_secret_key.\n    secret_key = config.Get(\"AdminUI.django_secret_key\", None)\n    if secret_key:\n      config.Set(\"AdminUI.csrf_secret_key\", secret_key)\n\n  if not secret_key:\n    key = utils.GeneratePassphrase(length=100)\n    config.Set(\"AdminUI.csrf_secret_key\", key)\n  else:\n    print(\"Not updating csrf key as it is already set.\")"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef GenerateKeys(config, overwrite_keys=False):\n  if not hasattr(key_utils, \"MakeCACert\"):\n    raise OpenSourceKeyUtilsRequiredError(\n        \"Generate keys can only run with open source key_utils.\")\n  if (config.Get(\"PrivateKeys.server_key\", default=None) and\n      not overwrite_keys):\n    print(config.Get(\"PrivateKeys.server_key\"))\n    raise KeysAlreadyExistError(\n        \"Config %s already has keys, use --overwrite_keys to \"\n        \"override.\" % config.parser)\n\n  length = grr_config.CONFIG[\"Server.rsa_key_length\"]\n  print(\"All keys will have a bit length of %d.\" % length)\n  print(\"Generating executable signing key\")\n  executable_key = rdf_crypto.RSAPrivateKey.GenerateKey(bits=length)\n  config.Set(\"PrivateKeys.executable_signing_private_key\",\n             executable_key.AsPEM())\n  config.Set(\"Client.executable_signing_public_key\",\n             executable_key.GetPublicKey().AsPEM())\n\n  print(\"Generating CA keys\")\n  ca_key = rdf_crypto.RSAPrivateKey.GenerateKey(bits=length)\n  ca_cert = key_utils.MakeCACert(ca_key)\n  config.Set(\"CA.certificate\", ca_cert.AsPEM())\n  config.Set(\"PrivateKeys.ca_key\", ca_key.AsPEM())\n\n  print(\"Generating Server keys\")\n  server_key = rdf_crypto.RSAPrivateKey.GenerateKey(bits=length)\n  server_cert = key_utils.MakeCASignedCert(u\"grr\", server_key, ca_cert, ca_key)\n  config.Set(\"Frontend.certificate\", server_cert.AsPEM())\n  config.Set(\"PrivateKeys.server_key\", server_key.AsPEM())\n\n  print(\"Generating secret key for csrf protection.\")\n  _GenerateCSRFKey(config)", "response": "Generate the keys we need for a GRR server."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef ProcessIterator(pids, process_regex_string, ignore_grr_process, error_list):\n  pids = set(pids)\n  if ignore_grr_process:\n    grr_pid = psutil.Process().pid\n  else:\n    grr_pid = -1\n\n  if process_regex_string:\n    process_regex = re.compile(process_regex_string)\n  else:\n    process_regex = None\n\n  if pids:\n    process_iterator = []\n    for pid in pids:\n      try:\n        process_iterator.append(psutil.Process(pid=pid))\n      except Exception as e:  # pylint: disable=broad-except\n        error_list.Append(\n            rdf_memory.ProcessMemoryError(\n                process=rdf_client.Process(pid=pid), error=str(e)))\n  else:\n    process_iterator = psutil.process_iter()\n\n  for p in process_iterator:\n    if process_regex and not process_regex.search(p.name()):\n      continue\n\n    if p.pid == grr_pid:\n      continue\n\n    yield p", "response": "Yields all processes that match certain criteria."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nfetches flow s data and return proper Flow object.", "response": "def Get(self):\n    \"\"\"Fetch flow's data and return proper Flow object.\"\"\"\n\n    args = flow_pb2.ApiGetFlowArgs(\n        client_id=self.client_id, flow_id=self.flow_id)\n    data = self._context.SendRequest(\"GetFlow\", args)\n    return Flow(data=data, context=self._context)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef WaitUntilDone(self, timeout=None):\n\n    f = utils.Poll(\n        generator=self.Get,\n        condition=lambda f: f.data.state != f.data.RUNNING,\n        timeout=timeout)\n    if f.data.state != f.data.TERMINATED:\n      raise errors.FlowFailedError(\n          \"Flow %s (%s) failed: %s\" %\n          (self.flow_id, self.client_id, f.data.context.current_state))\n    return f", "response": "Wait until the flow completes."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nget the first fingerprint type from the protobuf.", "response": "def GetFingerprint(self, name):\n    \"\"\"Gets the first fingerprint type from the protobuf.\"\"\"\n    for result in self.results:\n      if result.GetItem(\"name\") == name:\n        return result"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef InitializeContext(self, args):\n    if args is None:\n      args = rdf_flow_runner.FlowRunnerArgs()\n\n    output_plugins_states = []\n    for plugin_descriptor in args.output_plugins:\n      if not args.client_id:\n        self.Log(\n            \"Not initializing output plugin %s as flow does not run on \"\n            \"the client.\", plugin_descriptor.plugin_name)\n        continue\n\n      plugin_class = plugin_descriptor.GetPluginClass()\n      try:\n        plugin, plugin_state = plugin_class.CreatePluginAndDefaultState(\n            source_urn=self.flow_obj.output_urn,\n            args=plugin_descriptor.plugin_args,\n            token=self.token)\n        # TODO(amoser): Those do not need to be inside the state, they\n        # could be part of the plugin descriptor.\n        plugin_state[\"logs\"] = []\n        plugin_state[\"errors\"] = []\n\n        output_plugins_states.append(\n            rdf_flow_runner.OutputPluginState(\n                plugin_state=plugin_state, plugin_descriptor=plugin_descriptor))\n      except Exception as e:  # pylint: disable=broad-except\n        logging.exception(\"Plugin %s failed to initialize (%s), ignoring it.\",\n                          plugin, e)\n\n    parent_creator = None\n    if self.parent_runner:\n      parent_creator = self.parent_runner.context.creator\n\n    context = rdf_flow_runner.FlowContext(\n        create_time=rdfvalue.RDFDatetime.Now(),\n        creator=parent_creator or self.token.username,\n        current_state=\"Start\",\n        output_plugins_states=output_plugins_states,\n        state=rdf_flow_runner.FlowContext.State.RUNNING,\n    )\n\n    return context", "response": "Initializes the context of this flow."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef GetNewSessionID(self):\n    # Calculate a new session id based on the flow args. Note that our caller\n    # can specify the base path to the session id, but they can not influence\n    # the exact session id we pick. This ensures that callers can not engineer a\n    # session id clash forcing us to overwrite an existing flow.\n    base = self.runner_args.base_session_id\n    if base is None:\n      base = self.runner_args.client_id or aff4.ROOT_URN\n      base = base.Add(\"flows\")\n\n    return rdfvalue.SessionID(base=base, queue=self.runner_args.queue)", "response": "Returns a random session id for this flow based on the runner args."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef ScheduleKillNotification(self):\n    # Create a notification for the flow in the future that\n    # indicates that this flow is in progess. We'll delete this\n    # notification when we're done with processing completed\n    # requests. If we're stuck for some reason, the notification\n    # will be delivered later and the stuck flow will get\n    # terminated.\n    kill_timestamp = (rdfvalue.RDFDatetime().Now() + self.stuck_flows_timeout)\n    with queue_manager.QueueManager(token=self.token) as manager:\n      manager.QueueNotification(\n          session_id=self.session_id,\n          in_progress=True,\n          timestamp=kill_timestamp)\n\n    # kill_timestamp may get updated via flow.HeartBeat() calls, so we\n    # have to store it in the context.\n    self.context.kill_timestamp = kill_timestamp", "response": "Schedules a kill notification for this flow."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nprocesses the list of requests and responses for this flow.", "response": "def ProcessCompletedRequests(self, notification, unused_thread_pool=None):\n    \"\"\"Go through the list of requests and process the completed ones.\n\n    We take a snapshot in time of all requests and responses for this flow. We\n    then process as many completed requests as possible. If responses are not\n    quite here we leave it for next time.\n\n    It is safe to call this function as many times as needed. NOTE: We assume\n    that the flow queue is locked so another worker is not processing these\n    messages while we are. It is safe to insert new messages to the flow:state\n    queue.\n\n    Args:\n      notification: The notification object that triggered this processing.\n    \"\"\"\n    self.ScheduleKillNotification()\n    try:\n      self._ProcessCompletedRequests(notification)\n    finally:\n      self.FinalizeProcessCompletedRequests(notification)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nprocessing the completed requests.", "response": "def _ProcessCompletedRequests(self, notification):\n    \"\"\"Does the actual processing of the completed requests.\"\"\"\n    # First ensure that client messages are all removed. NOTE: We make a new\n    # queue manager here because we want only the client messages to be removed\n    # ASAP. This must happen before we actually run the flow to ensure the\n    # client requests are removed from the client queues.\n    with queue_manager.QueueManager(token=self.token) as manager:\n      for request, _ in manager.FetchCompletedRequests(\n          self.session_id, timestamp=(0, notification.timestamp)):\n        # Requests which are not destined to clients have no embedded request\n        # message.\n        if request.HasField(\"request\"):\n          manager.DeQueueClientRequest(request.request)\n\n    # The flow is dead - remove all outstanding requests and responses.\n    if not self.IsRunning():\n      self.queue_manager.DestroyFlowStates(self.session_id)\n      return\n\n    processing = []\n    while True:\n      try:\n        # Here we only care about completed requests - i.e. those requests with\n        # responses followed by a status message.\n        for request, responses in self.queue_manager.FetchCompletedResponses(\n            self.session_id, timestamp=(0, notification.timestamp)):\n\n          if request.id == 0:\n            continue\n\n          if not responses:\n            break\n\n          # We are missing a needed request - maybe its not completed yet.\n          if request.id > self.context.next_processed_request:\n            stats_collector_instance.Get().IncrementCounter(\n                \"grr_response_out_of_order\")\n            break\n\n          # Not the request we are looking for - we have seen it before\n          # already.\n          if request.id < self.context.next_processed_request:\n            self.queue_manager.DeleteRequest(request)\n            continue\n\n          if not responses:\n            continue\n\n          # Do we have all the responses here? This can happen if some of the\n          # responses were lost.\n          if len(responses) != responses[-1].response_id:\n            # If we can retransmit do so. Note, this is different from the\n            # automatic retransmission facilitated by the task scheduler (the\n            # Task.task_ttl field) which would happen regardless of these.\n            if request.transmission_count < 5:\n              stats_collector_instance.Get().IncrementCounter(\n                  \"grr_request_retransmission_count\")\n              request.transmission_count += 1\n              self.ReQueueRequest(request)\n            break\n\n          # If we get here its all good - run the flow.\n          if self.IsRunning():\n            self.flow_obj.HeartBeat()\n            self.RunStateMethod(request.next_state, request, responses)\n\n          # Quit early if we are no longer alive.\n          else:\n            break\n\n          # At this point we have processed this request - we can remove it and\n          # its responses from the queue.\n          self.queue_manager.DeleteRequest(request)\n          self.context.next_processed_request += 1\n          self.DecrementOutstandingRequests()\n\n        # Are there any more outstanding requests?\n        if not self.OutstandingRequests():\n          # Allow the flow to cleanup\n          if self.IsRunning() and self.context.current_state != \"End\":\n            self.RunStateMethod(\"End\")\n\n        # Rechecking the OutstandingRequests allows the End state (which was\n        # called above) to issue further client requests - hence postpone\n        # termination.\n        if not self.OutstandingRequests():\n          # TODO(user): Deprecate in favor of 'flow_completions' metric.\n          stats_collector_instance.Get().IncrementCounter(\n              \"grr_flow_completed_count\")\n\n          stats_collector_instance.Get().IncrementCounter(\n              \"flow_completions\", fields=[self.flow_obj.Name()])\n          logging.debug(\n              \"Destroying session %s(%s) for client %s\", self.session_id,\n              self.flow_obj.Name(), self.runner_args.client_id)\n\n          self.flow_obj.Terminate()\n\n        # We are done here.\n        return\n\n      except queue_manager.MoreDataException:\n        # Join any threads.\n        for event in processing:\n          event.wait()\n\n        # We did not read all the requests/responses in this run in order to\n        # keep a low memory footprint and have to make another pass.\n        self.FlushMessages()\n        self.flow_obj.Flush()\n        continue\n\n      finally:\n        # Join any threads.\n        for event in processing:\n          event.wait()"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nruns the state method specified by method_name.", "response": "def RunStateMethod(self, method_name, request=None, responses=None):\n    \"\"\"Completes the request by calling the state method.\n\n    Args:\n      method_name: The name of the state method to call.\n      request: A RequestState protobuf.\n      responses: A list of GrrMessages responding to the request.\n    \"\"\"\n    if self._TerminationPending():\n      return\n\n    client_id = None\n    try:\n      self.context.current_state = method_name\n      if request and responses:\n        client_id = request.client_id or self.runner_args.client_id\n        logging.debug(\"%s Running %s with %d responses from %s\",\n                      self.session_id, method_name, len(responses), client_id)\n\n      else:\n        logging.debug(\"%s Running state method %s\", self.session_id,\n                      method_name)\n\n      # Extend our lease if needed.\n      self.flow_obj.HeartBeat()\n      try:\n        method = getattr(self.flow_obj, method_name)\n      except AttributeError:\n        raise FlowRunnerError(\"Flow %s has no state method %s\" %\n                              (self.flow_obj.__class__.__name__, method_name))\n\n      # Prepare a responses object for the state method to use:\n      responses = flow_responses.Responses.FromLegacyResponses(\n          request=request, responses=responses)\n\n      self.SaveResourceUsage(responses.status)\n\n      stats_collector_instance.Get().IncrementCounter(\"grr_worker_states_run\")\n\n      if method_name == \"Start\":\n        stats_collector_instance.Get().IncrementCounter(\n            \"flow_starts\", fields=[self.flow_obj.Name()])\n        method()\n      else:\n        method(responses)\n\n      if self.sent_replies:\n        self.ProcessRepliesWithOutputPlugins(self.sent_replies)\n        self.sent_replies = []\n\n    # We don't know here what exceptions can be thrown in the flow but we have\n    # to continue. Thus, we catch everything.\n    except Exception as e:  # pylint: disable=broad-except\n      # This flow will terminate now\n\n      # TODO(user): Deprecate in favor of 'flow_errors'.\n      stats_collector_instance.Get().IncrementCounter(\"grr_flow_errors\")\n\n      stats_collector_instance.Get().IncrementCounter(\n          \"flow_errors\", fields=[self.flow_obj.Name()])\n      logging.exception(\"Flow %s raised %s.\", self.session_id, e)\n\n      self.Error(traceback.format_exc(), client_id=client_id)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncall the client asynchronously.", "response": "def CallClient(self,\n                 action_cls,\n                 request=None,\n                 next_state=None,\n                 request_data=None,\n                 **kwargs):\n    \"\"\"Calls the client asynchronously.\n\n    This sends a message to the client to invoke an Action. The run\n    action may send back many responses. These will be queued by the\n    framework until a status message is sent by the client. The status\n    message will cause the entire transaction to be committed to the\n    specified state.\n\n    Args:\n       action_cls: The function to call on the client.\n       request: The request to send to the client. If not specified (Or None) we\n         create a new RDFValue using the kwargs.\n       next_state: The state in this flow, that responses to this message should\n         go to.\n       request_data: A dict which will be available in the RequestState\n         protobuf. The Responses object maintains a reference to this protobuf\n         for use in the execution of the state method. (so you can access this\n         data by responses.request). Valid values are strings, unicode and\n         protobufs.\n       **kwargs: These args will be used to construct the client action semantic\n         protobuf.\n\n    Raises:\n       FlowRunnerError: If called on a flow that doesn't run on a single client.\n       ValueError: The request passed to the client does not have the correct\n                     type.\n    \"\"\"\n    client_id = self.runner_args.client_id\n    if client_id is None:\n      raise FlowRunnerError(\"CallClient() is used on a flow which was not \"\n                            \"started with a client.\")\n\n    if not isinstance(client_id, rdf_client.ClientURN):\n      # Try turning it into a ClientURN\n      client_id = rdf_client.ClientURN(client_id)\n\n    if action_cls.in_rdfvalue is None:\n      if request:\n        raise ValueError(\n            \"Client action %s does not expect args.\" % action_cls.__name__)\n    else:\n      if request is None:\n        # Create a new rdf request.\n        request = action_cls.in_rdfvalue(**kwargs)\n      else:\n        # Verify that the request type matches the client action requirements.\n        if not isinstance(request, action_cls.in_rdfvalue):\n          raise ValueError(\"Client action expected %s but got %s\" %\n                           (action_cls.in_rdfvalue, type(request)))\n\n    outbound_id = self.GetNextOutboundId()\n\n    # Create a new request state\n    state = rdf_flow_runner.RequestState(\n        id=outbound_id,\n        session_id=self.session_id,\n        next_state=next_state,\n        client_id=client_id)\n\n    if request_data is not None:\n      state.data = rdf_protodict.Dict(request_data)\n\n    # Send the message with the request state\n    msg = rdf_flows.GrrMessage(\n        session_id=utils.SmartUnicode(self.session_id),\n        name=action_cls.__name__,\n        request_id=outbound_id,\n        require_fastpoll=self.runner_args.require_fastpoll,\n        queue=client_id.Queue(),\n        payload=request,\n        generate_task_id=True)\n\n    cpu_usage = self.context.client_resources.cpu_usage\n    if self.runner_args.cpu_limit:\n      msg.cpu_limit = max(\n          self.runner_args.cpu_limit - cpu_usage.user_cpu_time -\n          cpu_usage.system_cpu_time, 0)\n\n      if msg.cpu_limit == 0:\n        raise FlowRunnerError(\"CPU limit exceeded.\")\n\n    if self.runner_args.network_bytes_limit:\n      msg.network_bytes_limit = max(\n          self.runner_args.network_bytes_limit -\n          self.context.network_bytes_sent, 0)\n      if msg.network_bytes_limit == 0:\n        raise FlowRunnerError(\"Network limit exceeded.\")\n\n    state.request = msg\n    self.QueueRequest(state)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef CallFlow(self,\n               flow_name=None,\n               next_state=None,\n               request_data=None,\n               client_id=None,\n               base_session_id=None,\n               **kwargs):\n    \"\"\"Creates a new flow and send its responses to a state.\n\n    This creates a new flow. The flow may send back many responses which will be\n    queued by the framework until the flow terminates. The final status message\n    will cause the entire transaction to be committed to the specified state.\n\n    Args:\n       flow_name: The name of the flow to invoke.\n       next_state: The state in this flow, that responses to this message should\n         go to.\n       request_data: Any dict provided here will be available in the\n         RequestState protobuf. The Responses object maintains a reference to\n         this protobuf for use in the execution of the state method. (so you can\n         access this data by responses.request). There is no format mandated on\n         this data but it may be a serialized protobuf.\n       client_id: If given, the flow is started for this client.\n       base_session_id: A URN which will be used to build a URN.\n       **kwargs: Arguments for the child flow.\n\n    Raises:\n       FlowRunnerError: If next_state is not one of the allowed next states.\n\n    Returns:\n       The URN of the child flow which was created.\n    \"\"\"\n    client_id = client_id or self.runner_args.client_id\n\n    # This looks very much like CallClient() above - we prepare a request state,\n    # and add it to our queue - any responses from the child flow will return to\n    # the request state and the stated next_state. Note however, that there is\n    # no client_id or actual request message here because we directly invoke the\n    # child flow rather than queue anything for it.\n    state = rdf_flow_runner.RequestState(\n        id=self.GetNextOutboundId(),\n        session_id=utils.SmartUnicode(self.session_id),\n        client_id=client_id,\n        next_state=next_state,\n        response_count=0)\n\n    if request_data:\n      state.data = rdf_protodict.Dict().FromDict(request_data)\n\n    # If the urn is passed explicitly (e.g. from the hunt runner) use that,\n    # otherwise use the urn from the flow_runner args. If both are None, create\n    # a new collection and give the urn to the flow object.\n    logs_urn = self._GetLogCollectionURN(\n        kwargs.pop(\"logs_collection_urn\", None) or\n        self.runner_args.logs_collection_urn)\n\n    # If we were called with write_intermediate_results, propagate down to\n    # child flows.  This allows write_intermediate_results to be set to True\n    # either at the top level parent, or somewhere in the middle of\n    # the call chain.\n    write_intermediate = (\n        kwargs.pop(\"write_intermediate_results\", False) or\n        self.runner_args.write_intermediate_results)\n\n    # Create the new child flow but do not notify the user about it.\n    child_urn = self.flow_obj.StartAFF4Flow(\n        client_id=client_id,\n        flow_name=flow_name,\n        base_session_id=base_session_id or self.session_id,\n        request_state=state,\n        token=self.token,\n        notify_to_user=False,\n        parent_flow=self.flow_obj,\n        queue=self.runner_args.queue,\n        write_intermediate_results=write_intermediate,\n        logs_collection_urn=logs_urn,\n        sync=True,\n        **kwargs)\n\n    self.QueueRequest(state)\n\n    return child_urn", "response": "Creates a new flow and sends the next_state to the next_state of the flow."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nallowing this flow to send a message to its parent flow.", "response": "def SendReply(self, response, tag=None):\n    \"\"\"Allows this flow to send a message to its parent flow.\n\n    If this flow does not have a parent, the message is ignored.\n\n    Args:\n      response: An RDFValue() instance to be sent to the parent.\n      tag: If specified, tag the result with the following tag. NOTE: supported\n        in REL_DB implementation only.\n\n    Raises:\n      ValueError: If responses is not of the correct type.\n    \"\"\"\n    del tag\n\n    if not isinstance(response, rdfvalue.RDFValue):\n      raise ValueError(\"SendReply can only send a Semantic Value\")\n\n    # Only send the reply if we have a parent, indicated by knowing our parent's\n    # request state.\n    if self.runner_args.request_state.session_id:\n\n      request_state = self.runner_args.request_state\n\n      request_state.response_count += 1\n\n      # Make a response message\n      msg = rdf_flows.GrrMessage(\n          session_id=request_state.session_id,\n          request_id=request_state.id,\n          response_id=request_state.response_count,\n          auth_state=rdf_flows.GrrMessage.AuthorizationState.AUTHENTICATED,\n          type=rdf_flows.GrrMessage.Type.MESSAGE,\n          payload=response,\n          args_rdf_name=response.__class__.__name__,\n          args_age=int(response.age))\n\n      # Queue the response now\n      self.queue_manager.QueueResponse(msg)\n\n      if self.runner_args.write_intermediate_results:\n        self.QueueReplyForResultCollection(response)\n\n    else:\n      # Only write the reply to the collection if we are the parent flow.\n      self.QueueReplyForResultCollection(response)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef Error(self, backtrace, client_id=None, status_code=None):\n    try:\n      self.queue_manager.DestroyFlowStates(self.session_id)\n    except queue_manager.MoreDataException:\n      pass\n\n    if not self.IsRunning():\n      return\n\n    # Set an error status\n    reply = rdf_flows.GrrStatus()\n    if status_code is None:\n      reply.status = rdf_flows.GrrStatus.ReturnedStatus.GENERIC_ERROR\n    else:\n      reply.status = status_code\n\n    client_id = client_id or self.runner_args.client_id\n\n    if backtrace:\n      reply.error_message = backtrace\n      logging.error(\"Error in flow %s (%s). Trace: %s\", self.session_id,\n                    client_id, backtrace)\n      self.context.backtrace = backtrace\n    else:\n      logging.error(\"Error in flow %s (%s).\", self.session_id, client_id)\n\n    self._SendTerminationMessage(reply)\n\n    self.context.state = rdf_flow_runner.FlowContext.State.ERROR\n\n    if self.ShouldSendNotifications():\n      flow_ref = None\n      if client_id:\n        flow_ref = rdf_objects.FlowReference(\n            client_id=client_id.Basename(), flow_id=self.session_id.Basename())\n      notification_lib.Notify(\n          self.token.username,\n          rdf_objects.UserNotification.Type.TYPE_FLOW_RUN_FAILED,\n          \"Flow (%s) terminated due to error\" % self.session_id,\n          rdf_objects.ObjectReference(\n              reference_type=rdf_objects.ObjectReference.Type.FLOW,\n              flow=flow_ref))\n\n    self.flow_obj.Flush()", "response": "Terminates this flow with an error."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef ProcessRepliesWithOutputPlugins(self, replies):\n    for output_plugin_state in self.context.output_plugins_states:\n      plugin_descriptor = output_plugin_state.plugin_descriptor\n      output_plugin_cls = plugin_descriptor.GetPluginClass()\n      output_plugin = output_plugin_cls(\n          source_urn=self.flow_obj.urn,\n          args=plugin_descriptor.plugin_args,\n          token=self.token)\n\n      # Extend our lease if needed.\n      self.flow_obj.HeartBeat()\n      try:\n        output_plugin.ProcessResponses(output_plugin_state.plugin_state,\n                                       replies)\n        output_plugin.Flush(output_plugin_state.plugin_state)\n        output_plugin.UpdateState(output_plugin_state.plugin_state)\n\n        log_item = output_plugin_lib.OutputPluginBatchProcessingStatus(\n            plugin_descriptor=plugin_descriptor,\n            status=\"SUCCESS\",\n            batch_size=len(replies))\n        output_plugin_state.Log(log_item)\n\n        self.Log(\"Plugin %s successfully processed %d flow replies.\",\n                 plugin_descriptor, len(replies))\n      except Exception as e:  # pylint: disable=broad-except\n        error = output_plugin_lib.OutputPluginBatchProcessingStatus(\n            plugin_descriptor=plugin_descriptor,\n            status=\"ERROR\",\n            summary=utils.SmartUnicode(e),\n            batch_size=len(replies))\n        output_plugin_state.Error(error)\n\n        self.Log(\"Plugin %s failed to process %d replies due to: %s\",\n                 plugin_descriptor, len(replies), e)", "response": "Processes replies with output plugins."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _SendTerminationMessage(self, status=None):\n    if not self.runner_args.request_state.session_id:\n      # No parent flow, nothing to do here.\n      return\n\n    if status is None:\n      status = rdf_flows.GrrStatus()\n\n    client_resources = self.context.client_resources\n    user_cpu = client_resources.cpu_usage.user_cpu_time\n    sys_cpu = client_resources.cpu_usage.system_cpu_time\n    status.cpu_time_used.user_cpu_time = user_cpu\n    status.cpu_time_used.system_cpu_time = sys_cpu\n    status.network_bytes_sent = self.context.network_bytes_sent\n    status.child_session_id = self.session_id\n\n    request_state = self.runner_args.request_state\n    request_state.response_count += 1\n\n    # Make a response message\n    msg = rdf_flows.GrrMessage(\n        session_id=request_state.session_id,\n        request_id=request_state.id,\n        response_id=request_state.response_count,\n        auth_state=rdf_flows.GrrMessage.AuthorizationState.AUTHENTICATED,\n        type=rdf_flows.GrrMessage.Type.STATUS,\n        payload=status)\n\n    # Queue the response now\n    self.queue_manager.QueueResponse(msg)\n    self.QueueNotification(session_id=request_state.session_id)", "response": "Send a termination message to the parent flow."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nupdating the cpu and network stats.", "response": "def UpdateProtoResources(self, status):\n    \"\"\"Save cpu and network stats, check limits.\"\"\"\n    user_cpu = status.cpu_time_used.user_cpu_time\n    system_cpu = status.cpu_time_used.system_cpu_time\n    self.context.client_resources.cpu_usage.user_cpu_time += user_cpu\n    self.context.client_resources.cpu_usage.system_cpu_time += system_cpu\n\n    user_cpu_total = self.context.client_resources.cpu_usage.user_cpu_time\n    system_cpu_total = self.context.client_resources.cpu_usage.system_cpu_time\n\n    self.context.network_bytes_sent += status.network_bytes_sent\n\n    if self.runner_args.cpu_limit:\n      if self.runner_args.cpu_limit < (user_cpu_total + system_cpu_total):\n        # We have exceeded our limit, stop this flow.\n        raise FlowRunnerError(\"CPU limit exceeded.\")\n\n    if self.runner_args.network_bytes_limit:\n      if (self.runner_args.network_bytes_limit <\n          self.context.network_bytes_sent):\n        # We have exceeded our byte limit, stop this flow.\n        raise FlowRunnerError(\"Network bytes limit exceeded.\")"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _HostPrefix(client_id):\n  if not client_id:\n    return \"\"\n\n  hostname = None\n  if data_store.RelationalDBEnabled():\n    client_snapshot = data_store.REL_DB.ReadClientSnapshot(client_id)\n    if client_snapshot:\n      hostname = client_snapshot.knowledge_base.fqdn\n  else:\n    client_fd = aff4.FACTORY.Open(client_id, mode=\"rw\")\n    hostname = client_fd.Get(client_fd.Schema.FQDN) or \"\"\n\n  if hostname:\n    return \"%s: \" % hostname\n  else:\n    return \"\"", "response": "Build a host prefix for a notification message based on a client id."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nmap legacy arguments to GRRUser. Notify arguments.", "response": "def _MapLegacyArgs(nt, message, ref):\n  \"\"\"Maps UserNotification object to legacy GRRUser.Notify arguments.\"\"\"\n  unt = rdf_objects.UserNotification.Type\n\n  if nt == unt.TYPE_CLIENT_INTERROGATED:\n    return [\n        \"Discovery\",\n        aff4.ROOT_URN.Add(ref.client.client_id),\n        _HostPrefix(ref.client.client_id) + message,\n        \"\",\n    ]\n  elif nt == unt.TYPE_CLIENT_APPROVAL_REQUESTED:\n    return [\n        \"GrantAccess\",\n        aff4.ROOT_URN.Add(\"ACL\").Add(ref.approval_request.subject_id).Add(\n            ref.approval_request.requestor_username).Add(\n                ref.approval_request.approval_id),\n        message,\n        \"\",\n    ]\n  elif nt == unt.TYPE_HUNT_APPROVAL_REQUESTED:\n    return [\n        \"GrantAccess\",\n        aff4.ROOT_URN.Add(\"ACL\").Add(\"hunts\").Add(\n            ref.approval_request.subject_id).Add(\n                ref.approval_request.requestor_username).Add(\n                    ref.approval_request.approval_id),\n        message,\n        \"\",\n    ]\n  elif nt == unt.TYPE_CRON_JOB_APPROVAL_REQUESTED:\n    return [\n        \"GrantAccess\",\n        aff4.ROOT_URN.Add(\"ACL\").Add(\"cron\").Add(\n            ref.approval_request.subject_id).Add(\n                ref.approval_request.requestor_username).Add(\n                    ref.approval_request.approval_id),\n        message,\n        \"\",\n    ]\n  elif nt == unt.TYPE_CLIENT_APPROVAL_GRANTED:\n    return [\n        \"ViewObject\",\n        aff4.ROOT_URN.Add(ref.client.client_id),\n        message,\n        \"\",\n    ]\n  elif nt == unt.TYPE_HUNT_APPROVAL_GRANTED:\n    return [\n        \"ViewObject\",\n        aff4.ROOT_URN.Add(\"hunts\").Add(ref.hunt.hunt_id),\n        message,\n        \"\",\n    ]\n  elif nt == unt.TYPE_CRON_JOB_APPROVAL_GRANTED:\n    return [\n        \"ViewObject\",\n        aff4.ROOT_URN.Add(\"cron\").Add(ref.cron_job.cron_job_id),\n        message,\n        \"\",\n    ]\n  elif nt == unt.TYPE_VFS_FILE_COLLECTED:\n    return [\n        \"ViewObject\",\n        ref.vfs_file.ToURN(),\n        _HostPrefix(ref.vfs_file.client_id) + message,\n        \"\",\n    ]\n  elif nt == unt.TYPE_VFS_FILE_COLLECTION_FAILED:\n    return [\n        \"ViewObject\",\n        ref.vfs_file.ToURN(),\n        _HostPrefix(ref.vfs_file.client_id) + message,\n        \"\",\n    ]\n  elif nt == unt.TYPE_HUNT_STOPPED:\n    urn = aff4.ROOT_URN.Add(\"hunts\").Add(ref.hunt.hunt_id)\n    return [\n        \"ViewObject\",\n        urn,\n        message,\n        urn,\n    ]\n  elif nt == unt.TYPE_FILE_ARCHIVE_GENERATED:\n    return [\n        \"ArchiveGenerationFinished\",\n        None,\n        message,\n        \"\",\n    ]\n  elif nt == unt.TYPE_FILE_ARCHIVE_GENERATION_FAILED:\n    return [\n        \"Error\",\n        None,\n        message,\n        \"\",\n    ]\n  elif nt == unt.TYPE_FLOW_RUN_COMPLETED:\n    urn = None\n    if ref.flow and ref.flow.client_id and ref.flow.flow_id:\n      urn = aff4.ROOT_URN.Add(ref.flow.client_id).Add(\"flows\").Add(\n          ref.flow.flow_id)\n    return [\n        \"ViewObject\",\n        urn,\n        _HostPrefix(ref.flow.client_id) + message,\n        \"\",\n    ]\n  elif nt == unt.TYPE_FLOW_RUN_FAILED:\n    client_id = None\n    urn = None\n    prefix = \"\"\n    if ref.flow is not None:\n      client_id = ref.flow.client_id\n\n      if client_id:\n        prefix = _HostPrefix(client_id)\n\n        if ref.flow.flow_id:\n          urn = aff4.ROOT_URN.Add(ref.flow.client_id).Add(\"flows\").Add(\n              ref.flow.flow_id)\n\n    return [\n        \"FlowStatus\",\n        client_id,\n        prefix + message,\n        urn,\n    ]\n  elif nt == unt.TYPE_VFS_LIST_DIRECTORY_COMPLETED:\n    return [\n        \"ViewObject\",\n        ref.vfs_file.ToURN(),\n        message,\n        \"\",\n    ]\n  elif nt == unt.TYPE_VFS_RECURSIVE_LIST_DIRECTORY_COMPLETED:\n    return [\n        \"ViewObject\",\n        ref.vfs_file.ToURN(),\n        message,\n        \"\",\n    ]\n  else:\n    raise NotImplementedError()"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _NotifyLegacy(username, notification_type, message, object_reference):\n\n  try:\n    with aff4.FACTORY.Open(\n        aff4.ROOT_URN.Add(\"users\").Add(username),\n        aff4_type=aff4_users.GRRUser,\n        mode=\"rw\") as fd:\n\n      args = _MapLegacyArgs(notification_type, message, object_reference)\n      args[0] += \":%s\" % notification_type\n      fd.Notify(*args)\n\n  except aff4.InstantiationError:\n    logging.error(\"Trying to notify non-existent user: %s\", username)", "response": "Schedules a legacy AFF4 user notification."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _Notify(username, notification_type, message, object_reference):\n\n  # Do not try to notify system users (e.g. Cron).\n  if username in aff4_users.GRRUser.SYSTEM_USERS:\n    return\n\n  if object_reference:\n    uc = object_reference.UnionCast()\n    if hasattr(uc, \"client_id\"):\n      message = _HostPrefix(uc.client_id) + message\n\n  n = rdf_objects.UserNotification(\n      username=username,\n      notification_type=notification_type,\n      state=rdf_objects.UserNotification.State.STATE_PENDING,\n      message=message,\n      reference=object_reference)\n  data_store.REL_DB.WriteUserNotification(n)", "response": "Schedules a new - style REL_DB user notification."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nadding an rdf value to a collection.", "response": "def StaticAdd(cls,\n                collection_urn,\n                rdf_value,\n                timestamp=None,\n                suffix=None,\n                mutation_pool=None):\n    \"\"\"Adds an rdf value to a collection.\n\n    Adds an rdf value to a collection. Does not require that the collection be\n    open. NOTE: The caller is responsible for ensuring that the collection\n    exists and is of the correct type.\n\n    Args:\n      collection_urn: The urn of the collection to add to.\n      rdf_value: The rdf value to add to the collection.\n      timestamp: The timestamp (in microseconds) to store the rdf value at.\n        Defaults to the current time.\n      suffix: A 'fractional timestamp' suffix to reduce the chance of\n        collisions. Defaults to a random number.\n      mutation_pool: A MutationPool object to write to.\n\n    Returns:\n      The pair (timestamp, suffix) which identifies the value within the\n      collection.\n\n    Raises:\n      ValueError: rdf_value has unexpected type.\n\n    \"\"\"\n    if not isinstance(rdf_value, cls.RDF_TYPE):\n      raise ValueError(\"This collection only accepts values of type %s.\" %\n                       cls.RDF_TYPE.__name__)\n    if mutation_pool is None:\n      raise ValueError(\"Mutation pool can't be none.\")\n    if timestamp is None:\n      timestamp = rdfvalue.RDFDatetime.Now()\n    if isinstance(timestamp, rdfvalue.RDFDatetime):\n      timestamp = timestamp.AsMicrosecondsSinceEpoch()\n\n    if not rdf_value.age:\n      rdf_value.age = rdfvalue.RDFDatetime.Now()\n\n    if not isinstance(collection_urn, rdfvalue.RDFURN):\n      collection_urn = rdfvalue.RDFURN(collection_urn)\n\n    _, timestamp, suffix = mutation_pool.CollectionAddItem(\n        collection_urn, rdf_value, timestamp, suffix=suffix)\n\n    return timestamp, suffix"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef Add(self, rdf_value, timestamp=None, suffix=None, mutation_pool=None):\n    return self.StaticAdd(\n        self.collection_id,\n        rdf_value,\n        timestamp=timestamp,\n        suffix=suffix,\n        mutation_pool=mutation_pool)", "response": "Adds an rdf value to the internal collection."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef Scan(self, after_timestamp=None, include_suffix=False, max_records=None):\n    suffix = None\n    if isinstance(after_timestamp, tuple):\n      suffix = after_timestamp[1]\n      after_timestamp = after_timestamp[0]\n\n    for item, timestamp, suffix in data_store.DB.CollectionScanItems(\n        self.collection_id,\n        self.RDF_TYPE,\n        after_timestamp=after_timestamp,\n        after_suffix=suffix,\n        limit=max_records):\n      if include_suffix:\n        yield ((timestamp, suffix), item)\n      else:\n        yield (timestamp, item)", "response": "Scans the collection for stored records."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef MultiResolve(self, records):\n    for value, timestamp in data_store.DB.CollectionReadItems(records):\n      rdf_value = self.RDF_TYPE.FromSerializedString(value)\n      rdf_value.age = timestamp\n      yield rdf_value", "response": "Lookup multiple values by their record objects."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _MaybeWriteIndex(self, i, ts, mutation_pool):\n    if i > self._max_indexed and i % self.INDEX_SPACING == 0:\n      # We only write the index if the timestamp is more than 5 minutes in the\n      # past: hacky defense against a late write changing the count.\n      if ts[0] < (rdfvalue.RDFDatetime.Now() -\n                  self.INDEX_WRITE_DELAY).AsMicrosecondsSinceEpoch():\n        mutation_pool.CollectionAddIndex(self.collection_id, i, ts[0], ts[1])\n        self._index[i] = ts\n        self._max_indexed = max(i, self._max_indexed)", "response": "Write index marker i."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nscanning records starting with index i.", "response": "def _IndexedScan(self, i, max_records=None):\n    \"\"\"Scan records starting with index i.\"\"\"\n    self._ReadIndex()\n\n    # The record number that we will read next.\n    idx = 0\n    # The timestamp that we will start reading from.\n    start_ts = 0\n    if i >= self._max_indexed:\n      start_ts = max((0, 0), (self._index[self._max_indexed][0],\n                              self._index[self._max_indexed][1] - 1))\n      idx = self._max_indexed\n    else:\n      try:\n        possible_idx = i - i % self.INDEX_SPACING\n        start_ts = (max(0, self._index[possible_idx][0]),\n                    self._index[possible_idx][1] - 1)\n        idx = possible_idx\n      except KeyError:\n        pass\n\n    if max_records is not None:\n      max_records += i - idx\n\n    with data_store.DB.GetMutationPool() as mutation_pool:\n      for (ts, value) in self.Scan(\n          after_timestamp=start_ts,\n          max_records=max_records,\n          include_suffix=True):\n        self._MaybeWriteIndex(idx, ts, mutation_pool)\n        if idx >= i:\n          yield (idx, ts, value)\n        idx += 1"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef AddAsMessage(self, rdfvalue_in, source, mutation_pool=None):\n    self.Add(\n        rdf_flows.GrrMessage(payload=rdfvalue_in, source=source),\n        mutation_pool=mutation_pool)", "response": "Adds rdfvalues as GrrMessages for testing."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef Parse(self, stat, file_object, knowledge_base):\n    _, _ = stat, knowledge_base\n    lines = [\n        l.strip()\n        for l in utils.ReadFileBytesAsUnicode(file_object).splitlines()\n    ]\n    for index, line in enumerate(lines):\n      user = self.ParseLine(index, line)\n      if user is not None:\n        yield user", "response": "Parse the passwd file."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef Parse(self, stat, file_object, knowledge_base):\n    _, _ = stat, knowledge_base\n    users = {}\n    wtmp = file_object.read()\n    while wtmp:\n      try:\n        record = UtmpStruct(wtmp)\n      except utils.ParsingError:\n        break\n\n      wtmp = wtmp[record.size:]\n      # Users only appear for USER_PROCESS events, others are system.\n      if record.ut_type != 7:\n        continue\n\n      # Lose the null termination\n      record.user = record.user.split(b\"\\x00\", 1)[0]\n\n      # Store the latest login time.\n      # TODO(user): remove the 0 here once RDFDatetime can support times\n      # pre-epoch properly.\n      try:\n        users[record.user] = max(users[record.user], record.sec, 0)\n      except KeyError:\n        users[record.user] = record.sec\n\n    for user, last_login in iteritems(users):\n      yield rdf_client.User(\n          username=utils.SmartUnicode(user), last_logon=last_login * 1000000)", "response": "Parse the WTMP file."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nparse the netgroup file and return User objects.", "response": "def Parse(self, stat, file_object, knowledge_base):\n    \"\"\"Parse the netgroup file and return User objects.\n\n    Lines are of the form:\n      group1 (-,user1,) (-,user2,) (-,user3,)\n\n    Groups are ignored, we return users in lines that match the filter regexes,\n    or all users in the file if no filters are specified.\n\n    We assume usernames are in the default regex format specified in the adduser\n    man page.  Notably no non-ASCII characters.\n\n    Args:\n      stat: unused statentry\n      file_object: netgroup VFSFile\n      knowledge_base: unused\n\n    Returns:\n      rdf_client.User\n    \"\"\"\n    _, _ = stat, knowledge_base\n    lines = [\n        l.strip()\n        for l in utils.ReadFileBytesAsUnicode(file_object).splitlines()\n    ]\n    return self.ParseLines(lines)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nidentifying the type of hash in a hash string.", "response": "def GetHashType(self, hash_str):\n    \"\"\"Identify the type of hash in a hash string.\n\n    Args:\n      hash_str: A string value that may be a hash.\n\n    Returns:\n      A string description of the type of hash.\n    \"\"\"\n    # Return the type of the first matching hash.\n    for hash_type, hash_re in self.hashes:\n      if hash_re.match(hash_str):\n        return hash_type\n    # No hash matched.\n    return \"EMPTY\""}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _ParseFile(self, file_obj, line_parser):\n    lines = [\n        l.strip() for l in utils.ReadFileBytesAsUnicode(file_obj).splitlines()\n    ]\n    try:\n      for index, line in enumerate(lines):\n        if line:\n          line_parser(line)\n    except (IndexError, KeyError) as e:\n      raise parser.ParseError(\"Invalid file at line %d: %s\" % (index + 1, e))", "response": "Parses a file line by line."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreconciling the entry that claim to use shadow files.", "response": "def ReconcileShadow(self, store_type):\n    \"\"\"Verify that entries that claim to use shadow files have a shadow entry.\n\n    If the entries of the non-shadowed file indicate that a shadow file is used,\n    check that there is actually an entry for that file in shadow.\n\n    Args:\n      store_type: The type of password store that should be used (e.g.\n        /etc/shadow or /etc/gshadow)\n    \"\"\"\n    for k, v in iteritems(self.entry):\n      if v.pw_entry.store == store_type:\n        shadow_entry = self.shadow.get(k)\n        if shadow_entry is not None:\n          v.pw_entry = shadow_entry\n        else:\n          v.pw_entry.store = \"UNKNOWN\""}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef MemberDiff(data1, set1_name, data2, set2_name):\n    set1 = set(data1)\n    set2 = set(data2)\n    diffs = []\n    msg = \"Present in %s, missing in %s: %s\"\n    if set1 != set2:\n      in_set1 = set1 - set2\n      in_set2 = set2 - set1\n      if in_set1:\n        diffs.append(msg % (set1_name, set2_name, \",\".join(in_set1)))\n      if in_set2:\n        diffs.append(msg % (set2_name, set1_name, \",\".join(in_set2)))\n    return diffs", "response": "Helper method to perform bidirectional set differences."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nprocesses multiple files together.", "response": "def ParseMultiple(self, stats, file_objs, kb):\n    \"\"\"Process files together.\"\"\"\n    fileset = {stat.pathspec.path: obj for stat, obj in zip(stats, file_objs)}\n    return self.ParseFileset(fileset)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef ParseGshadowEntry(self, line):\n    fields = (\"name\", \"passwd\", \"administrators\", \"members\")\n    if line:\n      rslt = dict(zip(fields, line.split(\":\")))\n      # Add the shadow state to the internal store.\n      name = rslt[\"name\"]\n      pw_entry = self.shadow.setdefault(name, rdf_client.PwEntry())\n      pw_entry.store = self.shadow_store\n      pw_entry.hash_type = self.GetHashType(rslt[\"passwd\"])\n      # Add the members to the internal store.\n      members = self.gshadow_members.setdefault(name, set())\n      for accts in rslt[\"administrators\"], rslt[\"members\"]:\n        if accts:\n          members.update(accts.split(\",\"))", "response": "Parses a group shadow entry."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef ParseGroupEntry(self, line):\n    fields = (\"name\", \"passwd\", \"gid\", \"members\")\n    if line:\n      rslt = dict(zip(fields, line.split(\":\")))\n      name = rslt[\"name\"]\n      group = self.entry.setdefault(name, rdf_client.Group(name=name))\n      group.pw_entry.store = self.GetPwStore(rslt[\"passwd\"])\n      if group.pw_entry.store == self.base_store:\n        group.pw_entry.hash_type = self.GetHashType(rslt[\"passwd\"])\n      # If the group contains NIS entries, they may not have a gid.\n      if rslt[\"gid\"]:\n        group.gid = int(rslt[\"gid\"])\n      group.members = set(rslt[\"members\"].split(\",\"))", "response": "Parses a group entry from the group file."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nadd shadow group members to the group if gshadow is used.", "response": "def MergeMembers(self):\n    \"\"\"Add shadow group members to the group if gshadow is used.\n\n    Normally group and shadow should be in sync, but no guarantees. Merges the\n    two stores as membership in either file may confer membership.\n    \"\"\"\n    for group_name, members in iteritems(self.gshadow_members):\n      group = self.entry.get(group_name)\n      if group and group.pw_entry.store == self.shadow_store:\n        group.members = members.union(group.members)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef FindAnomalies(self):\n    for grp_name, group in iteritems(self.entry):\n      shadow = self.shadow.get(grp_name)\n      gshadows = self.gshadow_members.get(grp_name, [])\n      if shadow is not None:\n        diff = self.MemberDiff(group.members, \"group\", gshadows, \"gshadow\")\n        if diff:\n          msg = \"Group/gshadow members differ in group: %s\" % grp_name\n          yield self._Anomaly(msg, diff)\n\n    diff = self.MemberDiff(self.entry, \"group\", self.gshadow_members, \"gshadow\")\n    if diff:\n      yield self._Anomaly(\"Mismatched group and gshadow files.\", diff)", "response": "Identify any anomalous group attributes or memberships."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef ParseFileset(self, fileset=None):\n    # Get relevant shadow attributes.\n    gshadow = fileset.get(\"/etc/gshadow\")\n    if gshadow:\n      self._ParseFile(gshadow, self.ParseGshadowEntry)\n    else:\n      logging.debug(\"No /etc/gshadow file.\")\n    group = fileset.get(\"/etc/group\")\n    if group:\n      self._ParseFile(group, self.ParseGroupEntry)\n    else:\n      logging.debug(\"No /etc/group file.\")\n    self.ReconcileShadow(self.shadow_store)\n    # Identify any anomalous group/shadow entries.\n    # This needs to be done before memberships are merged: merged memberships\n    # are the *effective* membership regardless of wierd configurations.\n    for anom in self.FindAnomalies():\n      yield anom\n    # Then add shadow group members to the group membership.\n    self.MergeMembers()\n    for group in itervalues(self.entry):\n      yield group", "response": "Parse a linux system group and gshadow files."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef ParseShadowEntry(self, line):\n    fields = (\"login\", \"passwd\", \"last_change\", \"min_age\", \"max_age\",\n              \"warn_time\", \"inactivity\", \"expire\", \"reserved\")\n    if line:\n      rslt = dict(zip(fields, line.split(\":\")))\n      pw_entry = self.shadow.setdefault(rslt[\"login\"], rdf_client.PwEntry())\n      pw_entry.store = self.shadow_store\n      pw_entry.hash_type = self.GetHashType(rslt[\"passwd\"])\n      # Tread carefully here in case these values aren't set.\n      last_change = rslt.get(\"last_change\")\n      if last_change:\n        pw_entry.age = int(last_change)\n      max_age = rslt.get(\"max_age\")\n      if max_age:\n        pw_entry.max_age = int(max_age)", "response": "Parses a shadow entry."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nprocess the passwd entry fields and primary group memberships.", "response": "def ParsePasswdEntry(self, line):\n    \"\"\"Process the passwd entry fields and primary group memberships.\"\"\"\n    fields = (\"uname\", \"passwd\", \"uid\", \"gid\", \"fullname\", \"homedir\", \"shell\")\n    if line:\n      rslt = dict(zip(fields, line.split(\":\")))\n      user = self.entry.setdefault(rslt[\"uname\"], rdf_client.User())\n      user.username = rslt[\"uname\"]\n      user.pw_entry.store = self.GetPwStore(rslt[\"passwd\"])\n      if user.pw_entry.store == self.base_store:\n        user.pw_entry.hash_type = self.GetHashType(rslt[\"passwd\"])\n      # If the passwd file contains NIS entries they may not have uid/gid set.\n      if rslt[\"uid\"]:\n        user.uid = int(rslt[\"uid\"])\n      if rslt[\"gid\"]:\n        user.gid = int(rslt[\"gid\"])\n      user.homedir = rslt[\"homedir\"]\n      user.shell = rslt[\"shell\"]\n      user.full_name = rslt[\"fullname\"]\n      # Map uid numbers to detect duplicates.\n      uids = self.uids.setdefault(user.uid, set())\n      uids.add(user.username)\n      # Map primary group memberships to populate memberships.\n      gid = self.gids.setdefault(user.gid, set())\n      gid.add(user.username)"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nunifies members of a group and accounts with the group as primary gid.", "response": "def _Members(self, group):\n    \"\"\"Unify members of a group and accounts with the group as primary gid.\"\"\"\n    group.members = set(group.members).union(self.gids.get(group.gid, []))\n    return group"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nadd aggregate group membership from group gshadow and passwd.", "response": "def AddGroupMemberships(self):\n    \"\"\"Adds aggregate group membership from group, gshadow and passwd.\"\"\"\n    self.groups = {g.name: self._Members(g) for g in itervalues(self.groups)}\n    # Map the groups a user is a member of, irrespective of primary/extra gid.\n    for g in itervalues(self.groups):\n      for user in g.members:\n        membership = self.memberships.setdefault(user, set())\n        membership.add(g.gid)\n    # Now add the completed membership to the user account.\n    for user in itervalues(self.entry):\n      user.gids = self.memberships.get(user.username)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef FindAnomalies(self):\n    # Find anomalous group entries.\n    findings = []\n    group_entries = {g.gid for g in itervalues(self.groups)}\n    for gid in set(self.gids) - group_entries:\n      undefined = \",\".join(self.gids.get(gid, []))\n      findings.append(\n          \"gid %d assigned without /etc/groups entry: %s\" % (gid, undefined))\n    if findings:\n      yield self._Anomaly(\"Accounts with invalid gid.\", findings)\n\n    # Find any shared user IDs.\n    findings = []\n    for uid, names in iteritems(self.uids):\n      if len(names) > 1:\n        findings.append(\"uid %d assigned to multiple accounts: %s\" %\n                        (uid, \",\".join(sorted(names))))\n    if findings:\n      yield self._Anomaly(\"Accounts with shared uid.\", findings)\n\n    # Find privileged groups with unusual members.\n    findings = []\n    root_grp = self.groups.get(\"root\")\n    if root_grp is not None:\n      root_members = sorted([m for m in root_grp.members if m != \"root\"])\n      if root_members:\n        findings.append(\"Accounts in 'root' group: %s\" % \",\".join(root_members))\n    if findings:\n      yield self._Anomaly(\"Privileged group with unusual members.\", findings)\n\n    # Find accounts without passwd/shadow entries.\n    diffs = self.MemberDiff(self.entry, \"passwd\", self.shadow, \"shadow\")\n    if diffs:\n      yield self._Anomaly(\"Mismatched passwd and shadow files.\", diffs)", "response": "Identify anomalies in the password shadow and group data."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef AddPassword(self, fileset):\n    passwd = fileset.get(\"/etc/passwd\")\n    if passwd:\n      self._ParseFile(passwd, self.ParsePasswdEntry)\n    else:\n      logging.debug(\"No /etc/passwd file.\")", "response": "Add the passwd entries to the shadow store."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nadds the shadow entries to the shadow store.", "response": "def AddShadow(self, fileset):\n    \"\"\"Add the shadow entries to the shadow store.\"\"\"\n    shadow = fileset.get(\"/etc/shadow\")\n    if shadow:\n      self._ParseFile(shadow, self.ParseShadowEntry)\n    else:\n      logging.debug(\"No /etc/shadow file.\")"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef ParseFileset(self, fileset=None):\n    self.AddPassword(fileset)\n    self.AddShadow(fileset)\n    self.ReconcileShadow(self.shadow_store)\n    # Get group memberships using the files that were already collected.\n    # Separate out groups and anomalies.\n    for rdf in LinuxSystemGroupParser().ParseFileset(fileset):\n      if isinstance(rdf, rdf_client.Group):\n        self.groups[rdf.name] = rdf\n      else:\n        yield rdf\n    self.AddGroupMemberships()\n    for user in itervalues(self.entry):\n      yield user\n    for grp in itervalues(self.groups):\n      yield grp\n    for anom in self.FindAnomalies():\n      yield anom", "response": "Parse a linux system login fileset and populate the user and group memberships."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _ExpandPath(self, target, vals, paths):\n    if target not in self._TARGETS:\n      return\n    expanded = []\n    for val in vals:\n      # Null entries specify the current directory, so :a::b:c: is equivalent\n      # to .:a:.:b:c:.\n      shellvar = self._SHELLVAR_RE.match(val)\n      if not val:\n        expanded.append(\".\")\n      elif shellvar:\n        # The value may actually be in braces as well. Always convert to upper\n        # case so we deal with stuff like lowercase csh path.\n        existing = paths.get(shellvar.group(1).upper())\n        if existing:\n          expanded.extend(existing)\n        else:\n          expanded.append(val)\n      else:\n        expanded.append(val)\n    paths[target] = expanded", "response": "Extract path information interpolating current path values as needed."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _ParseShVariables(self, lines):\n    paths = {}\n    for line in lines:\n      for entry in line:\n        if \"=\" in entry:\n          # Pad out the list so that it's always 2 elements, even if the split\n          # failed.\n          target, vals = (entry.split(\"=\", 1) + [\"\"])[:2]\n          if vals:\n            path_vals = vals.split(\":\")\n          else:\n            path_vals = []\n          self._ExpandPath(target, path_vals, paths)\n        elif entry not in self._SH_CONTINUATION:\n          # Stop processing the line unless the entry might allow paths to still\n          # be set, e.g.\n          #   reserved words: \"export\"\n          #   conditions: { PATH=VAL } && PATH=:$PATH || PATH=.\n          break\n    return paths", "response": "Parses the environment variables and paths from the line."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _ParseCshVariables(self, lines):\n    paths = {}\n    for line in lines:\n      if len(line) < 2:\n        continue\n      action = line[0]\n      if action == \"setenv\":\n        target = line[1]\n        path_vals = []\n        if line[2:]:\n          path_vals = line[2].split(\":\")\n        self._ExpandPath(target, path_vals, paths)\n      elif action == \"set\":\n        set_vals = self._CSH_SET_RE.search(\" \".join(line[1:]))\n        if set_vals:\n          target, vals = set_vals.groups()\n          # Automatically exported to ENV vars.\n          if target in (\"path\", \"term\", \"user\"):\n            target = target.upper()\n          path_vals = vals.split()\n          self._ExpandPath(target, path_vals, paths)\n    return paths", "response": "Parses the environment variables and path values from a list of lines."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nparsing the file and returns a set of attributed values.", "response": "def Parse(self, stat, file_obj, knowledge_base):\n    \"\"\"Identifies the paths set within a file.\n\n    Expands paths within the context of the file, but does not infer fully\n    expanded paths from external states. There are plenty of cases where path\n    attributes are unresolved, e.g. sourcing other files.\n\n    Lines are not handled literally. A field parser is used to:\n    - Break lines with multiple distinct statements into separate lines (e.g.\n      lines with a ';' separating stanzas.\n    - Strip out comments.\n    - Handle line continuations to capture multi-line configurations into one\n      statement.\n\n    Args:\n      stat: statentry\n      file_obj: VFSFile\n      knowledge_base: unused\n\n    Yields:\n      An attributed dict for each env vars. 'name' contains the path name, and\n      'vals' contains its vals.\n    \"\"\"\n    _ = knowledge_base\n    lines = self.parser.ParseEntries(utils.ReadFileBytesAsUnicode(file_obj))\n    if os.path.basename(stat.pathspec.path) in self._CSH_FILES:\n      paths = self._ParseCshVariables(lines)\n    else:\n      paths = self._ParseShVariables(lines)\n    for path_name, path_vals in iteritems(paths):\n      yield rdf_protodict.AttributedDict(\n          config=stat.pathspec.path, name=path_name, vals=path_vals)"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nseeks to an offset in the file.", "response": "def Seek(self, offset, whence=os.SEEK_SET):\n    \"\"\"Seek to an offset in the file.\"\"\"\n    if whence == os.SEEK_SET:\n      self.offset = offset\n    elif whence == os.SEEK_CUR:\n      self.offset += offset\n    elif whence == os.SEEK_END:\n      self.offset = self.size + offset\n    else:\n      raise ValueError(\"Illegal whence value %s\" % whence)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nguessing a container from the current object.", "response": "def OpenAsContainer(self):\n    \"\"\"Guesses a container from the current object.\"\"\"\n    if self.IsDirectory():\n      return self\n\n    # TODO(user): Add support for more containers here (e.g. registries, zip\n    # files etc).\n    else:  # For now just guess TSK.\n      tsk_handler = self._handlers[rdf_paths.PathSpec.PathType.TSK]\n      tsk_pathspec = rdf_paths.PathSpec(\n          path=\"/\", pathtype=rdf_paths.PathSpec.PathType.TSK)\n      return tsk_handler(\n          base_fd=self,\n          handlers=self._handlers,\n          pathspec=tsk_pathspec,\n          progress_callback=self.progress_callback)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef MatchBestComponentName(self, component):\n    fd = self.OpenAsContainer()\n\n    # Adjust the component casing\n    file_listing = set(fd.ListNames())\n\n    # First try an exact match\n    if component not in file_listing:\n      # Now try to match lower case\n      lower_component = component.lower()\n      for x in file_listing:\n        if lower_component == x.lower():\n          component = x\n          break\n\n    if fd.supported_pathtype != self.pathspec.pathtype:\n      new_pathspec = rdf_paths.PathSpec(\n          path=component, pathtype=fd.supported_pathtype)\n    else:\n      new_pathspec = self.pathspec.last.Copy()\n      new_pathspec.path = component\n\n    return new_pathspec", "response": "Returns the name of the component which matches the best our base listing."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nopen a file object for the given component.", "response": "def Open(cls, fd, component, handlers, pathspec=None, progress_callback=None):\n    \"\"\"Try to correct the casing of component.\n\n    This method is called when we failed to open the component directly. We try\n    to transform the component into something which is likely to work.\n\n    In this implementation, we correct the case of the component until we can\n    not open the path any more.\n\n    Args:\n      fd: The base fd we will use.\n      component: The component we should open.\n      handlers: A mapping from rdf_paths.PathSpec.PathType to classes\n        implementing VFSHandler.\n      pathspec: The rest of the pathspec object.\n      progress_callback: A callback to indicate that the open call is still\n        working but needs more time.\n\n    Returns:\n      A file object.\n\n    Raises:\n      IOError: If nothing could be opened still.\n    \"\"\"\n    # The handler for this component\n    try:\n      handler = handlers[component.pathtype]\n    except KeyError:\n      raise UnsupportedHandlerError(component.pathtype)\n\n    # We will not do any case folding unless requested.\n    if component.path_options == rdf_paths.PathSpec.Options.CASE_LITERAL:\n      return handler(base_fd=fd, pathspec=component, handlers=handlers)\n\n    path_components = client_utils.LocalPathToCanonicalPath(component.path)\n    path_components = [\"/\"] + list(filter(None, path_components.split(\"/\")))\n    for i, path_component in enumerate(path_components):\n      try:\n        if fd:\n          new_pathspec = fd.MatchBestComponentName(path_component)\n        else:\n          new_pathspec = component\n          new_pathspec.path = path_component\n\n        # The handler for this component\n        try:\n          handler = handlers[new_pathspec.pathtype]\n        except KeyError:\n          raise UnsupportedHandlerError(new_pathspec.pathtype)\n\n        fd = handler(\n            base_fd=fd,\n            handlers=handlers,\n            pathspec=new_pathspec,\n            progress_callback=progress_callback)\n      except IOError as e:\n        # Can not open the first component, we must raise here.\n        if i <= 1:\n          raise IOError(\"File not found: {}\".format(component))\n\n        # Do not try to use TSK to open a not-found registry entry, fail\n        # instead. Using TSK would lead to confusing error messages, hiding\n        # the fact that the Registry entry is simply not there.\n        if component.pathtype == rdf_paths.PathSpec.PathType.REGISTRY:\n          raise IOError(\"Registry entry not found: {}\".format(e))\n\n        # Insert the remaining path at the front of the pathspec.\n        pathspec.Insert(\n            0,\n            path=utils.JoinPath(*path_components[i:]),\n            pathtype=rdf_paths.PathSpec.PathType.TSK)\n        break\n\n    return fd"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef StaticAdd(cls,\n                collection_urn,\n                rdf_value,\n                timestamp=None,\n                suffix=None,\n                mutation_pool=None):\n    \"\"\"Adds an rdf value to a collection.\n\n    Adds an rdf value to a collection. Does not require that the collection be\n    open. NOTE: The caller is responsible for ensuring that the collection\n    exists and is of the correct type.\n\n    Args:\n      collection_urn: The urn of the collection to add to.\n      rdf_value: The rdf value to add to the collection. If this value is not\n        GrrMessage, it will be wrapped into GrrMessage (later when collection is\n        iterated, this value will still be returned wrapped in GrrMessage).\n      timestamp: The timestamp (in microseconds) to store the rdf value at.\n        Defaults to the current time.\n      suffix: A 'fractional timestamp' suffix to reduce the chance of\n        collisions. Defaults to a random number.\n      mutation_pool: A MutationPool object to write to.\n\n    Returns:\n      The pair (timestamp, suffix) which identifies the value within the\n      collection.\n\n    Raises:\n      ValueError: rdf_value has unexpected type.\n\n    \"\"\"\n    if rdf_value is None:\n      raise ValueError(\"Can't add None to MultiTypeCollection\")\n    if mutation_pool is None:\n      raise ValueError(\"Mutation pool can't be none.\")\n\n    if not isinstance(rdf_value, rdf_flows.GrrMessage):\n      rdf_value = rdf_flows.GrrMessage(payload=rdf_value)\n\n    value_type = rdf_value.args_rdf_name or rdf_flows.GrrMessage.__name__\n\n    # In order to make this fast, we never actually generate the\n    # subcollections, we just use them. This means that we cannot use\n    # ListChildren to get all the items stored in this\n    # MultiTypeCollection.\n    subpath = collection_urn.Add(value_type)\n    sequential_collection.GrrMessageCollection.StaticAdd(\n        subpath,\n        rdf_value,\n        timestamp=timestamp,\n        suffix=suffix,\n        mutation_pool=mutation_pool)\n\n    mutation_pool.CollectionAddStoredTypeIndex(collection_urn, value_type)", "response": "Adds an rdf value to a multi - type collection."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nscan for stored records of a certain type.", "response": "def ScanByType(self,\n                 type_name,\n                 after_timestamp=None,\n                 include_suffix=False,\n                 max_records=None):\n    \"\"\"Scans for stored records.\n\n    Scans through the collection, returning stored values ordered by timestamp.\n\n    Args:\n      type_name: Type of the records to scan.\n      after_timestamp: If set, only returns values recorded after timestamp.\n      include_suffix: If true, the timestamps returned are pairs of the form\n        (micros_since_epoc, suffix) where suffix is a 24 bit random refinement\n        to avoid collisions. Otherwise only micros_since_epoc is returned.\n      max_records: The maximum number of records to return. Defaults to\n        unlimited.\n\n    Yields:\n      Pairs (timestamp, rdf_value), indicating that rdf_value was stored at\n      timestamp.\n\n    \"\"\"\n    sub_collection_urn = self.collection_id.Add(type_name)\n    sub_collection = sequential_collection.GrrMessageCollection(\n        sub_collection_urn)\n    for item in sub_collection.Scan(\n        after_timestamp=after_timestamp,\n        include_suffix=include_suffix,\n        max_records=max_records):\n      yield item"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef StartHunt(args=None, runner_args=None, token=None, **kwargs):\n\n  # If no token is specified, raise.\n  if not token:\n    raise access_control.UnauthorizedAccess(\"A token must be specified.\")\n\n  # Build the runner args from the keywords.\n  if runner_args is None:\n    runner_args = rdf_hunts.HuntRunnerArgs()\n\n  flow.FilterArgsFromSemanticProtobuf(runner_args, kwargs)\n\n  # Is the required hunt a known hunt?\n  hunt_cls = GRRHunt.classes.get(runner_args.hunt_name)\n  if not hunt_cls or not issubclass(hunt_cls, GRRHunt):\n    raise RuntimeError(\"Unable to locate hunt %s\" % runner_args.hunt_name)\n\n  # Make a new hunt object and initialize its runner.\n  hunt_obj = aff4.FACTORY.Create(None, hunt_cls, mode=\"w\", token=token)\n\n  # Hunt is called using keyword args. We construct an args proto from the\n  # kwargs.\n  if hunt_obj.args_type and args is None:\n    args = hunt_obj.args_type()\n    flow.FilterArgsFromSemanticProtobuf(args, kwargs)\n\n  if hunt_obj.args_type and not isinstance(args, hunt_obj.args_type):\n    raise RuntimeError(\"Hunt args must be instance of %s\" % hunt_obj.args_type)\n\n  if kwargs:\n    raise type_info.UnknownArg(\"Unknown parameters to StartHunt: %s\" % kwargs)\n\n  # Store the hunt args.\n  hunt_obj.args = args\n  hunt_obj.runner_args = runner_args\n\n  # Hunts are always created in the paused state. The runner method Start\n  # should be called to start them.\n  hunt_obj.Set(hunt_obj.Schema.STATE(\"PAUSED\"))\n\n  runner = hunt_obj.CreateRunner(runner_args=runner_args)\n  # Allow the hunt to do its own initialization.\n  runner.RunStateMethod(\"Start\")\n\n  hunt_obj.Flush()\n\n  try:\n    flow_name = args.flow_runner_args.flow_name\n  except AttributeError:\n    flow_name = \"\"\n\n  event = rdf_events.AuditEvent(\n      user=token.username,\n      action=\"HUNT_CREATED\",\n      urn=hunt_obj.urn,\n      flow_name=flow_name,\n      description=runner_args.description)\n  events_lib.Events.PublishEvent(\"Audit\", event, token=token)\n\n  return hunt_obj", "response": "This class method creates new hunts."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nprocesses the completed requests and responses for this hunt.", "response": "def ProcessCompletedRequests(self, notification, thread_pool):\n    \"\"\"Go through the list of requests and process the completed ones.\n\n    We take a snapshot in time of all requests and responses for this hunt. We\n    then process as many completed requests as possible. If responses are not\n    quite here we leave it for next time.\n\n    Args:\n      notification: The notification object that triggered this processing.\n      thread_pool: The thread pool to process the responses on.\n    \"\"\"\n    # First ensure that client messages are all removed. NOTE: We make a new\n    # queue manager here because we want only the client messages to be removed\n    # ASAP. This must happen before we actually run the hunt to ensure the\n    # client requests are removed from the client queues.\n    with queue_manager.QueueManager(token=self.token) as manager:\n      for request, _ in manager.FetchCompletedRequests(\n          self.session_id, timestamp=(0, notification.timestamp)):\n        # Requests which are not destined to clients have no embedded request\n        # message.\n        if request.HasField(\"request\"):\n          manager.DeQueueClientRequest(request.request)\n\n    processing = []\n    while True:\n      try:\n        # Here we only care about completed requests - i.e. those requests with\n        # responses followed by a status message.\n        for request, responses in self.queue_manager.FetchCompletedResponses(\n            self.session_id, timestamp=(0, notification.timestamp)):\n\n          if request.id == 0 or not responses:\n            continue\n\n          # Do we have all the responses here? This can happen if some of the\n          # responses were lost.\n          if len(responses) != responses[-1].response_id:\n            # If we can retransmit do so. Note, this is different from the\n            # automatic retransmission facilitated by the task scheduler (the\n            # Task.task_ttl field) which would happen regardless of these.\n            if request.transmission_count < 5:\n              stats_collector_instance.Get().IncrementCounter(\n                  \"grr_request_retransmission_count\")\n              request.transmission_count += 1\n              self.QueueRequest(request)\n            break\n\n          # If we get here its all good - run the hunt.\n          self.hunt_obj.HeartBeat()\n          self._Process(\n              request, responses, thread_pool=thread_pool, events=processing)\n\n          # At this point we have processed this request - we can remove it and\n          # its responses from the queue.\n          self.queue_manager.DeleteRequest(request)\n          self.context.next_processed_request += 1\n\n        # We are done here.\n        return\n\n      except queue_manager.MoreDataException:\n        # Join any threads.\n        for event in processing:\n          event.wait()\n\n        # We did not read all the requests/responses in this run in order to\n        # keep a low memory footprint and have to make another pass.\n        self.FlushMessages()\n        self.hunt_obj.Flush()\n        continue\n\n      finally:\n        # Join any threads.\n        for event in processing:\n          event.wait()"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef RunStateMethod(self,\n                     method_name,\n                     request=None,\n                     responses=None,\n                     event=None,\n                     direct_response=None):\n    \"\"\"Completes the request by calling the state method.\n\n    Args:\n      method_name: The name of the state method to call.\n      request: A RequestState protobuf.\n      responses: A list of GrrMessages responding to the request.\n      event: A threading.Event() instance to signal completion of this request.\n      direct_response: A flow.Responses() object can be provided to avoid\n        creation of one.\n    \"\"\"\n    client_id = None\n    try:\n      self.context.current_state = method_name\n      if request and responses:\n        client_id = request.client_id or self.runner_args.client_id\n        logging.debug(\"%s Running %s with %d responses from %s\",\n                      self.session_id, method_name, len(responses), client_id)\n\n      else:\n        logging.debug(\"%s Running state method %s\", self.session_id,\n                      method_name)\n\n      # Extend our lease if needed.\n      self.hunt_obj.HeartBeat()\n      try:\n        method = getattr(self.hunt_obj, method_name)\n      except AttributeError:\n        raise flow_runner.FlowRunnerError(\n            \"Flow %s has no state method %s\" %\n            (self.hunt_obj.__class__.__name__, method_name))\n\n      if direct_response:\n        method(direct_response)\n      elif method_name == \"Start\":\n        method()\n      else:\n        # Prepare a responses object for the state method to use:\n        responses = flow_responses.Responses.FromLegacyResponses(\n            request=request, responses=responses)\n\n        if responses.status:\n          self.SaveResourceUsage(request.client_id, responses.status)\n\n        stats_collector_instance.Get().IncrementCounter(\"grr_worker_states_run\")\n\n        method(responses)\n\n    # We don't know here what exceptions can be thrown in the flow but we have\n    # to continue. Thus, we catch everything.\n    except Exception as e:  # pylint: disable=broad-except\n\n      # TODO(user): Deprecate in favor of 'flow_errors'.\n      stats_collector_instance.Get().IncrementCounter(\"grr_flow_errors\")\n\n      stats_collector_instance.Get().IncrementCounter(\n          \"flow_errors\", fields=[self.hunt_obj.Name()])\n      logging.exception(\"Hunt %s raised %s.\", self.session_id, e)\n\n      self.Error(traceback.format_exc(), client_id=client_id)\n\n    finally:\n      if event:\n        event.set()", "response": "Runs the state method specified by method_name."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef Publish(self, event_name, msg, delay=0):\n    events_lib.Events.PublishEvent(event_name, msg, delay=delay)", "response": "Sends the message to the event listeners."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ngets current CPU limit for subflows.", "response": "def _GetSubFlowCPULimit(self):\n    \"\"\"Get current CPU limit for subflows.\"\"\"\n\n    subflow_cpu_limit = None\n\n    if self.runner_args.per_client_cpu_limit:\n      subflow_cpu_limit = self.runner_args.per_client_cpu_limit\n\n    if self.runner_args.cpu_limit:\n      cpu_usage_data = self.context.client_resources.cpu_usage\n      remaining_cpu_quota = (\n          self.runner_args.cpu_limit - cpu_usage_data.user_cpu_time -\n          cpu_usage_data.system_cpu_time)\n      if subflow_cpu_limit is None:\n        subflow_cpu_limit = remaining_cpu_quota\n      else:\n        subflow_cpu_limit = min(subflow_cpu_limit, remaining_cpu_quota)\n\n      if subflow_cpu_limit == 0:\n        raise RuntimeError(\"Out of CPU quota.\")\n\n    return subflow_cpu_limit"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _GetSubFlowNetworkLimit(self):\n\n    subflow_network_limit = None\n\n    if self.runner_args.per_client_network_limit_bytes:\n      subflow_network_limit = self.runner_args.per_client_network_limit_bytes\n\n    if self.runner_args.network_bytes_limit:\n      remaining_network_quota = (\n          self.runner_args.network_bytes_limit -\n          self.context.network_bytes_sent)\n      if subflow_network_limit is None:\n        subflow_network_limit = remaining_network_quota\n      else:\n        subflow_network_limit = min(subflow_network_limit,\n                                    remaining_network_quota)\n\n    return subflow_network_limit", "response": "Get current network limit for subflows."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ncreate a new flow and send its responses to a state. This creates a new flow. The flow may send back many responses which will be queued by the framework until the flow terminates. The final status message will cause the entire transaction to be committed to the specified state. Args: flow_name: The name of the flow to invoke. next_state: The state in this flow, that responses to this message should go to. request_data: Any dict provided here will be available in the RequestState protobuf. The Responses object maintains a reference to this protobuf for use in the execution of the state method. (so you can access this data by responses.request). There is no format mandated on this data but it may be a serialized protobuf. client_id: If given, the flow is started for this client. base_session_id: A URN which will be used to build a URN. **kwargs: Arguments for the child flow. Returns: The URN of the child flow which was created. Raises: RuntimeError: In case of no cpu quota left to start more clients.", "response": "def _CallFlowLegacy(self,\n                      flow_name=None,\n                      next_state=None,\n                      request_data=None,\n                      client_id=None,\n                      base_session_id=None,\n                      **kwargs):\n    \"\"\"Creates a new flow and send its responses to a state.\n\n    This creates a new flow. The flow may send back many responses which will be\n    queued by the framework until the flow terminates. The final status message\n    will cause the entire transaction to be committed to the specified state.\n\n    Args:\n       flow_name: The name of the flow to invoke.\n       next_state: The state in this flow, that responses to this message should\n         go to.\n       request_data: Any dict provided here will be available in the\n         RequestState protobuf. The Responses object maintains a reference to\n         this protobuf for use in the execution of the state method. (so you can\n         access this data by responses.request). There is no format mandated on\n         this data but it may be a serialized protobuf.\n       client_id: If given, the flow is started for this client.\n       base_session_id: A URN which will be used to build a URN.\n       **kwargs: Arguments for the child flow.\n\n    Returns:\n       The URN of the child flow which was created.\n\n    Raises:\n       RuntimeError: In case of no cpu quota left to start more clients.\n    \"\"\"\n    client_id = client_id or self.runner_args.client_id\n\n    # We prepare a request state, and add it to our queue - any\n    # responses from the child flow will return to the request state\n    # and the stated next_state. Note however, that there is no\n    # client_id or actual request message here because we directly\n    # invoke the child flow rather than queue anything for it.\n    state = rdf_flow_runner.RequestState(\n        id=self.GetNextOutboundId(),\n        session_id=utils.SmartUnicode(self.session_id),\n        client_id=client_id,\n        next_state=next_state,\n        response_count=0)\n\n    if request_data:\n      state.data = rdf_protodict.Dict().FromDict(request_data)\n\n    # Pass our logs collection urn to the flow object.\n    logs_urn = self.hunt_obj.logs_collection_urn\n\n    # If we were called with write_intermediate_results, propagate down to\n    # child flows.  This allows write_intermediate_results to be set to True\n    # either at the top level parent, or somewhere in the middle of\n    # the call chain.\n    write_intermediate = kwargs.pop(\"write_intermediate_results\", False)\n\n    # Create the new child flow but do not notify the user about it.\n    child_urn = self.hunt_obj.StartAFF4Flow(\n        base_session_id=base_session_id or self.session_id,\n        client_id=client_id,\n        cpu_limit=self._GetSubFlowCPULimit(),\n        flow_name=flow_name,\n        logs_collection_urn=logs_urn,\n        network_bytes_limit=self._GetSubFlowNetworkLimit(),\n        notify_to_user=False,\n        parent_flow=self.hunt_obj,\n        queue=self.runner_args.queue,\n        request_state=state,\n        sync=False,\n        token=self.token,\n        write_intermediate_results=write_intermediate,\n        **kwargs)\n\n    self.QueueRequest(state)\n\n    return child_urn"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncreating a new flow and sends its responses to a state.", "response": "def _CallFlowRelational(self,\n                          flow_name=None,\n                          args=None,\n                          runner_args=None,\n                          client_id=None,\n                          **kwargs):\n    \"\"\"Creates a new flow and send its responses to a state.\n\n    This creates a new flow. The flow may send back many responses which will be\n    queued by the framework until the flow terminates. The final status message\n    will cause the entire transaction to be committed to the specified state.\n\n    Args:\n       flow_name: The name of the flow to invoke.\n       args: Flow arguments.\n       runner_args: Flow runner arguments.\n       client_id: If given, the flow is started for this client.\n       **kwargs: Arguments for the child flow.\n\n    Returns:\n       The URN of the child flow which was created.\n\n    Raises:\n       RuntimeError: In case of no cpu quota left to start more clients.\n    \"\"\"\n    if isinstance(client_id, rdfvalue.RDFURN):\n      client_id = client_id.Basename()\n\n    if flow_name is None and runner_args is not None:\n      flow_name = runner_args.flow_name\n\n    flow_cls = registry.FlowRegistry.FlowClassByName(flow_name)\n\n    flow_id = flow.StartFlow(\n        client_id=client_id,\n        creator=self.hunt_obj.creator,\n        cpu_limit=self._GetSubFlowCPULimit(),\n        network_bytes_limit=self._GetSubFlowNetworkLimit(),\n        flow_cls=flow_cls,\n        flow_args=args,\n        parent_hunt_id=self.hunt_obj.urn.Basename(),\n        **kwargs)\n\n    return rdfvalue.RDFURN(client_id).Add(\"flows\").Add(flow_id)"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncreates a new flow and send its responses to a state. This creates a new flow. The flow may send back many responses which will be queued by the framework until the flow terminates. The final status message will cause the entire transaction to be committed to the specified state. Args: flow_name: The name of the flow to invoke. next_state: The state in this flow, that responses to this message should go to. request_data: Any dict provided here will be available in the RequestState protobuf. The Responses object maintains a reference to this protobuf for use in the execution of the state method. (so you can access this data by responses.request). There is no format mandated on this data but it may be a serialized protobuf. client_id: If given, the flow is started for this client. base_session_id: A URN which will be used to build a URN. **kwargs: Arguments for the child flow. Returns: The URN of the child flow which was created. Raises: RuntimeError: In case of no cpu quota left to start more clients.", "response": "def CallFlow(self,\n               flow_name=None,\n               next_state=None,\n               request_data=None,\n               client_id=None,\n               base_session_id=None,\n               **kwargs):\n    \"\"\"Creates a new flow and send its responses to a state.\n\n    This creates a new flow. The flow may send back many responses which will be\n    queued by the framework until the flow terminates. The final status message\n    will cause the entire transaction to be committed to the specified state.\n\n    Args:\n       flow_name: The name of the flow to invoke.\n       next_state: The state in this flow, that responses to this message should\n         go to.\n       request_data: Any dict provided here will be available in the\n         RequestState protobuf. The Responses object maintains a reference to\n         this protobuf for use in the execution of the state method. (so you can\n         access this data by responses.request). There is no format mandated on\n         this data but it may be a serialized protobuf.\n       client_id: If given, the flow is started for this client.\n       base_session_id: A URN which will be used to build a URN.\n       **kwargs: Arguments for the child flow.\n\n    Returns:\n       The URN of the child flow which was created.\n\n    Raises:\n       RuntimeError: In case of no cpu quota left to start more clients.\n    \"\"\"\n    if data_store.RelationalDBEnabled():\n      if request_data is not None:\n        raise ValueError(\"Hunt's CallFlow does not support 'request_data' arg \"\n                         \"when REL_DB is enabled.\")\n\n      return self._CallFlowRelational(\n          flow_name=flow_name, client_id=client_id, **kwargs)\n    else:\n      return self._CallFlowLegacy(\n          flow_name=flow_name,\n          next_state=next_state,\n          request_data=request_data,\n          client_id=client_id,\n          base_session_id=base_session_id,\n          **kwargs)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _Process(self, request, responses, thread_pool=None, events=None):\n    # This function is called and runs within the main processing thread. We do\n    # not need to lock the hunt object while running in this method.\n    if request.next_state == \"AddClient\":\n      if not self.IsHuntStarted():\n        logging.debug(\n            \"Unable to start client %s on hunt %s which is in state %s\",\n            request.client_id, self.session_id,\n            self.hunt_obj.Get(self.hunt_obj.Schema.STATE))\n        return\n\n      # Get the client count.\n      client_count = int(\n          self.hunt_obj.Get(self.hunt_obj.Schema.CLIENT_COUNT, 0))\n\n      # Pause the hunt if we exceed the client limit.\n      if 0 < self.runner_args.client_limit <= client_count:\n        # Remove our rules from the foreman so we dont get more clients sent to\n        # this hunt. Hunt will be paused.\n        self.Pause()\n\n        # Ignore this client since it had gone over the limit.\n        return\n\n      # Update the client count.\n      self.hunt_obj.Set(self.hunt_obj.Schema.CLIENT_COUNT(client_count + 1))\n\n      # Add client to list of clients and optionally run it\n      # (if client_rate == 0).\n\n      self._AddClient(request.client_id)\n      return\n\n    if request.next_state == \"RegisterClient\":\n      state = self.hunt_obj.Get(self.hunt_obj.Schema.STATE)\n      # This allows the client limit to operate with a client rate. We still\n      # want clients to get registered for the hunt at times in the future.\n      # After they have been run, hunts only ever go into the paused state by\n      # hitting the client limit. If a user stops a hunt, it will go into the\n      # \"STOPPED\" state.\n      if state in [\"STARTED\", \"PAUSED\"]:\n        self._RegisterAndRunClient(request.client_id)\n      else:\n        logging.debug(\n            \"Not starting client %s on hunt %s which is not running: %s\",\n            request.client_id, self.session_id,\n            self.hunt_obj.Get(self.hunt_obj.Schema.STATE))\n      return\n\n    event = threading.Event()\n    events.append(event)\n    # In a hunt, all requests are independent and can be processed\n    # in separate threads.\n    thread_pool.AddTask(\n        target=self.RunStateMethod,\n        args=(request.next_state, request, responses, event),\n        name=\"Hunt processing\")", "response": "Processes the given request and responses."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nlog the message using the hunt s standard logging.", "response": "def Log(self, format_str, *args):\n    \"\"\"Logs the message using the hunt's standard logging.\n\n    Args:\n      format_str: Format string\n      *args: arguments to the format string\n\n    Raises:\n      RuntimeError: on parent missing logs_collection\n    \"\"\"\n    format_str = utils.SmartUnicode(format_str)\n\n    status = format_str\n    if args:\n      try:\n        # The status message is always in unicode\n        status = format_str % args\n      except TypeError:\n        logging.error(\n            \"Tried to log a format string with the wrong number \"\n            \"of arguments: %s\", format_str)\n\n    logging.info(\"%s: %s\", self.session_id, status)\n\n    self.context.status = utils.SmartUnicode(status)\n\n    log_entry = rdf_flows.FlowLog(\n        client_id=None,\n        urn=self.session_id,\n        flow_name=self.hunt_obj.__class__.__name__,\n        log_message=status)\n    logs_collection_urn = self.hunt_obj.logs_collection_urn\n    with data_store.DB.GetMutationPool() as pool:\n      grr_collections.LogCollection.StaticAdd(\n          logs_collection_urn, log_entry, mutation_pool=pool)"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nlogs an error for a client but does not terminate the hunt.", "response": "def Error(self, backtrace, client_id=None):\n    \"\"\"Logs an error for a client but does not terminate the hunt.\"\"\"\n    logging.error(\"Hunt Error: %s\", backtrace)\n    self.hunt_obj.LogClientError(client_id, backtrace=backtrace)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nupdate the resource usage of the hunt.", "response": "def SaveResourceUsage(self, client_id, status):\n    \"\"\"Update the resource usage of the hunt.\"\"\"\n    # Per client stats.\n    self.hunt_obj.ProcessClientResourcesStats(client_id, status)\n    # Overall hunt resource usage.\n    self.UpdateProtoResources(status)"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef InitializeContext(self, args):\n    if args is None:\n      args = rdf_hunts.HuntRunnerArgs()\n\n    context = rdf_hunts.HuntContext(\n        create_time=rdfvalue.RDFDatetime.Now(),\n        creator=self.token.username,\n        duration=args.expiry_time,\n        start_time=rdfvalue.RDFDatetime.Now(),\n        usage_stats=rdf_stats.ClientResourcesStats())\n\n    return context", "response": "Initializes the hunt context."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef GetNewSessionID(self, **_):\n    return rdfvalue.SessionID(base=\"aff4:/hunts\", queue=self.runner_args.queue)", "response": "Returns a random integer session id for this hunt."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef Start(self):\n    # We are already running.\n    if self.hunt_obj.Get(self.hunt_obj.Schema.STATE) == \"STARTED\":\n      return\n\n    # Determine when this hunt will expire.\n    self.context.duration = self.runner_args.expiry_time\n\n    # When the next client can be scheduled. Implements gradual client\n    # recruitment rate according to the client_rate.\n    self.context.next_client_due = rdfvalue.RDFDatetime.Now()\n\n    self._CreateAuditEvent(\"HUNT_STARTED\")\n\n    # Start the hunt.\n    self.hunt_obj.Set(self.hunt_obj.Schema.STATE(\"STARTED\"))\n    self.hunt_obj.Flush()\n\n    if self.runner_args.add_foreman_rules:\n      self._AddForemanRule()", "response": "This uploads the rules to the foreman and, thus, starts the hunt."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _AddForemanRule(self):\n    if data_store.RelationalDBEnabled():\n      # Relational DB uses ForemanCondition objects.\n      foreman_condition = foreman_rules.ForemanCondition(\n          creation_time=rdfvalue.RDFDatetime.Now(),\n          expiration_time=self.context.init_start_time + self.context.duration,\n          description=\"Hunt %s %s\" %\n          (self.session_id, self.runner_args.hunt_name),\n          client_rule_set=self.runner_args.client_rule_set,\n          hunt_id=self.session_id.Basename(),\n          hunt_name=self.runner_args.hunt_name)\n\n      # Make sure the rule makes sense.\n      foreman_condition.Validate()\n\n      data_store.REL_DB.WriteForemanRule(foreman_condition)\n    else:\n      foreman_rule = foreman_rules.ForemanRule(\n          created=rdfvalue.RDFDatetime.Now(),\n          # TODO: Hunt context should differentiate between initial\n          # and last start time (similarly to normal hunt objects).\n          expires=self.context.create_time + self.context.duration,\n          description=\"Hunt %s %s\" %\n          (self.session_id, self.runner_args.hunt_name),\n          client_rule_set=self.runner_args.client_rule_set)\n\n      foreman_rule.actions.Append(\n          hunt_id=self.session_id,\n          hunt_name=self.runner_args.hunt_name,\n          client_limit=self.runner_args.client_limit)\n\n      # Make sure the rule makes sense.\n      foreman_rule.Validate()\n\n      with aff4.FACTORY.Open(\n          \"aff4:/foreman\",\n          mode=\"rw\",\n          token=self.token,\n          aff4_type=aff4_grr.GRRForeman) as foreman:\n        rules = foreman.Get(\n            foreman.Schema.RULES, default=foreman.Schema.RULES())\n        rules.Append(foreman_rule)\n        foreman.Set(rules)", "response": "Adds a foreman rule for this hunt."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _RemoveForemanRule(self):\n    if data_store.RelationalDBEnabled():\n      data_store.REL_DB.RemoveForemanRule(hunt_id=self.session_id.Basename())\n      return\n\n    with aff4.FACTORY.Open(\n        \"aff4:/foreman\", mode=\"rw\", token=self.token) as foreman:\n      aff4_rules = foreman.Get(foreman.Schema.RULES)\n      aff4_rules = foreman.Schema.RULES(\n          # Remove those rules which fire off this hunt id.\n          [r for r in aff4_rules if r.hunt_id != self.session_id])\n      foreman.Set(aff4_rules)", "response": "Removes the foreman rule corresponding to this hunt."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _Complete(self):\n    self._RemoveForemanRule()\n    if \"w\" in self.hunt_obj.mode:\n      self.hunt_obj.Set(self.hunt_obj.Schema.STATE(\"COMPLETED\"))\n      self.hunt_obj.Flush()", "response": "Marks the hunt as completed."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\npausing the hunt (removes Foreman rules, does not touch expiry time).", "response": "def Pause(self):\n    \"\"\"Pauses the hunt (removes Foreman rules, does not touch expiry time).\"\"\"\n    if not self.IsHuntStarted():\n      return\n\n    self._RemoveForemanRule()\n\n    self.hunt_obj.Set(self.hunt_obj.Schema.STATE(\"PAUSED\"))\n    self.hunt_obj.Flush()\n\n    self._CreateAuditEvent(\"HUNT_PAUSED\")"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef Stop(self, reason=None):\n    self._RemoveForemanRule()\n    self.hunt_obj.Set(self.hunt_obj.Schema.STATE(\"STOPPED\"))\n    self.hunt_obj.Flush()\n\n    self._CreateAuditEvent(\"HUNT_STOPPED\")\n\n    if reason:\n      notification_lib.Notify(\n          self.token.username,\n          rdf_objects.UserNotification.Type.TYPE_HUNT_STOPPED, reason,\n          rdf_objects.ObjectReference(\n              reference_type=rdf_objects.ObjectReference.Type.HUNT,\n              hunt=rdf_objects.HuntReference(\n                  hunt_id=self.hunt_obj.urn.Basename())))", "response": "Cancels the hunt (removes Foreman rules, resets expiry time to 0)."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef IsHuntStarted(self):\n    state = self.hunt_obj.Get(self.hunt_obj.Schema.STATE)\n    if state != \"STARTED\":\n      return False\n\n    # Stop the hunt due to expiry.\n    if self.CheckExpiry():\n      return False\n\n    return True", "response": "Determines if this hunt is started."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef CallState(self,\n                messages=None,\n                next_state=\"\",\n                client_id=None,\n                request_data=None,\n                start_time=None):\n    \"\"\"This method is used to asynchronously schedule a new hunt state.\n\n    The state will be invoked in a later time and receive all the messages\n    we send.\n\n    Args:\n      messages: A list of rdfvalues to send. If the last one is not a GrrStatus,\n        we append an OK Status.\n      next_state: The state in this hunt to be invoked with the responses.\n      client_id: ClientURN to use in scheduled requests.\n      request_data: Any dict provided here will be available in the RequestState\n        protobuf. The Responses object maintains a reference to this protobuf\n        for use in the execution of the state method. (so you can access this\n        data by responses.request).\n      start_time: Schedule the state at this time. This delays notification and\n        messages for processing into the future.\n\n    Raises:\n      ValueError: on arguments error.\n    \"\"\"\n\n    if messages is None:\n      messages = []\n\n    if not next_state:\n      raise ValueError(\"next_state can't be empty.\")\n\n    # Now we construct a special response which will be sent to the hunt\n    # flow. Randomize the request_id so we do not overwrite other messages in\n    # the queue.\n    request_state = rdf_flow_runner.RequestState(\n        id=random.UInt32(),\n        session_id=self.context.session_id,\n        client_id=client_id,\n        next_state=next_state)\n\n    if request_data:\n      request_state.data = rdf_protodict.Dict().FromDict(request_data)\n\n    self.QueueRequest(request_state, timestamp=start_time)\n\n    # Add the status message if needed.\n    if not messages or not isinstance(messages[-1], rdf_flows.GrrStatus):\n      messages.append(rdf_flows.GrrStatus())\n\n    # Send all the messages\n    for i, payload in enumerate(messages):\n      if isinstance(payload, rdfvalue.RDFValue):\n        msg = rdf_flows.GrrMessage(\n            session_id=self.session_id,\n            request_id=request_state.id,\n            response_id=1 + i,\n            auth_state=rdf_flows.GrrMessage.AuthorizationState.AUTHENTICATED,\n            payload=payload,\n            type=rdf_flows.GrrMessage.Type.MESSAGE)\n\n        if isinstance(payload, rdf_flows.GrrStatus):\n          msg.type = rdf_flows.GrrMessage.Type.STATUS\n      else:\n        raise flow_runner.FlowRunnerError(\"Bad message %s of type %s.\" %\n                                          (payload, type(payload)))\n\n      self.QueueResponse(msg, timestamp=start_time)\n\n    # Notify the worker about it.\n    self.QueueNotification(session_id=self.session_id, timestamp=start_time)", "response": "This method is used to schedule a new state in a hunt."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef CreateRunner(self, **kw):\n    self.runner = HuntRunner(self, token=self.token, **kw)\n    return self.runner", "response": "Create a new runner."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nstart a new hunt for each client in the list of clients.", "response": "def StartClients(cls, hunt_id, client_ids, token=None):\n    \"\"\"This method is called by the foreman for each client it discovers.\n\n    Note that this function is performance sensitive since it is called by the\n    foreman for every client which needs to be scheduled.\n\n    Args:\n      hunt_id: The hunt to schedule.\n      client_ids: List of clients that should be added to the hunt.\n      token: An optional access token to use.\n    \"\"\"\n    token = token or access_control.ACLToken(username=\"Hunt\", reason=\"hunting\")\n\n    with queue_manager.QueueManager(token=token) as flow_manager:\n      for client_id in client_ids:\n        # Now we construct a special response which will be sent to the hunt\n        # flow. Randomize the request_id so we do not overwrite other messages\n        # in the queue.\n        state = rdf_flow_runner.RequestState(\n            id=random.UInt32(),\n            session_id=hunt_id,\n            client_id=client_id,\n            next_state=\"AddClient\")\n\n        # Queue the new request.\n        flow_manager.QueueRequest(state)\n\n        # Send a response.\n        msg = rdf_flows.GrrMessage(\n            session_id=hunt_id,\n            request_id=state.id,\n            response_id=1,\n            auth_state=rdf_flows.GrrMessage.AuthorizationState.AUTHENTICATED,\n            type=rdf_flows.GrrMessage.Type.STATUS,\n            payload=rdf_flows.GrrStatus())\n\n        flow_manager.QueueResponse(msg)\n\n        # And notify the worker about it.\n        flow_manager.QueueNotification(session_id=hunt_id)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef CallFlow(self,\n               flow_name=None,\n               next_state=None,\n               request_data=None,\n               client_id=None,\n               **kwargs):\n    \"\"\"Create a new child flow from a hunt.\"\"\"\n    base_session_id = None\n    if client_id:\n      # The flow is stored in the hunt namespace,\n      base_session_id = self.urn.Add(client_id.Basename())\n\n    # Actually start the new flow.\n    child_urn = self.runner.CallFlow(\n        flow_name=flow_name,\n        next_state=next_state,\n        base_session_id=base_session_id,\n        client_id=client_id,\n        request_data=request_data,\n        **kwargs)\n\n    if client_id:\n      # But we also create a symlink to it from the client's namespace.\n      hunt_link_urn = client_id.Add(\"flows\").Add(\"%s:hunt\" %\n                                                 (self.urn.Basename()))\n\n      hunt_link = aff4.FACTORY.Create(\n          hunt_link_urn, aff4.AFF4Symlink, token=self.token)\n\n      hunt_link.Set(hunt_link.Schema.SYMLINK_TARGET(child_urn))\n      hunt_link.Close()\n\n    return child_urn", "response": "Create a new child flow from a hunt."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef Start(self):\n    with data_store.DB.GetMutationPool() as mutation_pool:\n      self.CreateCollections(mutation_pool)\n\n    if not self.runner_args.description:\n      self.SetDescription()", "response": "Initializes this hunt from arguments."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef LogClientError(self, client_id, log_message=None, backtrace=None):\n    self.RegisterClientError(\n        client_id, log_message=log_message, backtrace=backtrace)", "response": "Logs an error for a client."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nprocessing a status message from a client and update the stats.", "response": "def ProcessClientResourcesStats(self, client_id, status):\n    \"\"\"Process status message from a client and update the stats.\n\n    Args:\n      client_id: Client id.\n      status: The status object returned from the client.\n    \"\"\"\n    if hasattr(status, \"child_session_id\"):\n      flow_path = status.child_session_id\n    else:\n      flow_path = \"aff4:/%s/flows/%s\" % (status.client_id, status.flow_id)\n\n    resources = rdf_client_stats.ClientResources()\n    resources.client_id = client_id\n    resources.session_id = flow_path\n    resources.cpu_usage.user_cpu_time = status.cpu_time_used.user_cpu_time\n    resources.cpu_usage.system_cpu_time = status.cpu_time_used.system_cpu_time\n    resources.network_bytes_sent = status.network_bytes_sent\n    self.context.usage_stats.RegisterResources(resources)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nget all the clients in a dict of status.", "response": "def GetClientsByStatus(self):\n    \"\"\"Get all the clients in a dict of {status: [client_list]}.\"\"\"\n    started = self.GetClients()\n    completed = self.GetCompletedClients()\n    outstanding = started - completed\n\n    return {\n        \"STARTED\": started,\n        \"COMPLETED\": completed,\n        \"OUTSTANDING\": outstanding\n    }"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ntakes in a client list and return dicts with their age and hostname.", "response": "def GetClientStates(self, client_list, client_chunk=50):\n    \"\"\"Take in a client list and return dicts with their age and hostname.\"\"\"\n    for client_group in collection.Batch(client_list, client_chunk):\n      for fd in aff4.FACTORY.MultiOpen(\n          client_group,\n          mode=\"r\",\n          aff4_type=aff4_grr.VFSGRRClient,\n          token=self.token):\n        result = {}\n        result[\"age\"] = fd.Get(fd.Schema.PING)\n        result[\"hostname\"] = fd.Get(fd.Schema.HOSTNAME)\n        yield (fd.urn, result)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nlist clients conforming to a givent query.", "response": "def SearchClients(query=None, context=None):\n  \"\"\"List clients conforming to a givent query.\"\"\"\n\n  args = client_pb2.ApiSearchClientsArgs(query=query)\n\n  items = context.SendIteratorRequest(\"SearchClients\", args)\n  return utils.MapItemsIterator(lambda data: Client(data=data, context=context),\n                                items)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nfetch and return a proper ClientApproval object.", "response": "def Get(self):\n    \"\"\"Fetch and return a proper ClientApproval object.\"\"\"\n\n    args = user_pb2.ApiGetClientApprovalArgs(\n        client_id=self.client_id,\n        approval_id=self.approval_id,\n        username=self.username)\n    result = self._context.SendRequest(\"GetClientApproval\", args)\n    return ClientApproval(\n        data=result, username=self._context.username, context=self._context)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef File(self, path):\n\n    return vfs.FileRef(\n        client_id=self.client_id, path=path, context=self._context)", "response": "Returns a reference to a file with a given path on client s VFS."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn a reference to a flow with a given id on this client.", "response": "def Flow(self, flow_id):\n    \"\"\"Return a reference to a flow with a given id on this client.\"\"\"\n\n    return flow.FlowRef(\n        client_id=self.client_id, flow_id=flow_id, context=self._context)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef CreateFlow(self, name=None, args=None, runner_args=None):\n\n    if not name:\n      raise ValueError(\"name can't be empty\")\n\n    request = flow_pb2.ApiCreateFlowArgs(client_id=self.client_id)\n\n    request.flow.name = name\n    if runner_args:\n      request.flow.runner_args.CopyFrom(runner_args)\n\n    if args:\n      request.flow.args.value = args.SerializeToString()\n      request.flow.args.type_url = utils.GetTypeUrl(args)\n\n    data = self._context.SendRequest(\"CreateFlow\", request)\n    return flow.Flow(data=data, context=self._context)", "response": "Create a new flow on this client."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nlist flows that ran on this client.", "response": "def ListFlows(self):\n    \"\"\"List flows that ran on this client.\"\"\"\n\n    args = flow_pb2.ApiListFlowsArgs(client_id=self.client_id)\n\n    items = self._context.SendIteratorRequest(\"ListFlows\", args)\n    return utils.MapItemsIterator(\n        lambda data: flow.Flow(data=data, context=self._context), items)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn a reference to an approval.", "response": "def Approval(self, username, approval_id):\n    \"\"\"Returns a reference to an approval.\"\"\"\n\n    return ClientApprovalRef(\n        client_id=self.client_id,\n        username=username,\n        approval_id=approval_id,\n        context=self._context)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncreate a new approval for the current user to access this client.", "response": "def CreateApproval(self,\n                     reason=None,\n                     notified_users=None,\n                     email_cc_addresses=None,\n                     keep_client_alive=False):\n    \"\"\"Create a new approval for the current user to access this client.\"\"\"\n\n    if not reason:\n      raise ValueError(\"reason can't be empty\")\n\n    if not notified_users:\n      raise ValueError(\"notified_users list can't be empty.\")\n\n    approval = user_pb2.ApiClientApproval(\n        reason=reason,\n        notified_users=notified_users,\n        email_cc_addresses=email_cc_addresses or [])\n    args = user_pb2.ApiCreateClientApprovalArgs(\n        client_id=self.client_id,\n        approval=approval,\n        keep_client_alive=keep_client_alive)\n\n    data = self._context.SendRequest(\"CreateClientApproval\", args)\n    return ClientApproval(\n        data=data, username=self._context.username, context=self._context)"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nfetches client s data and return a proper Client object.", "response": "def Get(self):\n    \"\"\"Fetch client's data and return a proper Client object.\"\"\"\n\n    args = client_pb2.ApiGetClientArgs(client_id=self.client_id)\n    result = self._context.SendRequest(\"GetClient\", args)\n    return Client(data=result, context=self._context)"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns a list of MetricMetadata for communicator - related metrics.", "response": "def GetMetricMetadata():\n  \"\"\"Returns a list of MetricMetadata for communicator-related metrics.\"\"\"\n  return [\n      stats_utils.CreateCounterMetadata(\"grr_client_unknown\"),\n      stats_utils.CreateCounterMetadata(\"grr_decoding_error\"),\n      stats_utils.CreateCounterMetadata(\"grr_decryption_error\"),\n      stats_utils.CreateCounterMetadata(\"grr_authenticated_messages\"),\n      stats_utils.CreateCounterMetadata(\"grr_unauthenticated_messages\"),\n      stats_utils.CreateCounterMetadata(\"grr_rsa_operations\"),\n      stats_utils.CreateCounterMetadata(\n          \"grr_encrypted_cipher_cache\", fields=[(\"type\", str)]),\n  ]"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef Encrypt(self, data, iv=None):\n    if iv is None:\n      iv = rdf_crypto.EncryptionKey.GenerateKey(length=128)\n    cipher = rdf_crypto.AES128CBCCipher(self.cipher.key, iv)\n    return iv, cipher.Encrypt(data)", "response": "Symmetrically encrypt the data using the optional iv."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef Decrypt(self, data, iv):\n    key = rdf_crypto.EncryptionKey(self.cipher.key)\n    iv = rdf_crypto.EncryptionKey(iv)\n    return rdf_crypto.AES128CBCCipher(key, iv).Decrypt(data)", "response": "Symmetrically decrypt the data."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _VerifyHMAC(self, comms=None):\n    # Check the encrypted message integrity using HMAC.\n    if self.hmac_type == \"SIMPLE_HMAC\":\n      msg = comms.encrypted\n      digest = comms.hmac\n    elif self.hmac_type == \"FULL_HMAC\":\n      msg = b\"\".join([\n          comms.encrypted, comms.encrypted_cipher,\n          comms.encrypted_cipher_metadata,\n          comms.packet_iv.SerializeToString(),\n          struct.pack(\"<I\", comms.api_version)\n      ])\n      digest = comms.full_hmac\n    else:\n      raise DecryptionError(\"HMAC type no supported.\")\n\n    try:\n      rdf_crypto.HMAC(self.cipher.hmac_key).Verify(msg, digest)\n    except rdf_crypto.VerificationError as e:\n      raise DecryptionError(\"HMAC verification failed: %s\" % e)\n\n    return True", "response": "Verifies the HMAC.\n\n    This method raises a DecryptionError if the received HMAC does not\n    verify. If the HMAC verifies correctly, True is returned.\n\n    Args:\n      comms: The comms RdfValue to verify.\n\n    Raises:\n      DecryptionError: The HMAC did not verify.\n\n    Returns:\n      True"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nverifying the signature on the encrypted cipher block.", "response": "def VerifyCipherSignature(self, remote_public_key):\n    \"\"\"Verifies the signature on the encrypted cipher block.\n\n    This method returns True if the signature verifies correctly with\n    the key given.\n\n    Args:\n      remote_public_key: The remote public key.\n\n    Returns:\n      None\n    Raises:\n      rdf_crypto.VerificationError: A signature and a key were both given but\n                                    verification fails.\n\n    \"\"\"\n    if self.cipher_metadata.signature and remote_public_key:\n\n      stats_collector_instance.Get().IncrementCounter(\"grr_rsa_operations\")\n      remote_public_key.Verify(self.serialized_cipher,\n                               self.cipher_metadata.signature)\n      return True"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef EncodeMessageList(cls, message_list, packed_message_list):\n    # By default uncompress\n    uncompressed_data = message_list.SerializeToString()\n    packed_message_list.message_list = uncompressed_data\n\n    compressed_data = zlib.compress(uncompressed_data)\n\n    # Only compress if it buys us something.\n    if len(compressed_data) < len(uncompressed_data):\n      packed_message_list.compression = (\n          rdf_flows.PackedMessageList.CompressionType.ZCOMPRESSION)\n      packed_message_list.message_list = compressed_data", "response": "Encodes the MessageList into the packed_message_list rdfvalue."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _GetServerCipher(self):\n\n    if self.server_cipher is not None:\n      expiry = self.server_cipher_age + rdfvalue.Duration(\"1d\")\n      if expiry > rdfvalue.RDFDatetime.Now():\n        return self.server_cipher\n\n    remote_public_key = self._GetRemotePublicKey(self.server_name)\n    self.server_cipher = Cipher(self.common_name, self.private_key,\n                                remote_public_key)\n    self.server_cipher_age = rdfvalue.RDFDatetime.Now()\n    return self.server_cipher", "response": "Returns the server cipher for self. server_name."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef EncodeMessages(self,\n                     message_list,\n                     result,\n                     destination=None,\n                     timestamp=None,\n                     api_version=3):\n    \"\"\"Accepts a list of messages and encodes for transmission.\n\n    This function signs and then encrypts the payload.\n\n    Args:\n       message_list: A MessageList rdfvalue containing a list of GrrMessages.\n       result: A ClientCommunication rdfvalue which will be filled in.\n       destination: The CN of the remote system this should go to.\n       timestamp: A timestamp to use for the signed messages. If None - use the\n         current time.\n       api_version: The api version which this should be encoded in.\n\n    Returns:\n       A nonce (based on time) which is inserted to the encrypted payload. The\n       client can verify that the server is able to decrypt the message and\n       return the nonce.\n\n    Raises:\n       RuntimeError: If we do not support this api version.\n    \"\"\"\n    if api_version not in [3]:\n      raise RuntimeError(\n          \"Unsupported api version: %s, expected 3.\" % api_version)\n    # TODO(amoser): This is actually not great, we have two\n    # communicator classes already, one for the client, one for the\n    # server. This should be different methods, not a single one that\n    # gets passed a destination (server side) or not (client side).\n    if destination is None:\n      destination = self.server_name\n      # For the client it makes sense to cache the server cipher since\n      # it's the only cipher it ever uses.\n      cipher = self._GetServerCipher()\n    else:\n      remote_public_key = self._GetRemotePublicKey(destination)\n      cipher = Cipher(self.common_name, self.private_key, remote_public_key)\n\n    # Make a nonce for this transaction\n    if timestamp is None:\n      self.timestamp = timestamp = int(time.time() * 1000000)\n\n    packed_message_list = rdf_flows.PackedMessageList(timestamp=timestamp)\n    self.EncodeMessageList(message_list, packed_message_list)\n\n    result.encrypted_cipher_metadata = cipher.encrypted_cipher_metadata\n\n    # Include the encrypted cipher.\n    result.encrypted_cipher = cipher.encrypted_cipher\n\n    serialized_message_list = packed_message_list.SerializeToString()\n\n    # Encrypt the message symmetrically.\n    # New scheme cipher is signed plus hmac over message list.\n    result.packet_iv, result.encrypted = cipher.Encrypt(serialized_message_list)\n\n    # This is to support older endpoints.\n    result.hmac = cipher.HMAC(result.encrypted)\n\n    # Newer endpoints only look at this HMAC. It is recalculated for each packet\n    # in the session. Note that encrypted_cipher and encrypted_cipher_metadata\n    # do not change between all packets in this session.\n    result.full_hmac = cipher.HMAC(result.encrypted, result.encrypted_cipher,\n                                   result.encrypted_cipher_metadata,\n                                   result.packet_iv.SerializeToString(),\n                                   struct.pack(\"<I\", api_version))\n\n    result.api_version = api_version\n\n    if isinstance(result, rdfvalue.RDFValue):\n      # Store the number of messages contained.\n      result.num_messages = len(message_list)\n\n    return timestamp", "response": "This function signs and encrypts a list of GrrMessages and encrypts the resulting message list."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef DecompressMessageList(cls, packed_message_list):\n    compression = packed_message_list.compression\n    if compression == rdf_flows.PackedMessageList.CompressionType.UNCOMPRESSED:\n      data = packed_message_list.message_list\n\n    elif (compression ==\n          rdf_flows.PackedMessageList.CompressionType.ZCOMPRESSION):\n      try:\n        data = zlib.decompress(packed_message_list.message_list)\n      except zlib.error as e:\n        raise DecodingError(\"Failed to decompress: %s\" % e)\n    else:\n      raise DecodingError(\"Compression scheme not supported\")\n\n    try:\n      result = rdf_flows.MessageList.FromSerializedString(data)\n    except rdfvalue.DecodeError:\n      raise DecodingError(\"RDFValue parsing failed.\")\n\n    return result", "response": "Decompress the message data from a PackedMessageList rdfvalue."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nextract and verifies a server message.", "response": "def DecodeMessages(self, response_comms):\n    \"\"\"Extract and verify server message.\n\n    Args:\n        response_comms: A ClientCommunication rdfvalue\n\n    Returns:\n       list of messages and the CN where they came from.\n\n    Raises:\n       DecryptionError: If the message failed to decrypt properly.\n    \"\"\"\n    # Have we seen this cipher before?\n    cipher_verified = False\n    try:\n      cipher = self.encrypted_cipher_cache.Get(response_comms.encrypted_cipher)\n      stats_collector_instance.Get().IncrementCounter(\n          \"grr_encrypted_cipher_cache\", fields=[\"hits\"])\n\n      # Even though we have seen this encrypted cipher already, we should still\n      # make sure that all the other fields are sane and verify the HMAC.\n      cipher.VerifyReceivedHMAC(response_comms)\n      cipher_verified = True\n\n      # If we have the cipher in the cache, we know the source and\n      # should have a corresponding public key.\n      source = cipher.GetSource()\n      remote_public_key = self._GetRemotePublicKey(source)\n    except KeyError:\n      stats_collector_instance.Get().IncrementCounter(\n          \"grr_encrypted_cipher_cache\", fields=[\"misses\"])\n      cipher = ReceivedCipher(response_comms, self.private_key)\n\n      source = cipher.GetSource()\n      try:\n        remote_public_key = self._GetRemotePublicKey(source)\n        if cipher.VerifyCipherSignature(remote_public_key):\n          # At this point we know this cipher is legit, we can cache it.\n          self.encrypted_cipher_cache.Put(response_comms.encrypted_cipher,\n                                          cipher)\n          cipher_verified = True\n\n      except UnknownClientCertError:\n        # We don't know who we are talking to.\n        remote_public_key = None\n\n    # Decrypt the message with the per packet IV.\n    plain = cipher.Decrypt(response_comms.encrypted, response_comms.packet_iv)\n    try:\n      packed_message_list = rdf_flows.PackedMessageList.FromSerializedString(\n          plain)\n    except rdfvalue.DecodeError as e:\n      raise DecryptionError(e)\n\n    message_list = self.DecompressMessageList(packed_message_list)\n\n    # Are these messages authenticated?\n    # pyformat: disable\n    auth_state = self.VerifyMessageSignature(\n        response_comms,\n        packed_message_list,\n        cipher,\n        cipher_verified,\n        response_comms.api_version,\n        remote_public_key)\n    # pyformat: enable\n\n    # Mark messages as authenticated and where they came from.\n    for msg in message_list.job:\n      msg.auth_state = auth_state\n      msg.source = cipher.cipher_metadata.source\n\n    return (message_list.job, cipher.cipher_metadata.source,\n            packed_message_list.timestamp)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nverifying the signature of the message list.", "response": "def VerifyMessageSignature(self, unused_response_comms, packed_message_list,\n                             cipher, cipher_verified, api_version,\n                             remote_public_key):\n    \"\"\"Verify the message list signature.\n\n    This is the way the messages are verified in the client.\n\n    In the client we also check that the nonce returned by the server is correct\n    (the timestamp doubles as a nonce). If the nonce fails we deem the response\n    unauthenticated since it might have resulted from a replay attack.\n\n    Args:\n      packed_message_list: The PackedMessageList rdfvalue from the server.\n      cipher: The cipher belonging to the remote end.\n      cipher_verified: If True, the cipher's signature is not verified again.\n      api_version: The api version we should use.\n      remote_public_key: The public key of the source.\n\n    Returns:\n      An rdf_flows.GrrMessage.AuthorizationState.\n\n    Raises:\n       DecryptionError: if the message is corrupt.\n    \"\"\"\n    # This is not used atm since we only support a single api version (3).\n    _ = api_version\n    result = rdf_flows.GrrMessage.AuthorizationState.UNAUTHENTICATED\n\n    if cipher_verified or cipher.VerifyCipherSignature(remote_public_key):\n      stats_collector_instance.Get().IncrementCounter(\n          \"grr_authenticated_messages\")\n      result = rdf_flows.GrrMessage.AuthorizationState.AUTHENTICATED\n\n    # Check for replay attacks. We expect the server to return the same\n    # timestamp nonce we sent.\n    if packed_message_list.timestamp != self.timestamp:  # pytype: disable=attribute-error\n      result = rdf_flows.GrrMessage.AuthorizationState.UNAUTHENTICATED\n\n    if not cipher.cipher_metadata:\n      # Fake the metadata\n      cipher.cipher_metadata = rdf_flows.CipherMetadata(\n          source=packed_message_list.source)\n\n    return result"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef RegisterLateBindingCallback(target_name, callback, **kwargs):\n  _LATE_BINDING_STORE.setdefault(target_name, []).append((callback, kwargs))", "response": "Registers a callback to be invoked when the RDFValue named is declared."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nparses a human readable string of a byte string.", "response": "def ParseFromHumanReadable(self, string):\n    \"\"\"Parse a human readable string of a byte string.\n\n    Args:\n      string: The string to parse.\n\n    Raises:\n      DecodeError: If the string can not be parsed.\n    \"\"\"\n    if not string:\n      return None\n\n    match = self.REGEX.match(string.strip().lower())\n    if not match:\n      raise DecodeError(\"Unknown specification for ByteSize %s\" % string)\n\n    multiplier = self.DIVIDERS.get(match.group(2))\n    if not multiplier:\n      raise DecodeError(\"Invalid multiplier %s\" % match.group(2))\n\n    # The value may be represented as a float, but if not dont lose accuracy.\n    value = match.group(1)\n    if \".\" in value:\n      value = float(value)\n    else:\n      value = int(value)\n\n    self._value = int(value * multiplier)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn True if the item should be dropped.", "response": "def FilterItem(self, launchditem):\n    \"\"\"Should this job be filtered.\n\n    Args:\n      launchditem: job NSCFDictionary\n    Returns:\n      True if the item should be filtered (dropped)\n    \"\"\"\n    for regex in self.blacklist_regex:\n      if regex.match(launchditem.get(\"Label\", \"\")):\n        return True\n    return False"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef Parse(self, persistence, knowledge_base, download_pathtype):\n    pathspecs = []\n\n    if isinstance(persistence, rdf_client.OSXServiceInformation):\n      if persistence.program:\n        pathspecs = rdf_paths.PathSpec(\n            path=persistence.program, pathtype=download_pathtype)\n      elif persistence.args:\n        pathspecs = rdf_paths.PathSpec(\n            path=persistence.args[0], pathtype=download_pathtype)\n\n    for pathspec in pathspecs:\n      yield rdf_standard.PersistenceFile(pathspec=pathspec)", "response": "Parse persistence collector output."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef Trim(lst, limit):\n  limit = max(0, limit)\n\n  clipping = lst[limit:]\n  del lst[limit:]\n  return clipping", "response": "Trims a given list so that it is not longer than given limit."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ngroups items by given key function.", "response": "def Group(items, key):\n  \"\"\"Groups items by given key function.\n\n  Args:\n    items: An iterable or an iterator of items.\n    key: A function which given each item will return the key.\n\n  Returns:\n    A dict with keys being each unique key and values being a list of items of\n    that key.\n  \"\"\"\n  result = {}\n\n  for item in items:\n    result.setdefault(key(item), []).append(item)\n\n  return result"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef Batch(items, size):\n  batch = []\n\n  for item in items:\n    batch.append(item)\n    if len(batch) == size:\n      yield batch\n      batch = []\n\n  if batch:\n    yield batch", "response": "Divides items into batches of specified size."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nchecking whether an iterable of items of one iterable are a prefix of another.", "response": "def StartsWith(this, that):\n  \"\"\"Checks whether an items of one iterable are a prefix of another.\n\n  Args:\n    this: An iterable that needs to be checked.\n    that: An iterable of which items must match the prefix of `this`.\n\n  Returns:\n    `True` if `that` is a prefix of `this`, `False` otherwise.\n  \"\"\"\n  this_iter = iter(this)\n  that_iter = iter(that)\n\n  while True:\n    try:\n      this_value = next(that_iter)\n    except StopIteration:\n      return True\n\n    try:\n      that_value = next(this_iter)\n    except StopIteration:\n      return False\n\n    if this_value != that_value:\n      return False"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nunzips specified iterable of pairs to pair of two iterables.", "response": "def Unzip(iterable):\n  \"\"\"Unzips specified iterable of pairs to pair of two iterables.\n\n  This function is an inversion of the standard `zip` function and the following\n  hold:\n\n    * \u2200 l, r. l, r == unzip(zip(l, r))\n    * \u2200 p. p == zip(unzip(p))\n\n  Examples:\n    >>> Unzip([(\"foo\", 1), (\"bar\", 2), (\"baz\", 3)])\n    ([\"foo\", \"bar\", \"baz\"], [1, 2, 3])\n\n  Args:\n    iterable: An iterable of pairs to unzip.\n\n  Returns:\n    A pair of iterables after unzipping.\n  \"\"\"\n  lefts = []\n  rights = []\n\n  for left, right in iterable:\n    lefts.append(left)\n    rights.append(right)\n\n  return lefts, rights"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncompute a cartesian product of dict with iterable values. This utility function, accepts a dictionary with iterable values, computes cartesian products of these values and yields dictionaries of expanded values. Examples: >>> list(DictProduct({\"a\": [1, 2], \"b\": [3, 4]})) [{\"a\": 1, \"b\": 3}, {\"a\": 1, \"b\": 4}, {\"a\": 2, \"b\": 3}, {\"a\": 2, \"b\": 4}] Args: dictionary: A dictionary with iterable values. Yields: Dictionaries with values being a result of cartesian product of values of the input dictionary.", "response": "def DictProduct(dictionary):\n  \"\"\"Computes a cartesian product of dict with iterable values.\n\n  This utility function, accepts a dictionary with iterable values, computes\n  cartesian products of these values and yields dictionaries of expanded values.\n\n  Examples:\n    >>> list(DictProduct({\"a\": [1, 2], \"b\": [3, 4]}))\n    [{\"a\": 1, \"b\": 3}, {\"a\": 1, \"b\": 4}, {\"a\": 2, \"b\": 3}, {\"a\": 2, \"b\": 4}]\n\n  Args:\n    dictionary: A dictionary with iterable values.\n\n  Yields:\n    Dictionaries with values being a result of cartesian product of values of\n    the input dictionary.\n  \"\"\"\n  keys, values = Unzip(iteritems(dictionary))\n  for product_values in itertools.product(*values):\n    yield dict(zip(keys, product_values))"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef AsIter(arg):\n  if isinstance(arg, string_types):\n    rslt = [arg]\n  elif isinstance(arg, collections.Iterable):\n    rslt = arg\n  elif not arg:\n    rslt = []\n  else:\n    rslt = [arg]\n  return tuple(rslt)", "response": "Encapsulates an argument in a tuple if it s not already iterable."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _GenStates(self):\n    self.GenCommentState()\n    self.GenFwdState()\n    self.GenQuotedState()\n    self.GenCatchallState()", "response": "Generate the lexer states."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef GenFwdState(self):\n    for c in self.cont:\n      self._AddToken(\"FWD\", c, None, None)\n    for s in self.sep:\n      self._AddToken(\"FWD\", s, None, None)\n    self._AddToken(\"FWD\", \".\", \"PushBack,PopState\", None)", "response": "Generates forwarding state rules."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef GenQuotedState(self):\n    for i, q in enumerate(self.quot):\n      label = \"%s_STRING\" % i\n      escaped = re.escape(q)\n      self._AddToken(label, escaped, \"PopState\", None)\n      self._AddToken(label, q, \"PopState\", None)\n      if self.ml_quote:\n        self._AddToken(label, r\"\\n\", None, None)\n      else:\n        self._AddToken(label, r\"\\n\", \"BadLine\", None)\n      self._AddToken(label, \".\", \"AddToField\", None)", "response": "Generates the string matching rules for the given quotation."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef GenCatchallState(self):\n    for c in self.comments:\n      self._AddToken(\".\", c, \"PushState,EndField\", \"COMMENT\")\n    for c in self.cont:\n      self._AddToken(\".\", c, \"PushState\", \"FWD\")\n    for t in self.term:\n      self._AddToken(\".\", t, \"EndEntry\", None)\n    for s in self.sep:\n      self._AddToken(\".\", s, \"EndField\", None)\n    for i, q in enumerate(self.quot):\n      self._AddToken(\".\", q, \"PushState\", \"%s_STRING\" % i)\n    self._AddToken(\".\", \".\", \"AddToField\", None)", "response": "Generates the state handlers that cover both the INITIAL and QUOTATION states."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nparse a line of the sshd config file.", "response": "def ParseLine(self, line):\n    \"\"\"Extracts keyword/value settings from the sshd config.\n\n    The keyword is always the first string item.\n    Values are the remainder of the string. In cases where an sshd config\n    allows multiple values, these are split according to whatever separator(s)\n    sshd_config permits for that value.\n\n    Keywords and values are normalized. Keywords are converted to lowercase.\n    Values are converted into integers, booleans or strings. Strings are always\n    lowercased.\n\n    Args:\n      line: A line of the configuration file.\n    \"\"\"\n    kv = line.split(None, 1)\n    keyword = kv[0].lower()\n    # Safely set the argument string if it wasn't found.\n    values = kv[1:] or [\"\"]\n    # Then split any parameters that are actually repeated items.\n    separators = self._repeated.get(keyword)\n    if separators:\n      repeated = []\n      for v in values:\n        repeated.extend(re.split(separators, v))\n      # Remove empty matches.\n      values = [v for v in repeated if v]\n\n    # Now convert the values to the right types.\n    if keyword in self._integers:\n      values = [int(v) for v in values]\n    elif keyword in self._booleans:\n      values = [v.lower() in self._true for v in values]\n    else:\n      values = [v.lower() for v in values]\n    # Only repeated arguments should be treated as a list.\n    if keyword not in self._repeated:\n      values = values[0]\n    # Switch sections for new match blocks.\n    if keyword == \"match\":\n      self._NewMatchSection(values)\n    # If it's an alias, resolve it.\n    if keyword in self._aliases:\n      keyword = self._aliases[keyword]\n    # Add the keyword/values to the section.\n    self.processor(keyword, values)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _ParseEntry(self, key, val):\n    if key in self._repeated:\n      setting = self.section.setdefault(key, [])\n      setting.extend(val)\n    else:\n      self.section.setdefault(key, val)", "response": "Parses a configuration setting entry."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _ParseMatchGrp(self, key, val):\n    if key in self._match_keywords:\n      self._ParseEntry(key, val)", "response": "Parses a match group parameter."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _NewMatchSection(self, val):\n    section = {\"criterion\": val, \"config\": {}}\n    self.matches.append(section)\n    # Now add configuration items to config section of the match block.\n    self.section = section[\"config\"]\n    # Switch to a match-specific processor on a new match_block.\n    self.processor = self._ParseMatchGrp", "response": "Create a new configuration section for each match clause."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nparse the sshd configuration file.", "response": "def Parse(self, stat, file_object, knowledge_base):\n    \"\"\"Parse the sshd configuration.\n\n    Process each of the lines in the configuration file.\n\n    Assembes an sshd_config file into a dictionary with the configuration\n    keyword as the key, and the configuration settings as value(s).\n\n    Args:\n      stat: unused\n      file_object: An open configuration file object.\n      knowledge_base: unused\n\n    Yields:\n      The configuration as an rdfvalue.\n    \"\"\"\n    _, _ = stat, knowledge_base\n    # Clean out any residual state.\n    self._field_parser.Flush()\n    lines = [\n        l.strip()\n        for l in utils.ReadFileBytesAsUnicode(file_object).splitlines()\n    ]\n    for line in lines:\n      # Remove comments (will break if it includes a quoted/escaped #)\n      line = line.split(\"#\")[0].strip()\n      if line:\n        self._field_parser.ParseLine(line)\n    for result in self._field_parser.GenerateResults():\n      yield result"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nparsing the mount command output.", "response": "def Parse(self, cmd, args, stdout, stderr, return_val, time_taken,\n            knowledge_base):\n    \"\"\"Parse the mount command output.\"\"\"\n    _ = stderr, time_taken, args, knowledge_base  # Unused.\n    self.CheckReturn(cmd, return_val)\n    result = rdf_protodict.AttributedDict()\n    for entry in self._field_parser.ParseEntries(stdout):\n      line_str = \" \".join(entry)\n      mount_rslt = self.mount_re.match(line_str)\n      if mount_rslt:\n        device, mount_point, fs_type, option_str = mount_rslt.groups()\n        result = rdf_client_fs.Filesystem()\n        result.device = device\n        result.mount_point = mount_point\n        result.type = fs_type\n        # Parse these options as a dict as some items may be key/values.\n        # KeyValue parser uses OrderedDict as the native parser method. Use it.\n        options = KeyValueParser(term=\",\").ParseToOrderedDict(option_str)\n        # Keys without values get assigned [] by default. Because these keys are\n        # actually true, if declared, change any [] values to True.\n        for k, v in iteritems(options):\n          options[k] = v or [True]\n        result.options = rdf_protodict.AttributedDict(**options)\n        yield result"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef ParseAction(self, action):\n    rslt = rdf_config_file.LogTarget()\n    for dst_str, dst_re in iteritems(self.destinations):\n      dst = dst_re.match(action)\n      if dst:\n        rslt.transport = dst_str\n        rslt.destination = dst.group(1)\n        break\n    return rslt", "response": "Parses the rsyslog action string and returns a LogTarget object."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nparses a key value formatted source listing and return a list of potential URLs.", "response": "def ParseURIFromKeyValues(self, data, separator, uri_key):\n    \"\"\"Parse key/value formatted source listing and return potential URLs.\n\n    The fundamental shape of this format is as follows:\n    key: value   # here : = separator\n    key : value\n    URI: [URL]   # here URI = uri_key\n      [URL]      # this is where it becomes trickey because [URL]\n      [URL]      # can contain 'separator' specially if separator is :\n    key: value\n\n    The key uri_key is of interest to us and since the next line\n    in the config could contain another [URL], we need to keep track of context\n    when we hit uri_key  to be able to check if the next line(s)\n    have more [URL].\n\n    Args:\n      data: unprocessed lines from a file\n      separator: how the key/value pairs are seperated\n      uri_key: starting name of the key containing URI.\n\n    Returns:\n      A list of potential URLs found in data\n    \"\"\"\n    precondition.AssertType(data, Text)\n    precondition.AssertType(separator, Text)\n\n    kv_entries = KeyValueParser(kv_sep=separator).ParseEntries(data)\n    spaced_entries = FieldParser().ParseEntries(data)\n\n    uris = []\n    check_uri_on_next_line = False\n    for kv_entry, sp_entry in zip(kv_entries, spaced_entries):\n      for k, v in iteritems(kv_entry):\n        # This line could be a URL if a) from  key:value, value is empty OR\n        # b) if separator is : and first character of v starts with /.\n        if (check_uri_on_next_line and\n            (not v or (separator == \":\" and v[0].startswith(\"/\")))):\n          uris.append(sp_entry[0])\n        else:\n          check_uri_on_next_line = False\n          if k.lower().startswith(uri_key) and v:\n            check_uri_on_next_line = True\n            uris.append(v[0])  # v is a list\n\n    return uris"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ngiving a file this will return all potenial APT source URIs.", "response": "def FindPotentialURIs(self, file_obj):\n    \"\"\"Given a file, this will return all potenial APT source URIs.\"\"\"\n    rfc822_format = \"\"  # will contain all lines not in legacy format\n    uris_to_parse = []\n\n    for line in utils.ReadFileBytesAsUnicode(file_obj).splitlines(True):\n      # check if legacy style line - if it is then extract URL\n      m = re.search(r\"^\\s*deb(?:-\\S+)?(?:\\s+\\[[^\\]]*\\])*\\s+(\\S+)(?:\\s|$)\", line)\n      if m:\n        uris_to_parse.append(m.group(1))\n      else:\n        rfc822_format += line\n\n    uris_to_parse.extend(self.ParseURIFromKeyValues(rfc822_format, \":\", \"uri\"))\n    return uris_to_parse"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef ParseLine(self, entries):\n    # If no entries were found, short circuit.\n    if not entries:\n      return\n    keyword = entries[0].lower()\n    # Set the argument string if it wasn't found.\n    values = entries[1:] or [\"\"]\n\n    # Convert any types we need too.\n    if keyword in self._integers:\n      values = [int(v) for v in values]\n    if keyword in self._floats:\n      values = [float(v) for v in values]\n\n    if keyword not in self._repeated | self._duplicates:\n      # We have a plain and simple single key/value config line.\n      if isinstance(values[0], string_types):\n        self.config[keyword] = \" \".join(values)\n      else:\n        self.config[keyword] = values\n\n    elif keyword in self._repeated:\n      # The keyword can have multiple single-word options, so add them as a list\n      # and overwrite previous settings.\n      self.config[keyword] = values\n\n    elif keyword in self._duplicates:\n      if keyword in self._address_based:\n        # If we have an address keyed keyword, join the keyword and address\n        # together to make the complete key for this data.\n        address = values[0].lower()\n        values = values[1:] or [\"\"]\n        # Add/overwrite the address in this 'keyed' keywords dictionary.\n        existing_keyword_config = self.keyed.setdefault(keyword, [])\n        # Create a dict which stores the server name and the options.\n        # Flatten the remaining options into a single string.\n        existing_keyword_config.append({\n            \"address\": address,\n            \"options\": \" \".join(values)\n        })\n\n      # Are we toggling an option?\n      elif keyword in self._boolean:\n        for option in values:\n          if keyword == \"enable\":\n            self.config[option] = True\n          else:\n            # As there are only two items in this set, we can assume disable.\n            self.config[option] = False\n\n      else:\n        # We have a non-keyed & non-boolean keyword, so add to the collected\n        # data so far. Order matters technically.\n        prev_settings = self.config.setdefault(keyword, [])\n        prev_settings.append(\" \".join(values))", "response": "Parses a single line of ntpd config file."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nparsing a ntp config into rdf.", "response": "def Parse(self, stat, file_object, knowledge_base):\n    \"\"\"Parse a ntp config into rdf.\"\"\"\n    _, _ = stat, knowledge_base\n\n    # TODO(hanuszczak): This parser only allows single use because it messes\n    # with its state. This should be fixed.\n    field_parser = NtpdFieldParser()\n    for line in field_parser.ParseEntries(\n        utils.ReadFileBytesAsUnicode(file_object)):\n      field_parser.ParseLine(line)\n\n    yield rdf_config_file.NtpConfig(\n        config=field_parser.config,\n        server=field_parser.keyed.get(\"server\"),\n        restrict=field_parser.keyed.get(\"restrict\"),\n        fudge=field_parser.keyed.get(\"fudge\"),\n        trap=field_parser.keyed.get(\"trap\"),\n        peer=field_parser.keyed.get(\"peer\"),\n        broadcast=field_parser.keyed.get(\"broadcast\"),\n        manycastclient=field_parser.keyed.get(\"manycastclient\"))"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _ExtractList(self, fields, ignores=(\",\",), terminators=()):\n    extracted = []\n    i = 0\n    for i, field in enumerate(fields):\n      # Space-separated comma; ignore, but this is not a finished list.\n      # Similar for any other specified ignores (eg, equals sign).\n      if field in ignores:\n        continue\n\n      # However, some fields are specifically meant to terminate iteration.\n      if field in terminators:\n        break\n\n      extracted.append(field.strip(\"\".join(ignores)))\n      # Check for continuation; this will either be a trailing comma or the\n      # next field after this one being a comma. The lookahead here is a bit\n      # nasty.\n      if not (field.endswith(\",\") or\n              set(fields[i + 1:i + 2]).intersection(ignores)):\n        break\n\n    return extracted, fields[i + 1:]", "response": "Extracts a list from the given fields."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef ParseSudoersEntry(self, entry, sudoers_config):\n\n    key = entry[0]\n    if key in SudoersFieldParser.ALIAS_TYPES:\n      # Alias.\n      alias_entry = rdf_config_file.SudoersAlias(\n          type=SudoersFieldParser.ALIAS_TYPES.get(key), name=entry[1])\n\n      # Members of this alias, comma-separated.\n      members, _ = self._ExtractList(entry[2:], ignores=(\",\", \"=\"))\n      field = SudoersFieldParser.ALIAS_FIELDS.get(key)\n      getattr(alias_entry, field).Extend(members)\n\n      sudoers_config.aliases.append(alias_entry)\n    elif key.startswith(SudoersFieldParser.DEFAULTS_KEY):\n      # Default.\n      # Identify scope if one exists (Defaults<scope> ...)\n      scope = None\n      if len(key) > len(SudoersFieldParser.DEFAULTS_KEY):\n        scope = key[len(SudoersFieldParser.DEFAULTS_KEY) + 1:]\n\n      # There can be multiple defaults on a line, for the one scope.\n      entry = entry[1:]\n      defaults, _ = self._ExtractList(entry)\n      for default in defaults:\n        default_entry = rdf_config_file.SudoersDefault(scope=scope)\n\n        # Extract key name and value(s).\n        default_name = default\n        value = []\n        if \"=\" in default_name:\n          default_name, remainder = default_name.split(\"=\", 1)\n          value = [remainder]\n        default_entry.name = default_name\n        if entry:\n          default_entry.value = \" \".join(value)\n\n        sudoers_config.defaults.append(default_entry)\n    elif key in SudoersFieldParser.INCLUDE_KEYS:\n      # TODO(user): make #includedir more obvious in the RDFValue somewhere\n      target = \" \".join(entry[1:])\n      sudoers_config.includes.append(target)\n    else:\n      users, entry = self._ExtractList(entry)\n      hosts, entry = self._ExtractList(entry, terminators=(\"=\",))\n\n      # Remove = from <user> <host> = <specs>\n      if entry[0] == \"=\":\n        entry = entry[1:]\n\n      # Command specification.\n      sudoers_entry = rdf_config_file.SudoersEntry(\n          users=users, hosts=hosts, cmdspec=entry)\n\n      sudoers_config.entries.append(sudoers_entry)", "response": "Parses an entry and adds it to the given SudoersConfig rdfvalue."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef Preprocess(self, data):\n    # Add whitespace to line continuations.\n    data = data.replace(\":\\\\\", \": \\\\\")\n\n    # Strip comments manually because sudoers has multiple meanings for '#'.\n    data = SudoersFieldParser.COMMENTS_RE.sub(\"\", data)\n    return data", "response": "Preprocess the given data ready for parsing."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nvalidates an RDFValue instance.", "response": "def Validate(self, value):\n    \"\"\"Validate an RDFValue instance.\n\n    Args:\n      value: An RDFValue instance or something which may be used to instantiate\n        the correct instance.\n\n    Raises:\n      TypeValueError: If the value is not a valid RDFValue instance or the\n        required type.\n\n    Returns:\n      A Valid RDFValue instance.\n    \"\"\"\n    # Allow None as a default.\n    if value is None:\n      return\n\n    if not isinstance(value, self.rdfclass):\n      # Try to coerce the type to the correct rdf_class.\n      try:\n        return self.rdfclass(value)\n      except rdfvalue.InitializeError:\n        raise TypeValueError(\"Value for arg %s should be an %s\" %\n                             (self.name, self.rdfclass.__name__))\n\n    return value"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nvalidates the value. Args: value: Value is expected to be a dict-like object that a given RDFStruct can be initialized from. Raises: TypeValueError: If the value is not a valid dict-like object that a given RDFStruct can be initialized from. Returns: A valid instance of self.rdfclass or None.", "response": "def Validate(self, value):\n    \"\"\"Validate the value.\n\n    Args:\n      value: Value is expected to be a dict-like object that a given RDFStruct\n        can be initialized from.\n\n    Raises:\n      TypeValueError: If the value is not a valid dict-like object that a given\n        RDFStruct can be initialized from.\n\n    Returns:\n      A valid instance of self.rdfclass or None.\n    \"\"\"\n    if value is None:\n      return None\n\n    if not isinstance(value, self.rdfclass):\n      # Try to coerce the type to the correct rdf_class.\n      try:\n        r = self.rdfclass()\n        r.FromDict(value)\n        return r\n      except (AttributeError, TypeError, rdfvalue.InitializeError):\n        # AttributeError is raised if value contains items that don't\n        # belong to the given rdfstruct.\n        # TypeError will be raised if value is not a dict-like object.\n        raise TypeValueError(\"Value for arg %s should be an %s\" %\n                             (self.name, self.rdfclass.__name__))\n\n    return value"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a copy of this set with a new element added.", "response": "def Add(self, other):\n    \"\"\"Returns a copy of this set with a new element added.\"\"\"\n    new_descriptors = []\n    for desc in self.descriptors + other.descriptors:\n      if desc not in new_descriptors:\n        new_descriptors.append(desc)\n\n    return TypeDescriptorSet(*new_descriptors)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nappend the descriptor to this set.", "response": "def Append(self, desc):\n    \"\"\"Append the descriptor to this set.\"\"\"\n    if desc not in self.descriptors:\n      self.descriptors.append(desc)\n      self.descriptor_map[desc.name] = desc\n      self.descriptor_names.append(desc.name)"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns a copy of this set without elements with given names.", "response": "def Remove(self, *descriptor_names):\n    \"\"\"Returns a copy of this set without elements with given names.\"\"\"\n    new_descriptor_map = self.descriptor_map.copy()\n    for name in descriptor_names:\n      new_descriptor_map.pop(name, None)\n\n    new_descriptors = [\n        desc for desc in self.descriptors\n        if desc in itervalues(new_descriptor_map)\n    ]\n    return TypeDescriptorSet(*new_descriptors)"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nparses and validate the args.", "response": "def ParseArgs(self, args):\n    \"\"\"Parse and validate the args.\n\n    Note we pop all the args we consume here - so if there are any args we dont\n    know about, args will not be an empty dict after this. This allows the same\n    args to be parsed by several TypeDescriptorSets.\n\n    Args:\n      args: A dictionary of arguments that this TypeDescriptorSet might use. If\n        this dict does not have a required parameter, we still yield its default\n        value.\n\n    Yields:\n      A (name, value) tuple of the parsed args.\n    \"\"\"\n    for descriptor in self:\n      # Get the value from the kwargs or, if not specified, the default.\n      value = args.pop(descriptor.name, None)\n\n      if value is None:\n        # No need to validate the default value.\n        value = descriptor.default\n      else:\n        try:\n          # Validate this value - this should raise if the value provided is not\n          # acceptable to the type descriptor.\n          value = descriptor.Validate(value)\n        except Exception:\n          logging.error(\"Invalid value %s for arg %s\", value, descriptor.name)\n          raise\n\n      yield descriptor.name, value"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nparse a bool from a string.", "response": "def FromString(self, string):\n    \"\"\"Parse a bool from a string.\"\"\"\n    if string.lower() in (\"false\", \"no\", \"n\"):\n      return False\n\n    if string.lower() in (\"true\", \"yes\", \"y\"):\n      return True\n\n    raise TypeValueError(\"%s is not recognized as a boolean value.\" % string)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nvalidate a potential list.", "response": "def Validate(self, value):\n    \"\"\"Validate a potential list.\"\"\"\n    if isinstance(value, string_types):\n      raise TypeValueError(\"Value must be an iterable not a string.\")\n\n    elif not isinstance(value, (list, tuple)):\n      raise TypeValueError(\"%r not a valid List\" % value)\n\n    # Validate each value in the list validates against our type.\n    return [self.validator.Validate(val) for val in value]"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef Register():\n  # pyformat: disable\n\n  # Command parsers.\n  parsers.SINGLE_RESPONSE_PARSER_FACTORY.Register(\n      \"Dpkg\", linux_cmd_parser.DpkgCmdParser)\n  parsers.SINGLE_RESPONSE_PARSER_FACTORY.Register(\n      \"Dmidecode\", linux_cmd_parser.DmidecodeCmdParser)\n  parsers.SINGLE_RESPONSE_PARSER_FACTORY.Register(\n      \"Mount\", config_file.MountCmdParser)\n  parsers.SINGLE_RESPONSE_PARSER_FACTORY.Register(\n      \"OsxSpHardware\", osx_file_parser.OSXSPHardwareDataTypeParser)\n  parsers.SINGLE_RESPONSE_PARSER_FACTORY.Register(\n      \"Ps\", linux_cmd_parser.PsCmdParser)\n  parsers.SINGLE_RESPONSE_PARSER_FACTORY.Register(\n      \"Rpm\", linux_cmd_parser.RpmCmdParser)\n  parsers.SINGLE_RESPONSE_PARSER_FACTORY.Register(\n      \"SshdConfig\", config_file.SshdConfigCmdParser)\n  parsers.SINGLE_RESPONSE_PARSER_FACTORY.Register(\n      \"Sysctl\", linux_sysctl_parser.SysctlCmdParser)\n  parsers.SINGLE_RESPONSE_PARSER_FACTORY.Register(\n      \"YumList\", linux_cmd_parser.YumListCmdParser)\n  parsers.SINGLE_RESPONSE_PARSER_FACTORY.Register(\n      \"YumRepolist\", linux_cmd_parser.YumRepolistCmdParser)\n\n  # Grep parsers.\n  parsers.SINGLE_RESPONSE_PARSER_FACTORY.Register(\n      \"Passwd\", linux_file_parser.PasswdBufferParser)\n  parsers.SINGLE_RESPONSE_PARSER_FACTORY.Register(\n      \"Netgroup\", linux_file_parser.NetgroupBufferParser)\n\n  # WMI query parsers.\n  parsers.MULTI_RESPONSE_PARSER_FACTORY.Register(\n      \"WmiEventConsumer\", wmi_parser.WMIEventConsumerParser)\n  parsers.MULTI_RESPONSE_PARSER_FACTORY.Register(\n      \"WmiInstalledSoftware\", wmi_parser.WMIInstalledSoftwareParser)\n  parsers.MULTI_RESPONSE_PARSER_FACTORY.Register(\n      \"WmiHotfixesSoftware\", wmi_parser.WMIHotfixesSoftwareParser)\n  parsers.MULTI_RESPONSE_PARSER_FACTORY.Register(\n      \"WmiUser\", wmi_parser.WMIUserParser)\n  parsers.MULTI_RESPONSE_PARSER_FACTORY.Register(\n      \"WmiLogicalDisks\", wmi_parser.WMILogicalDisksParser)\n  parsers.MULTI_RESPONSE_PARSER_FACTORY.Register(\n      \"WmiCsp\", wmi_parser.WMIComputerSystemProductParser)\n  parsers.MULTI_RESPONSE_PARSER_FACTORY.Register(\n      \"WmiInterfaces\", wmi_parser.WMIInterfacesParser)\n\n  # Registry value parsers.\n  parsers.SINGLE_RESPONSE_PARSER_FACTORY.Register(\n      \"WinCcs\", windows_registry_parser.CurrentControlSetKBParser)\n  parsers.SINGLE_RESPONSE_PARSER_FACTORY.Register(\n      \"WinCodepage\", windows_registry_parser.CodepageParser)\n  parsers.SINGLE_RESPONSE_PARSER_FACTORY.Register(\n      \"WinEnvironment\", windows_registry_parser.WinEnvironmentParser)\n  parsers.SINGLE_RESPONSE_PARSER_FACTORY.Register(\n      \"WinServices\", windows_registry_parser.WinServicesParser)\n  parsers.SINGLE_RESPONSE_PARSER_FACTORY.Register(\n      \"WinSystemDrive\", windows_registry_parser.WinSystemDriveParser)\n  parsers.SINGLE_RESPONSE_PARSER_FACTORY.Register(\n      \"WinSystemRoot\", windows_registry_parser.WinSystemRootParser)\n  parsers.SINGLE_RESPONSE_PARSER_FACTORY.Register(\n      \"WinTimezone\", windows_registry_parser.WinTimezoneParser)\n\n  # Registry parsers.\n  parsers.SINGLE_RESPONSE_PARSER_FACTORY.Register(\n      \"WinAllUsersProfileEnvVar\",\n      windows_registry_parser.AllUsersProfileEnvironmentVariable)\n  parsers.SINGLE_RESPONSE_PARSER_FACTORY.Register(\n      \"WinProfileDirEnvVar\",\n      windows_registry_parser.ProfilesDirectoryEnvironmentVariable)\n  parsers.SINGLE_RESPONSE_PARSER_FACTORY.Register(\n      \"WinUserSids\",\n      windows_registry_parser.WinUserSids)\n\n  # Artifact file parsers.\n  parsers.SINGLE_RESPONSE_PARSER_FACTORY.Register(\n      \"DarwinPersistenceMechanism\",\n      osx_launchd.DarwinPersistenceMechanismsParser)\n  parsers.SINGLE_RESPONSE_PARSER_FACTORY.Register(\n      \"WindowsPersistenceMechanism\",\n      windows_persistence.WindowsPersistenceMechanismsParser)\n\n  # Registry multi-parsers.\n  parsers.MULTI_RESPONSE_PARSER_FACTORY.Register(\n      \"WinUserSpecialDirs\", windows_registry_parser.WinUserSpecialDirs)\n\n  # Artifact file multi-parsers.\n  parsers.MULTI_RESPONSE_PARSER_FACTORY.Register(\n      \"OsxUsers\", osx_file_parser.OSXUsersParser)\n\n  # File parsers.\n  parsers.SINGLE_FILE_PARSER_FACTORY.Register(\n      \"ChromeHistory\", chrome_history.ChromeHistoryParser)\n  parsers.SINGLE_FILE_PARSER_FACTORY.Register(\n      \"CronAtAllAllowDeny\", config_file.CronAtAllowDenyParser)\n  parsers.SINGLE_FILE_PARSER_FACTORY.Register(\n      \"CronTab\", cron_file_parser.CronTabParser)\n  parsers.SINGLE_FILE_PARSER_FACTORY.Register(\n      \"FirefoxHistory\", firefox3_history.FirefoxHistoryParser)\n  parsers.SINGLE_FILE_PARSER_FACTORY.Register(\n      \"IeHistory\", ie_history.IEHistoryParser)\n  parsers.SINGLE_FILE_PARSER_FACTORY.Register(\n      \"LinuxWtmp\", linux_file_parser.LinuxWtmpParser)\n  parsers.SINGLE_FILE_PARSER_FACTORY.Register(\n      \"Mtab\", config_file.MtabParser)\n  parsers.SINGLE_FILE_PARSER_FACTORY.Register(\n      \"Netgroup\", linux_file_parser.NetgroupParser)\n  parsers.SINGLE_FILE_PARSER_FACTORY.Register(\n      \"NfsExports\", config_file.NfsExportsParser)\n  parsers.SINGLE_FILE_PARSER_FACTORY.Register(\n      \"Ntpd\", config_file.NtpdParser)\n  parsers.SINGLE_FILE_PARSER_FACTORY.Register(\n      \"PackageSource\", config_file.PackageSourceParser)\n  parsers.SINGLE_FILE_PARSER_FACTORY.Register(\n      \"Passwd\", linux_file_parser.PasswdParser)\n  parsers.SINGLE_FILE_PARSER_FACTORY.Register(\n      \"Path\", linux_file_parser.PathParser)\n  parsers.SINGLE_FILE_PARSER_FACTORY.Register(\n      \"SshdConfigFile\", config_file.SshdConfigParser)\n  parsers.SINGLE_FILE_PARSER_FACTORY.Register(\n      \"Sudoers\", config_file.SudoersParser)\n  parsers.SINGLE_FILE_PARSER_FACTORY.Register(\n      \"OsxLaunchdPlist\", osx_file_parser.OSXLaunchdPlistParser)\n  parsers.SINGLE_FILE_PARSER_FACTORY.Register(\n      \"OSXInstallHistoryPlist\", osx_file_parser.OSXInstallHistoryPlistParser)\n\n  try:\n    from debian import deb822  # pylint: disable=g-import-not-at-top\n    parsers.SINGLE_FILE_PARSER_FACTORY.Register(\n        \"DpkgStatusParser\",\n        lambda: linux_software_parser.DebianPackagesStatusParser(deb822))\n  except ImportError:\n    pass\n\n\n  # File multi-parsers.\n  parsers.MULTI_FILE_PARSER_FACTORY.Register(\n      \"LinuxBaseShadow\", linux_file_parser.LinuxBaseShadowParser)\n  parsers.MULTI_FILE_PARSER_FACTORY.Register(\n      \"LinuxLsbInit\", linux_service_parser.LinuxLSBInitParser)\n  parsers.MULTI_FILE_PARSER_FACTORY.Register(\n      \"LinuxXinetd\", linux_service_parser.LinuxXinetdParser)\n  parsers.MULTI_FILE_PARSER_FACTORY.Register(\n      \"LinuxSysvInit\", linux_service_parser.LinuxSysVInitParser)\n  parsers.MULTI_FILE_PARSER_FACTORY.Register(\n      \"LinuxPam\", linux_pam_parser.PAMParser)\n  parsers.MULTI_FILE_PARSER_FACTORY.Register(\n      \"LinuxReleaseInfo\", linux_release_parser.LinuxReleaseParser)\n  parsers.MULTI_FILE_PARSER_FACTORY.Register(\n      \"PciDevicesInfo\", linux_file_parser.PCIDevicesInfoParser)\n  parsers.MULTI_FILE_PARSER_FACTORY.Register(\n      \"ProcSys\", linux_sysctl_parser.ProcSysParser)\n  parsers.MULTI_FILE_PARSER_FACTORY.Register(\n      \"Rsyslog\", config_file.RsyslogParser)", "response": "Registers all known command parsers to the registry."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning the rdfurn of the current audit log.", "response": "def _CurrentAuditLog():\n  \"\"\"Get the rdfurn of the current audit log.\"\"\"\n  now_sec = rdfvalue.RDFDatetime.Now().AsSecondsSinceEpoch()\n  rollover_seconds = AUDIT_ROLLOVER_TIME.seconds\n  # This gives us a filename that only changes every\n  # AUDIT_ROLLOVER_TIfilME seconds, but is still a valid timestamp.\n  current_log = (now_sec // rollover_seconds) * rollover_seconds\n  return _AuditLogBase().Add(str(current_log))"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef UnicodeFromCodePage(string):\n  # get the current code page\n  codepage = ctypes.windll.kernel32.GetOEMCP()\n  try:\n    return string.decode(\"cp%s\" % codepage)\n  except UnicodeError:\n    try:\n      return string.decode(\"utf16\", \"ignore\")\n    except UnicodeError:\n      # Fall back on utf8 but ignore errors\n      return string.decode(\"utf8\", \"ignore\")", "response": "Attempt to coerce string into a unicode object."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nenumerating all interfaces from the client.", "response": "def EnumerateInterfacesFromClient(args):\n  \"\"\"Enumerate all MAC addresses of all NICs.\n\n  Args:\n    args: Unused.\n\n  Yields:\n    `rdf_client_network.Interface` instances.\n  \"\"\"\n  del args  # Unused.\n\n  pythoncom.CoInitialize()\n  for interface in (wmi.WMI().Win32_NetworkAdapterConfiguration() or []):\n    addresses = []\n    for ip_address in interface.IPAddress or []:\n      addresses.append(\n          rdf_client_network.NetworkAddress(human_readable_address=ip_address))\n\n    response = rdf_client_network.Interface(ifname=interface.Description)\n    if interface.MACAddress:\n      response.mac_address = binascii.unhexlify(\n          interface.MACAddress.replace(\":\", \"\"))\n    if addresses:\n      response.addresses = addresses\n\n    yield response"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nenumerating all local filesystems mounted on this system.", "response": "def EnumerateFilesystemsFromClient(args):\n  \"\"\"List all local filesystems mounted on this system.\"\"\"\n  del args  # Unused.\n  for drive in win32api.GetLogicalDriveStrings().split(\"\\x00\"):\n    if not drive:\n      continue\n    try:\n      volume = win32file.GetVolumeNameForVolumeMountPoint(drive).rstrip(\"\\\\\")\n\n      label, _, _, _, fs_type = win32api.GetVolumeInformation(drive)\n    except win32api.error:\n      continue\n    yield rdf_client_fs.Filesystem(\n        device=volume,\n        mount_point=\"/%s:/\" % drive[0],\n        type=fs_type,\n        label=UnicodeFromCodePage(label))"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef QueryService(svc_name):\n  hscm = win32service.OpenSCManager(None, None,\n                                    win32service.SC_MANAGER_ALL_ACCESS)\n  result = None\n  try:\n    hs = win32serviceutil.SmartOpenService(hscm, svc_name,\n                                           win32service.SERVICE_ALL_ACCESS)\n    result = win32service.QueryServiceConfig(hs)\n    win32service.CloseServiceHandle(hs)\n  finally:\n    win32service.CloseServiceHandle(hscm)\n\n  return result", "response": "Query service and get its config."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nrunning the WMI query and return the data.", "response": "def WmiQueryFromClient(args):\n  \"\"\"Run the WMI query and return the data.\"\"\"\n  query = args.query\n  base_object = args.base_object or r\"winmgmts:\\root\\cimv2\"\n\n  if not query.upper().startswith(\"SELECT \"):\n    raise RuntimeError(\"Only SELECT WMI queries allowed.\")\n\n  for response_dict in RunWMIQuery(query, baseobj=base_object):\n    yield response_dict"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef RunWMIQuery(query, baseobj=r\"winmgmts:\\root\\cimv2\"):\n  pythoncom.CoInitialize()  # Needs to be called if using com from a thread.\n  wmi_obj = win32com.client.GetObject(baseobj)\n  # This allows our WMI to do some extra things, in particular\n  # it gives it access to find the executable path for all processes.\n  wmi_obj.Security_.Privileges.AddAsString(\"SeDebugPrivilege\")\n\n  # Run query\n  try:\n    query_results = wmi_obj.ExecQuery(query)\n  except pythoncom.com_error as e:\n    raise RuntimeError(\"Failed to run WMI query \\'%s\\' err was %s\" % (query, e))\n\n  # Extract results from the returned COMObject and return dicts.\n  try:\n    for result in query_results:\n      response = rdf_protodict.Dict()\n      properties = (\n          list(result.Properties_) +\n          list(getattr(result, \"SystemProperties_\", [])))\n\n      for prop in properties:\n        if prop.Name not in IGNORE_PROPS:\n          # Protodict can handle most of the types we care about, but we may\n          # get some objects that we don't know how to serialize, so we tell the\n          # dict to set the value to an error message and keep going\n          response.SetItem(prop.Name, prop.Value, raise_on_error=False)\n      yield response\n\n  except pythoncom.com_error as e:\n    raise RuntimeError(\"WMI query data error on query \\'%s\\' err was %s\" %\n                       (e, query))", "response": "Run a WMI query and return a result."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef Run(self, unused_args):\n    # Don't use winreg.KEY_WOW64_64KEY since it breaks on Windows 2000\n    subkey = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE,\n                            \"Software\\\\Microsoft\\\\Windows NT\\\\CurrentVersion\",\n                            0, winreg.KEY_READ)\n    install_date = winreg.QueryValueEx(subkey, \"InstallDate\")\n    self.SendReply(rdfvalue.RDFDatetime.FromSecondsSinceEpoch(install_date[0]))", "response": "Estimate the install date of this system."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef Run(self, unused_arg):\n    logging.debug(\"Disabling service\")\n\n    win32serviceutil.ChangeServiceConfig(\n        None,\n        config.CONFIG[\"Nanny.service_name\"],\n        startType=win32service.SERVICE_DISABLED)\n    svc_config = QueryService(config.CONFIG[\"Nanny.service_name\"])\n    if svc_config[1] == win32service.SERVICE_DISABLED:\n      logging.info(\"Disabled service successfully\")\n      self.SendReply(rdf_protodict.DataBlob(string=\"Service disabled.\"))\n    else:\n      self.SendReply(\n          rdf_protodict.DataBlob(string=\"Service failed to disable.\"))", "response": "This kills us with no cleanups."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef OpenKey(key, sub_key):\n  regopenkeyex = advapi32[\"RegOpenKeyExW\"]\n  regopenkeyex.restype = ctypes.c_long\n  regopenkeyex.argtypes = [\n      ctypes.c_void_p, ctypes.c_wchar_p, ctypes.c_ulong, ctypes.c_ulong,\n      ctypes.POINTER(ctypes.c_void_p)\n  ]\n\n  new_key = KeyHandle()\n  # Don't use KEY_WOW64_64KEY (0x100) since it breaks on Windows 2000\n  rc = regopenkeyex(\n      key.handle, sub_key, 0, KEY_READ,\n      ctypes.cast(\n          ctypes.byref(new_key.handle), ctypes.POINTER(ctypes.c_void_p)))\n  if rc != ERROR_SUCCESS:\n    raise ctypes.WinError(2)\n\n  return new_key", "response": "This calls the Windows OpenKeyEx function in a Unicode safe way."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nconvert a Windows Registry value to the corresponding Python data type.", "response": "def _Reg2Py(data, size, data_type):\n  \"\"\"Converts a Windows Registry value to the corresponding Python data type.\"\"\"\n  if data_type == winreg.REG_DWORD:\n    if size == 0:\n      return 0\n    # DWORD is an unsigned 32-bit integer, see:\n    # https://docs.microsoft.com/en-us/openspecs/windows_protocols/ms-dtyp/262627d8-3418-4627-9218-4ffe110850b2\n    return ctypes.cast(data, ctypes.POINTER(ctypes.c_uint32)).contents.value\n  elif data_type == winreg.REG_SZ or data_type == winreg.REG_EXPAND_SZ:\n    return ctypes.wstring_at(data, size // 2).rstrip(u\"\\x00\")\n  elif data_type == winreg.REG_MULTI_SZ:\n    return ctypes.wstring_at(data, size // 2).rstrip(u\"\\x00\").split(u\"\\x00\")\n  else:\n    if size == 0:\n      return None\n    return ctypes.string_at(data, size)"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nlist the names of all keys and values.", "response": "def ListNames(self):\n    \"\"\"List the names of all keys and values.\"\"\"\n\n    # TODO: This check is flawed, because the current definition of\n    # \"IsDirectory\" is the negation of \"is a file\". One registry path can\n    # actually refer to a key (\"directory\"), a value of the same name (\"file\")\n    # and the default value of the key at the same time.\n    if not self.IsDirectory():\n      return\n\n    # Handle the special case where no hive is specified and just list the hives\n    if self.hive is None:\n      for name in dir(winreg):\n        if name.startswith(\"HKEY_\"):\n          yield name\n\n      return\n\n    try:\n      with OpenKey(self.hive, self.local_path) as key:\n        (self.number_of_keys, self.number_of_values,\n         self.last_modified) = QueryInfoKey(key)\n\n        found_keys = set()\n\n        # First keys\n        for i in range(self.number_of_keys):\n          try:\n            key_name = EnumKey(key, i)\n            found_keys.add(key_name)\n            yield key_name\n          except OSError:\n            pass\n\n        # Now Values\n        for i in range(self.number_of_values):\n          try:\n            name, unused_value, unused_value_type = EnumValue(key, i)\n\n            # A key might contain a sub-key and value of the same name. Do not\n            # yield the same name twice in this case. With only the name,\n            # the caller cannot differentiate between a key and a value anyway.\n            if name not in found_keys:\n              yield name\n          except OSError:\n            pass\n\n    except OSError as e:\n      raise IOError(\"Unable to list key %s: %s\" % (self.key_name, e))"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef CreateAFF4Object(stat_response, client_id_urn, mutation_pool, token=None):\n\n  urn = stat_response.pathspec.AFF4Path(client_id_urn)\n\n  if stat.S_ISDIR(stat_response.st_mode):\n    ftype = standard.VFSDirectory\n  else:\n    ftype = aff4_grr.VFSFile\n\n  with aff4.FACTORY.Create(\n      urn, ftype, mode=\"w\", mutation_pool=mutation_pool, token=token) as fd:\n    fd.Set(fd.Schema.STAT(stat_response))\n    fd.Set(fd.Schema.PATHSPEC(stat_response.pathspec))", "response": "This creates a File or a Directory from a stat response."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nfilter out duplicates from passed PathInfo objects.", "response": "def _FilterOutPathInfoDuplicates(path_infos):\n  \"\"\"Filters out duplicates from passed PathInfo objects.\n\n  Args:\n    path_infos: An iterable with PathInfo objects.\n\n  Returns:\n    A list of PathInfo objects with duplicates removed. Duplicates are\n    removed following this logic: they're sorted by (ctime, mtime, atime,\n    inode number) in the descending order and then the first one is taken\n    and the others are dropped.\n  \"\"\"\n  pi_dict = {}\n\n  for pi in path_infos:\n    path_key = (pi.path_type, pi.GetPathID())\n    pi_dict.setdefault(path_key, []).append(pi)\n\n  def _SortKey(pi):\n    return (\n        pi.stat_entry.st_ctime,\n        pi.stat_entry.st_mtime,\n        pi.stat_entry.st_atime,\n        pi.stat_entry.st_ino,\n    )\n\n  for pi_values in pi_dict.values():\n    if len(pi_values) > 1:\n      pi_values.sort(key=_SortKey, reverse=True)\n\n  return [v[0] for v in pi_dict.values()]"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef WriteStatEntries(stat_entries, client_id, mutation_pool, token=None):\n\n  for stat_response in stat_entries:\n    if stat_response.pathspec.last.stream_name:\n      # This is an ads. In that case we always need to create a file or\n      # we won't be able to access the data. New clients send the correct mode\n      # already but to make sure, we set this to a regular file anyways.\n      # Clear all file type bits:\n      stat_response.st_mode &= ~stat_type_mask\n      stat_response.st_mode |= stat.S_IFREG\n\n  if data_store.AFF4Enabled():\n    for stat_entry in stat_entries:\n      CreateAFF4Object(\n          stat_entry,\n          client_id_urn=rdf_client.ClientURN(client_id),\n          mutation_pool=mutation_pool,\n          token=token)\n\n  if data_store.RelationalDBEnabled():\n    path_infos = [rdf_objects.PathInfo.FromStatEntry(s) for s in stat_entries]\n    # NOTE: TSK may return duplicate entries. This is may be either due to\n    # a bug in TSK implementation, or due to the fact that TSK is capable\n    # of returning deleted files information. Our VFS data model only supports\n    # storing multiple versions of the files when we collect the versions\n    # ourselves. At the moment we can't store multiple versions of the files\n    # \"as returned by TSK\".\n    #\n    # Current behaviour is to simply drop excessive version before the\n    # WritePathInfo call. This way files returned by TSK will still make it\n    # into the flow's results, but not into the VFS data.\n    data_store.REL_DB.WritePathInfos(client_id,\n                                     _FilterOutPathInfoDuplicates(path_infos))", "response": "Writes stat entries into the AFF4 data store."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nstart the Glob. This is the main entry point for this flow mixin. First we convert the pattern into regex components, and then we interpolate each component. Finally, we generate a cartesian product of all combinations. Args: paths: A list of GlobExpression instances. pathtype: The pathtype to use for creating pathspecs. root_path: A pathspec where to start searching from. process_non_regular_files: Work with all kinds of files - not only with regular ones. collect_ext_attrs: Whether to gather information about file extended attributes.", "response": "def GlobForPaths(self,\n                   paths,\n                   pathtype=\"OS\",\n                   root_path=None,\n                   process_non_regular_files=False,\n                   collect_ext_attrs=False):\n    \"\"\"Starts the Glob.\n\n    This is the main entry point for this flow mixin.\n\n    First we convert the pattern into regex components, and then we\n    interpolate each component. Finally, we generate a cartesian product of all\n    combinations.\n\n    Args:\n      paths: A list of GlobExpression instances.\n      pathtype: The pathtype to use for creating pathspecs.\n      root_path: A pathspec where to start searching from.\n      process_non_regular_files: Work with all kinds of files - not only with\n        regular ones.\n      collect_ext_attrs: Whether to gather information about file extended\n        attributes.\n    \"\"\"\n    patterns = []\n\n    if not paths:\n      # Nothing to do.\n      return\n\n    self.state.pathtype = pathtype\n    self.state.root_path = root_path\n    self.state.process_non_regular_files = process_non_regular_files\n    self.state.collect_ext_attrs = collect_ext_attrs\n\n    # Transform the patterns by substitution of client attributes. When the\n    # client has multiple values for an attribute, this generates multiple\n    # copies of the pattern, one for each variation. e.g.:\n    # /home/%%Usernames%%/* -> [ /home/user1/*, /home/user2/* ]\n    for path in paths:\n      patterns.extend(\n          path.Interpolate(knowledge_base=self.client_knowledge_base))\n\n    # Sort the patterns so that if there are files whose paths conflict with\n    # directory paths, the files get handled after the conflicting directories\n    # have been added to the component tree.\n    patterns.sort(key=len, reverse=True)\n\n    # Expand each glob pattern into a list of components. A component is either\n    # a wildcard or a literal component.\n    # e.g. /usr/lib/*.exe -> ['/usr/lib', '.*.exe']\n\n    # We build a tree for each component such that duplicated components are\n    # merged. We do not need to reissue the same client requests for the same\n    # components. For example, the patterns:\n    # '/home/%%Usernames%%*' -> {'/home/': {\n    #      'syslog.*\\\\Z(?ms)': {}, 'test.*\\\\Z(?ms)': {}}}\n    # Note: The component tree contains serialized pathspecs in dicts.\n    for pattern in patterns:\n      # The root node.\n      curr_node = self.state.component_tree\n\n      components = self.ConvertGlobIntoPathComponents(pattern)\n      for i, curr_component in enumerate(components):\n        is_last_component = i == len(components) - 1\n        next_node = curr_node.get(curr_component.SerializeToString(), {})\n        if is_last_component and next_node:\n          # There is a conflicting directory already existing in the tree.\n          # Replace the directory node with a node representing this file.\n          curr_node[curr_component.SerializeToString()] = {}\n        else:\n          curr_node = curr_node.setdefault(curr_component.SerializeToString(),\n                                           {})\n\n    root_path = next(iterkeys(self.state.component_tree))\n    self.CallStateInline(\n        messages=[None],\n        next_state=\"ProcessEntry\",\n        request_data=dict(component_path=[root_path]))"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef GlobReportMatch(self, stat_response):\n    # By default write the stat_response to the AFF4 VFS.\n    with data_store.DB.GetMutationPool() as pool:\n      WriteStatEntries([stat_response],\n                       client_id=self.client_id,\n                       mutation_pool=pool,\n                       token=self.token)", "response": "Called when we have found a match a StatEntry."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef FindNode(self, component_path):\n    # Find the node that the component path is referring to.\n    node = self.state.component_tree\n    for component in component_path:\n      node = node[component]\n\n    return node", "response": "Find the node in the component tree from component_path."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _MatchPath(self, pathspec, response):\n    to_match = response.pathspec.Basename()\n    if pathspec.path_options == rdf_paths.PathSpec.Options.CASE_INSENSITIVE:\n      return to_match.lower() == pathspec.path.lower()\n    elif pathspec.path_options == rdf_paths.PathSpec.Options.CASE_LITERAL:\n      return to_match == pathspec.path\n    elif pathspec.path_options == rdf_paths.PathSpec.Options.REGEX:\n      return bool(re.match(pathspec.path, to_match, flags=re.IGNORECASE))\n    elif pathspec.path_options == rdf_paths.PathSpec.Options.RECURSIVE:\n      return True\n    raise ValueError(\"Unknown Pathspec type.\")", "response": "Returns True if the responses match the pathspec."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef ProcessEntry(self, responses):\n    if not responses.success:\n      return\n\n    # The Find client action does not return a StatEntry but a\n    # FindSpec. Normalize to a StatEntry.\n    stat_responses = [\n        r.hit if isinstance(r, rdf_client_fs.FindSpec) else r for r in responses\n    ]\n\n    # If this was a pure path matching call without any regex / recursion, we\n    # know exactly which node in the component tree we have to process next and\n    # get it from the component_path. If this was a regex match though, we\n    # sent the client a combined regex that matches all nodes in order to save\n    # round trips and client processing time. In that case we only get the\n    # base node and have to check for all subnodes if the response actually\n    # matches that subnode before we continue processing.\n    component_path = responses.request_data.get(\"component_path\")\n    if component_path is not None:\n\n      for response in stat_responses:\n        self._ProcessResponse(response, [component_path])\n\n    else:\n      # This is a combined match.\n      base_path = responses.request_data[\"base_path\"]\n      base_node = self.FindNode(base_path)\n      for response in stat_responses:\n        matching_components = []\n        for next_node in base_node:\n          pathspec = rdf_paths.PathSpec.FromSerializedString(next_node)\n\n          if self._MatchPath(pathspec, response):\n            matching_path = base_path + [next_node]\n            matching_components.append(matching_path)\n\n        if matching_components:\n          self._ProcessResponse(\n              response, matching_components, base_wildcard=True)", "response": "Processes the responses from the client."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nquery the database file.", "response": "def Query(self, sql_query):\n    \"\"\"Query the database file.\"\"\"\n    results = {}\n\n    try:\n      self._connection = sqlite.connect(self.name)\n      self._connection.execute(\"PRAGMA journal_mode=%s\" % self.journal_mode)\n      self._cursor = self._connection.cursor()\n      results = self._cursor.execute(sql_query).fetchall()\n\n    except sqlite.Error as error_string:\n      logging.warning(\"SQLite error %s\", error_string)\n\n    return results"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef GetClientURNsForHostnames(hostnames, token=None):\n\n  if data_store.RelationalDBEnabled():\n    index = ClientIndex()\n  else:\n    index = CreateClientIndex(token=token)\n\n  keywords = set()\n  for hostname in hostnames:\n    if hostname.startswith(\"host:\"):\n      keywords.add(hostname)\n    else:\n      keywords.add(\"host:%s\" % hostname)\n  results = index.ReadClientPostingLists(keywords)\n\n  result = {}\n  for keyword, hits in iteritems(results):\n    result[keyword[len(\"host:\"):]] = hits\n  return result", "response": "Gets all GRR client_ids for a given list of hostnames or FQDNs."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef GetMostRecentClient(client_list, token=None):\n  last = rdfvalue.RDFDatetime(0)\n  client_urn = None\n  for client in aff4.FACTORY.MultiOpen(client_list, token=token):\n    client_last = client.Get(client.Schema.LAST)\n    if client_last > last:\n      last = client_last\n      client_urn = client.urn\n  return client_urn", "response": "Returns the urn of the most recent client from a list of clients."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef BulkLabel(label, hostnames, owner=None, token=None, client_index=None):\n  if client_index is None:\n    client_index = CreateClientIndex(token=token)\n\n  fqdns = set()\n  for hostname in hostnames:\n    fqdns.add(hostname.lower())\n\n  labelled_urns = client_index.LookupClients([\"+label:%s\" % label])\n\n  # If a labelled client fqdn isn't in the set of target fqdns remove the label.\n  # Labelled clients with a target fqdn need no action and are removed from the\n  # set of target fqdns.\n  for client in aff4.FACTORY.MultiOpen(\n      labelled_urns, token=token, aff4_type=aff4_grr.VFSGRRClient, mode=\"rw\"):\n    fqdn = utils.SmartStr(client.Get(\"FQDN\")).lower()\n    if fqdn not in fqdns:\n      client_index.RemoveClientLabels(client)\n      client.RemoveLabel(label, owner=owner)\n      client.Flush()\n      client_index.AddClient(client)\n    else:\n      fqdns.discard(fqdn)\n\n  # The residual set of fqdns needs labelling.\n  # Get the latest URN for these clients and open them to add the label.\n  urns = []\n  keywords = [\"+host:%s\" % fqdn for fqdn in fqdns]\n  for client_list in client_index.ReadClientPostingLists(keywords).itervalues():\n    for client_id in client_list:\n      urns.append(rdfvalue.RDFURN(client_id))\n\n  for client in aff4.FACTORY.MultiOpen(\n      urns, token=token, aff4_type=aff4_grr.VFSGRRClient, mode=\"rw\"):\n    client.AddLabel(label, owner=owner)\n    client.Flush()\n    client_index.AddClient(client)", "response": "Assign a label to a group of clients based on hostname."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nsearching for clients associated with keywords.", "response": "def LookupClients(self, keywords):\n    \"\"\"Returns a list of client URNs associated with keywords.\n\n    Args:\n      keywords: The list of keywords to search by.\n\n    Returns:\n      A list of client URNs.\n\n    Raises:\n      ValueError: A string (single keyword) was passed instead of an iterable.\n    \"\"\"\n    if isinstance(keywords, string_types):\n      raise ValueError(\n          \"Keywords should be an iterable, not a string (got %s).\" % keywords)\n\n    start_time, end_time, filtered_keywords, unversioned_keywords = (\n        self._AnalyzeKeywords(keywords))\n\n    last_seen_map = None\n    if unversioned_keywords:\n      last_seen_map = {}\n\n    # TODO(user): Make keyword index datetime aware so that\n    # AsMicrosecondsSinceEpoch is unnecessary.\n\n    raw_results = self.Lookup(\n        list(map(self._NormalizeKeyword, filtered_keywords)),\n        start_time=start_time.AsMicrosecondsSinceEpoch(),\n        end_time=end_time.AsMicrosecondsSinceEpoch(),\n        last_seen_map=last_seen_map)\n    if not raw_results:\n      return []\n\n    if unversioned_keywords:\n      universal_last_seen_raw = {}\n      self.ReadPostingLists(\n          list(map(self._NormalizeKeyword, raw_results)),\n          start_time=start_time.AsMicrosecondsSinceEpoch(),\n          end_time=end_time.AsMicrosecondsSinceEpoch(),\n          last_seen_map=universal_last_seen_raw)\n\n      universal_last_seen = {}\n      for (_, client_id), ts in iteritems(universal_last_seen_raw):\n        universal_last_seen[client_id] = ts\n\n      old_results = set()\n      for keyword in unversioned_keywords:\n        for result in raw_results:\n          if last_seen_map[(keyword, result)] < universal_last_seen[result]:\n            old_results.add(result)\n      raw_results -= old_results\n\n    return [rdf_client.ClientURN(result) for result in raw_results]"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nread all clients associated with any of the given keywords.", "response": "def ReadClientPostingLists(self, keywords):\n    \"\"\"Looks up all clients associated with any of the given keywords.\n\n    Args:\n      keywords: A list of keywords we are interested in.\n\n    Returns:\n      A dict mapping each keyword to a list of matching clients.\n    \"\"\"\n\n    start_time, end_time, filtered_keywords, _ = self._AnalyzeKeywords(keywords)\n\n    # TODO(user): Make keyword index datetime aware so that\n    # AsMicrosecondsSinceEpoch is unecessary.\n    return self.ReadPostingLists(\n        filtered_keywords,\n        start_time=start_time.AsMicrosecondsSinceEpoch(),\n        end_time=end_time.AsMicrosecondsSinceEpoch())"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef AnalyzeClient(self, client):\n\n    client_id = self._ClientIdFromURN(client.urn)\n\n    # Start with both the client id itself, and a universal keyword, used to\n    # find all clients.\n    #\n    # TODO(user): Remove the universal keyword once we have a better way\n    # to do this, i.e., once we have a storage library which can list all\n    # clients directly.\n\n    keywords = [self._NormalizeKeyword(client_id), \".\"]\n\n    def TryAppend(prefix, keyword):\n      precondition.AssertType(prefix, Text)\n      if keyword:\n        keyword_string = self._NormalizeKeyword(Text(keyword))\n        keywords.append(keyword_string)\n        if prefix:\n          keywords.append(prefix + \":\" + keyword_string)\n\n    def TryAppendPrefixes(prefix, keyword, delimiter):\n      if keyword is None:\n        return 0\n      TryAppend(prefix, keyword)\n      segments = keyword.split(delimiter)\n      for i in range(1, len(segments)):\n        TryAppend(prefix, delimiter.join(segments[0:i]))\n      return len(segments)\n\n    def TryAppendIP(ip):\n      TryAppend(\"ip\", ip)\n      # IP4v?\n      if TryAppendPrefixes(\"ip\", str(ip), \".\") == 4:\n        return\n      # IP6v?\n      TryAppendPrefixes(\"ip\", str(ip), \":\")\n\n    def TryAppendMac(mac):\n      TryAppend(\"mac\", mac)\n      if len(mac) == 12:\n        # If looks like a mac address without \":\" symbols, also add the keyword\n        # with them.\n        TryAppend(\"mac\", \":\".join([mac[i:i + 2] for i in range(0, 12, 2)]))\n\n    s = client.Schema\n    TryAppend(\"host\", client.Get(s.HOSTNAME))\n    TryAppendPrefixes(\"host\", client.Get(s.HOSTNAME), \"-\")\n    TryAppend(\"host\", client.Get(s.FQDN))\n    TryAppendPrefixes(\"host\", client.Get(s.FQDN), \".\")\n    TryAppend(\"\", client.Get(s.SYSTEM))\n    TryAppend(\"\", client.Get(s.UNAME))\n    TryAppend(\"\", client.Get(s.OS_RELEASE))\n    TryAppend(\"\", client.Get(s.OS_VERSION))\n    TryAppend(\"\", client.Get(s.KERNEL))\n    TryAppend(\"\", client.Get(s.ARCH))\n\n    kb = client.Get(s.KNOWLEDGE_BASE)\n    if kb:\n      for user in kb.users:\n        TryAppend(\"user\", user.username)\n        TryAppend(\"\", user.full_name)\n        if user.full_name:\n          for name in user.full_name.split():\n            # full_name often includes nicknames and similar, wrapped in\n            # punctuation, e.g. \"Thomas 'TJ' Jones\". We remove the most common\n            # wrapping characters.\n            TryAppend(\"\", name.strip(\"\\\"'()\"))\n\n    for username in client.Get(s.USERNAMES, []):\n      TryAppend(\"user\", username)\n\n    for interface in client.Get(s.INTERFACES, []):\n      if interface.mac_address:\n        TryAppendMac(interface.mac_address.human_readable_address)\n      for ip in interface.GetIPAddresses():\n        TryAppendIP(ip)\n\n    # We should have all mac and ip addresses already, but some test data only\n    # has it attached directly, so just in case we look there also.\n    if client.Get(s.MAC_ADDRESS):\n      for mac in str(client.Get(s.MAC_ADDRESS)).split(\"\\n\"):\n        TryAppendMac(mac)\n    ip_list = client.Get(s.HOST_IPS, \"\")\n    for ip in str(ip_list).split(\"\\n\"):\n      TryAppendIP(ip)\n\n    client_info = client.Get(s.CLIENT_INFO)\n    if client_info:\n      TryAppend(\"client\", client_info.client_name)\n      TryAppend(\"client\", client_info.client_version)\n      if client_info.labels:\n        for label in client_info.labels:\n          TryAppend(\"label\", label)\n\n    for label in client.GetLabelsNames():\n      TryAppend(\"label\", label)\n\n    return client_id, keywords", "response": "Analyze a client record and returns the client_id and keywords."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nadds a client to the index.", "response": "def AddClient(self, client):\n    \"\"\"Adds a client to the index.\n\n    Args:\n      client: A VFSGRRClient record to add or update.\n    \"\"\"\n\n    client_id, keywords = self.AnalyzeClient(client)\n    self.AddKeywordsForName(client_id, keywords)"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nremoves all labels for a given client object.", "response": "def RemoveClientLabels(self, client):\n    \"\"\"Removes all labels for a given client object.\n\n    Args:\n      client: A VFSGRRClient record.\n    \"\"\"\n    keywords = []\n    for label in client.GetLabelsNames():\n      keyword = self._NormalizeKeyword(utils.SmartStr(label))\n      # This might actually delete a keyword with the same name as the label (if\n      # there is one). Usually the client keywords will be rebuilt after the\n      # deletion of the old labels though, so this can only destroy historic\n      # index data; normal search functionality will not be affected.\n      keywords.append(keyword)\n      keywords.append(\"label:%s\" % keyword)\n\n    self.RemoveKeywordsForName(self._ClientIdFromURN(client.urn), keywords)"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nextracts a start time from a list of keywords if present.", "response": "def _AnalyzeKeywords(self, keywords):\n    \"\"\"Extracts a start time from a list of keywords if present.\"\"\"\n    start_time = rdfvalue.RDFDatetime.Now() - rdfvalue.Duration(\"180d\")\n    filtered_keywords = []\n\n    for k in keywords:\n      if k.startswith(self.START_TIME_PREFIX):\n        try:\n          start_time = rdfvalue.RDFDatetime.FromHumanReadable(\n              k[self.START_TIME_PREFIX_LEN:])\n        except ValueError:\n          pass\n      else:\n        filtered_keywords.append(k)\n\n    if not filtered_keywords:\n      filtered_keywords.append(\".\")\n\n    return start_time, filtered_keywords"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef LookupClients(self, keywords):\n    if isinstance(keywords, string_types):\n      raise ValueError(\n          \"Keywords should be an iterable, not a string (got %s).\" % keywords)\n\n    start_time, filtered_keywords = self._AnalyzeKeywords(keywords)\n\n    keyword_map = data_store.REL_DB.ListClientsForKeywords(\n        list(map(self._NormalizeKeyword, filtered_keywords)),\n        start_time=start_time)\n\n    results = itervalues(keyword_map)\n    relevant_set = set(next(results))\n\n    for hits in results:\n      relevant_set &= set(hits)\n\n      if not relevant_set:\n        return []\n\n    return sorted(relevant_set)", "response": "Searches for clients associated with a list of keywords."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef ReadClientPostingLists(self, keywords):\n\n    start_time, filtered_keywords = self._AnalyzeKeywords(keywords)\n\n    return data_store.REL_DB.ListClientsForKeywords(\n        filtered_keywords, start_time=start_time)", "response": "Reads all clients associated with any of the given keywords."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nanalyzes a client object record and returns a list of keywords related to it.", "response": "def AnalyzeClient(self, client):\n    \"\"\"Finds the client_id and keywords for a client.\n\n    Args:\n      client: A Client object record to find keywords for.\n\n    Returns:\n      A list of keywords related to client.\n    \"\"\"\n\n    # Start with a universal keyword, used to find all clients.\n    #\n    # TODO(user): Remove the universal keyword once we have a better way\n    # to do this, i.e., once we have a storage library which can list all\n    # clients directly.\n\n    keywords = set([\".\"])\n\n    def TryAppend(prefix, keyword):\n      precondition.AssertType(prefix, Text)\n      precondition.AssertType(keyword, Text)\n      if keyword:\n        keyword_string = self._NormalizeKeyword(keyword)\n        keywords.add(keyword_string)\n        if prefix:\n          keywords.add(prefix + \":\" + keyword_string)\n\n    def TryAppendPrefixes(prefix, keyword, delimiter):\n      TryAppend(prefix, keyword)\n      segments = keyword.split(delimiter)\n      for i in range(1, len(segments)):\n        TryAppend(prefix, delimiter.join(segments[0:i]))\n      return len(segments)\n\n    def TryAppendIP(ip):\n      TryAppend(\"ip\", ip)\n      # IP4v?\n      if TryAppendPrefixes(\"ip\", Text(ip), \".\") == 4:\n        return\n      # IP6v?\n      TryAppendPrefixes(\"ip\", Text(ip), \":\")\n\n    def TryAppendMac(mac):\n      TryAppend(\"mac\", mac)\n      if len(mac) == 12:\n        # If looks like a mac address without \":\" symbols, also add the keyword\n        # with them.\n        TryAppend(\"mac\", \":\".join([mac[i:i + 2] for i in range(0, 12, 2)]))\n\n    TryAppend(\"host\", client.knowledge_base.fqdn)\n    host = client.knowledge_base.fqdn.split(\".\", 1)[0]\n    TryAppendPrefixes(\"host\", host, \"-\")\n    TryAppendPrefixes(\"host\", client.knowledge_base.fqdn, \".\")\n    TryAppend(\"\", client.knowledge_base.os)\n    TryAppend(\"\", client.Uname())\n    TryAppend(\"\", client.os_release)\n    TryAppend(\"\", client.os_version)\n    TryAppend(\"\", client.kernel)\n    TryAppend(\"\", client.arch)\n\n    kb = client.knowledge_base\n    if kb:\n      for user in kb.users:\n        TryAppend(\"user\", user.username)\n        TryAppend(\"\", user.full_name)\n        if user.full_name:\n          for name in user.full_name.split():\n            # full_name often includes nicknames and similar, wrapped in\n            # punctuation, e.g. \"Thomas 'TJ' Jones\". We remove the most common\n            # wrapping characters.\n            TryAppend(\"\", name.strip(\"\\\"'()\"))\n\n    for ip in client.GetIPAddresses():\n      TryAppendIP(ip)\n    for mac in client.GetMacAddresses():\n      TryAppendMac(mac)\n\n    client_info = client.startup_info.client_info\n    if client_info:\n      TryAppend(\"client\", client_info.client_name)\n      TryAppend(\"client\", Text(client_info.client_version))\n      if client_info.labels:\n        for label in client_info.labels:\n          TryAppend(\"label\", label)\n\n    return keywords"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef AddClient(self, client):\n    keywords = self.AnalyzeClient(client)\n    keywords.add(self._NormalizeKeyword(client.client_id))\n\n    data_store.REL_DB.AddClientKeywords(client.client_id, keywords)", "response": "Adds a client to the index."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nremove all labels for a given client.", "response": "def RemoveAllClientLabels(self, client_id):\n    \"\"\"Removes all labels for a given client.\n\n    Args:\n      client_id: The client_id.\n    \"\"\"\n    labels_to_remove = set(\n        [l.name for l in data_store.REL_DB.ReadClientLabels(client_id)])\n    self.RemoveClientLabels(client_id, labels_to_remove)"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nremove all labels for a given client.", "response": "def RemoveClientLabels(self, client_id, labels):\n    \"\"\"Removes all labels for a given client.\n\n    Args:\n      client_id: The client_id.\n      labels: A list of labels to remove.\n    \"\"\"\n    for label in labels:\n      keyword = self._NormalizeKeyword(label)\n      # This might actually delete a keyword with the same name as the label (if\n      # there is one).\n      data_store.REL_DB.RemoveClientKeyword(client_id, keyword)\n      data_store.REL_DB.RemoveClientKeyword(client_id, \"label:%s\" % keyword)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nverify the signature of a message.", "response": "def VerifyMessageSignature(self, response_comms, packed_message_list, cipher,\n                             cipher_verified, api_version, remote_public_key):\n    \"\"\"Verifies the message list signature.\n\n    In the server we check that the timestamp is later than the ping timestamp\n    stored with the client. This ensures that client responses can not be\n    replayed.\n\n    Args:\n      response_comms: The raw response_comms rdfvalue.\n      packed_message_list: The PackedMessageList rdfvalue from the server.\n      cipher: The cipher object that should be used to verify the message.\n      cipher_verified: If True, the cipher's signature is not verified again.\n      api_version: The api version we should use.\n      remote_public_key: The public key of the source.\n\n    Returns:\n      An rdf_flows.GrrMessage.AuthorizationState.\n    \"\"\"\n    if (not cipher_verified and\n        not cipher.VerifyCipherSignature(remote_public_key)):\n      stats_collector_instance.Get().IncrementCounter(\n          \"grr_unauthenticated_messages\")\n      return rdf_flows.GrrMessage.AuthorizationState.UNAUTHENTICATED\n\n    try:\n      client_id = cipher.cipher_metadata.source\n      try:\n        client = self.client_cache.Get(client_id)\n      except KeyError:\n        client = aff4.FACTORY.Create(\n            client_id,\n            aff4.AFF4Object.classes[\"VFSGRRClient\"],\n            mode=\"rw\",\n            token=self.token)\n        self.client_cache.Put(client_id, client)\n        stats_collector_instance.Get().SetGaugeValue(\n            \"grr_frontendserver_client_cache_size\", len(self.client_cache))\n\n      ip = response_comms.orig_request.source_ip\n      client.Set(client.Schema.CLIENT_IP(ip))\n\n      # The very first packet we see from the client we do not have its clock\n      remote_time = client.Get(client.Schema.CLOCK) or rdfvalue.RDFDatetime(0)\n      client_time = packed_message_list.timestamp or rdfvalue.RDFDatetime(0)\n\n      # This used to be a strict check here so absolutely no out of\n      # order messages would be accepted ever. Turns out that some\n      # proxies can send your request with some delay even if the\n      # client has already timed out (and sent another request in\n      # the meantime, making the first one out of order). In that\n      # case we would just kill the whole flow as a\n      # precaution. Given the behavior of those proxies, this seems\n      # now excessive and we have changed the replay protection to\n      # only trigger on messages that are more than one hour old.\n\n      if client_time < remote_time - rdfvalue.Duration(\"1h\"):\n        logging.warning(\"Message desynchronized for %s: %s >= %s\", client_id,\n                        remote_time, client_time)\n        # This is likely an old message\n        return rdf_flows.GrrMessage.AuthorizationState.DESYNCHRONIZED\n\n      stats_collector_instance.Get().IncrementCounter(\n          \"grr_authenticated_messages\")\n\n      # Update the client and server timestamps only if the client\n      # time moves forward.\n      if client_time > remote_time:\n        client.Set(client.Schema.CLOCK, client_time)\n        client.Set(client.Schema.PING, rdfvalue.RDFDatetime.Now())\n\n        clock = client_time\n        ping = rdfvalue.RDFDatetime.Now()\n\n        for label in client.Get(client.Schema.LABELS, []):\n          stats_collector_instance.Get().IncrementCounter(\n              \"client_pings_by_label\", fields=[label.name])\n      else:\n        clock = None\n        ping = None\n        logging.warning(\"Out of order message for %s: %s >= %s\", client_id,\n                        remote_time, client_time)\n\n      client.Flush()\n      if data_store.RelationalDBEnabled():\n        source_ip = response_comms.orig_request.source_ip\n        if source_ip:\n          last_ip = rdf_client_network.NetworkAddress(\n              human_readable_address=response_comms.orig_request.source_ip)\n        else:\n          last_ip = None\n\n        if ping or clock or last_ip:\n          try:\n            data_store.REL_DB.WriteClientMetadata(\n                client_id.Basename(),\n                last_ip=last_ip,\n                last_clock=clock,\n                last_ping=ping,\n                fleetspeak_enabled=False)\n          except db.UnknownClientError:\n            pass\n\n    except communicator.UnknownClientCertError:\n      pass\n\n    return rdf_flows.GrrMessage.AuthorizationState.AUTHENTICATED"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nverifies the signature of a message list.", "response": "def VerifyMessageSignature(self, response_comms, packed_message_list, cipher,\n                             cipher_verified, api_version, remote_public_key):\n    \"\"\"Verifies the message list signature.\n\n    In the server we check that the timestamp is later than the ping timestamp\n    stored with the client. This ensures that client responses can not be\n    replayed.\n\n    Args:\n      response_comms: The raw response_comms rdfvalue.\n      packed_message_list: The PackedMessageList rdfvalue from the server.\n      cipher: The cipher object that should be used to verify the message.\n      cipher_verified: If True, the cipher's signature is not verified again.\n      api_version: The api version we should use.\n      remote_public_key: The public key of the source.\n\n    Returns:\n      An rdf_flows.GrrMessage.AuthorizationState.\n    \"\"\"\n    if (not cipher_verified and\n        not cipher.VerifyCipherSignature(remote_public_key)):\n      stats_collector_instance.Get().IncrementCounter(\n          \"grr_unauthenticated_messages\")\n      return rdf_flows.GrrMessage.AuthorizationState.UNAUTHENTICATED\n\n    try:\n      client_id = cipher.cipher_metadata.source.Basename()\n      metadata = data_store.REL_DB.ReadClientMetadata(client_id)\n      client_time = packed_message_list.timestamp or rdfvalue.RDFDatetime(0)\n      update_metadata = True\n\n      # This used to be a strict check here so absolutely no out of\n      # order messages would be accepted ever. Turns out that some\n      # proxies can send your request with some delay even if the\n      # client has already timed out (and sent another request in\n      # the meantime, making the first one out of order). In that\n      # case we would just kill the whole flow as a\n      # precaution. Given the behavior of those proxies, this seems\n      # now excessive and we have changed the replay protection to\n      # only trigger on messages that are more than one hour old.\n      if metadata and metadata.clock:\n        stored_client_time = metadata.clock\n\n        if client_time < stored_client_time - rdfvalue.Duration(\"1h\"):\n          logging.warning(\"Message desynchronized for %s: %s >= %s\", client_id,\n                          stored_client_time, client_time)\n          # This is likely an old message\n          return rdf_flows.GrrMessage.AuthorizationState.DESYNCHRONIZED\n\n        # Update the client and server timestamps only if the client time moves\n        # forward.\n        if client_time < stored_client_time:\n          logging.warning(\"Out of order message for %s: %s > %s\", client_id,\n                          stored_client_time, client_time)\n          update_metadata = False\n\n      stats_collector_instance.Get().IncrementCounter(\n          \"grr_authenticated_messages\")\n\n      for label in data_store.REL_DB.ReadClientLabels(client_id):\n        stats_collector_instance.Get().IncrementCounter(\n            \"client_pings_by_label\", fields=[label.name])\n\n      if not update_metadata:\n        return rdf_flows.GrrMessage.AuthorizationState.AUTHENTICATED\n\n      source_ip = response_comms.orig_request.source_ip\n      if source_ip:\n        last_ip = rdf_client_network.NetworkAddress(\n            human_readable_address=response_comms.orig_request.source_ip)\n      else:\n        last_ip = None\n\n      data_store.REL_DB.WriteClientMetadata(\n          client_id,\n          last_ip=last_ip,\n          last_clock=client_time,\n          last_ping=rdfvalue.RDFDatetime.Now(),\n          fleetspeak_enabled=False)\n\n    except communicator.UnknownClientCertError:\n      pass\n\n    return rdf_flows.GrrMessage.AuthorizationState.AUTHENTICATED"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nprocessing a queue of messages as passed from the client. We basically dispatch all the GrrMessages in the queue to the task scheduler for backend processing. We then retrieve from the TS the messages destined for this client. Args: request_comms: A ClientCommunication rdfvalue with messages sent by the client. source should be set to the client CN. response_comms: A ClientCommunication rdfvalue of jobs destined to this client. Returns: tuple of (source, message_count) where message_count is the number of messages received from the client with common name source.", "response": "def HandleMessageBundles(self, request_comms, response_comms):\n    \"\"\"Processes a queue of messages as passed from the client.\n\n    We basically dispatch all the GrrMessages in the queue to the task scheduler\n    for backend processing. We then retrieve from the TS the messages destined\n    for this client.\n\n    Args:\n       request_comms: A ClientCommunication rdfvalue with messages sent by the\n         client. source should be set to the client CN.\n       response_comms: A ClientCommunication rdfvalue of jobs destined to this\n         client.\n\n    Returns:\n       tuple of (source, message_count) where message_count is the number of\n       messages received from the client with common name source.\n    \"\"\"\n    messages, source, timestamp = self._communicator.DecodeMessages(\n        request_comms)\n\n    now = time.time()\n    if messages:\n      # Receive messages in line.\n      self.ReceiveMessages(source, messages)\n\n    # We send the client a maximum of self.max_queue_size messages\n    required_count = max(0, self.max_queue_size - request_comms.queue_size)\n    tasks = []\n\n    message_list = rdf_flows.MessageList()\n    # Only give the client messages if we are able to receive them in a\n    # reasonable time.\n    if time.time() - now < 10:\n      tasks = self.DrainTaskSchedulerQueueForClient(source, required_count)\n      message_list.job = tasks\n\n    # Encode the message_list in the response_comms using the same API version\n    # the client used.\n    self._communicator.EncodeMessages(\n        message_list,\n        response_comms,\n        destination=source,\n        timestamp=timestamp,\n        api_version=request_comms.api_version)\n\n    return source, len(messages)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ndraining the Task Scheduler queue for a given client.", "response": "def DrainTaskSchedulerQueueForClient(self, client, max_count=None):\n    \"\"\"Drains the client's Task Scheduler queue.\n\n    1) Get all messages in the client queue.\n    2) Sort these into a set of session_ids.\n    3) Use data_store.DB.ResolvePrefix() to query all requests.\n    4) Delete all responses for retransmitted messages (if needed).\n\n    Args:\n       client: The ClientURN object specifying this client.\n       max_count: The maximum number of messages we will issue for the client.\n         If not given, uses self.max_queue_size .\n\n    Returns:\n       The tasks respresenting the messages returned. If we can not send them,\n       we can reschedule them for later.\n    \"\"\"\n    if max_count is None:\n      max_count = self.max_queue_size\n\n    if max_count <= 0:\n      return []\n\n    client = rdf_client.ClientURN(client)\n    start_time = time.time()\n    # Drain the queue for this client\n    if data_store.RelationalDBEnabled():\n      action_requests = data_store.REL_DB.LeaseClientActionRequests(\n          client.Basename(),\n          lease_time=rdfvalue.Duration.FromSeconds(self.message_expiry_time),\n          limit=max_count)\n      result = [\n          rdf_flow_objects.GRRMessageFromClientActionRequest(r)\n          for r in action_requests\n      ]\n    else:\n      new_tasks = queue_manager.QueueManager(token=self.token).QueryAndOwn(\n          queue=client.Queue(),\n          limit=max_count,\n          lease_seconds=self.message_expiry_time)\n\n      initial_ttl = rdf_flows.GrrMessage().task_ttl\n      check_before_sending = []\n      result = []\n      for task in new_tasks:\n        if task.task_ttl < initial_ttl - 1:\n          # This message has been leased before.\n          check_before_sending.append(task)\n        else:\n          result.append(task)\n\n      if check_before_sending:\n        with queue_manager.QueueManager(token=self.token) as manager:\n          status_found = manager.MultiCheckStatus(check_before_sending)\n\n          # All messages that don't have a status yet should be sent again.\n          for task in check_before_sending:\n            if task not in status_found:\n              result.append(task)\n            else:\n              manager.DeQueueClientRequest(task)\n\n    stats_collector_instance.Get().IncrementCounter(\"grr_messages_sent\",\n                                                    len(result))\n    if result:\n      logging.debug(\"Drained %d messages for %s in %s seconds.\", len(result),\n                    client,\n                    time.time() - start_time)\n\n    return result"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nenroling a Fleetspeak - enabled client for use with GRR.", "response": "def EnrolFleetspeakClient(self, client_id):\n    \"\"\"Enrols a Fleetspeak-enabled client for use with GRR.\n\n    Args:\n      client_id: GRR client-id for the client.\n\n    Returns:\n      True if the client is new, and actually got enrolled. This method\n      is a no-op if the client already exists (in which case False is returned).\n    \"\"\"\n    client_urn = rdf_client.ClientURN(client_id)\n\n    # If already enrolled, return.\n    if data_store.RelationalDBEnabled():\n      try:\n        data_store.REL_DB.ReadClientMetadata(client_id)\n        return False\n      except db.UnknownClientError:\n        pass\n    else:\n      if aff4.FACTORY.ExistsWithType(\n          client_urn, aff4_type=aff4_grr.VFSGRRClient, token=self.token):\n        return False\n\n    logging.info(\"Enrolling a new Fleetspeak client: %r\", client_id)\n\n    if data_store.RelationalDBEnabled():\n      now = rdfvalue.RDFDatetime.Now()\n      data_store.REL_DB.WriteClientMetadata(\n          client_id, first_seen=now, fleetspeak_enabled=True, last_ping=now)\n\n    if data_store.AFF4Enabled():\n      # TODO(fleetspeak-team,grr-team): If aff4 isn't reliable enough, we can\n      # catch exceptions from it and forward them to Fleetspeak by failing its\n      # gRPC call. Fleetspeak will then retry with a random, perhaps healthier,\n      # instance of the GRR frontend.\n      with aff4.FACTORY.Create(\n          client_urn,\n          aff4_type=aff4_grr.VFSGRRClient,\n          mode=\"rw\",\n          token=self.token) as client:\n\n        client.Set(client.Schema.FLEETSPEAK_ENABLED, rdfvalue.RDFBool(True))\n\n        index = client_index.CreateClientIndex(token=self.token)\n        index.AddClient(client)\n        if data_store.RelationalDBEnabled():\n          client_obj = rdf_objects.ClientSnapshot(\n              client_id=client_urn.Basename())\n          index = client_index.ClientIndex()\n          index.AddClient(client_obj)\n\n    # Publish the client enrollment message.\n    events.Events.PublishEvent(\"ClientEnrollment\", client_urn, token=self.token)\n    return True"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef ReceiveMessagesRelationalFlows(self, client_id, messages):\n    now = time.time()\n    unprocessed_msgs = []\n    message_handler_requests = []\n    dropped_count = 0\n    for session_id, msgs in iteritems(\n        collection.Group(messages, operator.attrgetter(\"session_id\"))):\n\n      # Remove and handle messages to WellKnownFlows\n      leftover_msgs = self.HandleWellKnownFlows(msgs)\n\n      for msg in leftover_msgs:\n        if (msg.auth_state != msg.AuthorizationState.AUTHENTICATED and\n            msg.session_id != self.unauth_allowed_session_id):\n          dropped_count += 1\n          continue\n\n        if session_id in queue_manager.session_id_map:\n          message_handler_requests.append(\n              rdf_objects.MessageHandlerRequest(\n                  client_id=msg.source.Basename(),\n                  handler_name=queue_manager.session_id_map[session_id],\n                  request_id=msg.response_id,\n                  request=msg.payload))\n        else:\n          unprocessed_msgs.append(msg)\n\n    if dropped_count:\n      logging.info(\"Dropped %d unauthenticated messages for %s\", dropped_count,\n                   client_id)\n\n    if unprocessed_msgs:\n      flow_responses = []\n      for message in unprocessed_msgs:\n        flow_responses.append(\n            rdf_flow_objects.FlowResponseForLegacyResponse(message))\n\n      data_store.REL_DB.WriteFlowResponses(flow_responses)\n\n      for msg in unprocessed_msgs:\n        if msg.type == rdf_flows.GrrMessage.Type.STATUS:\n          stat = rdf_flows.GrrStatus(msg.payload)\n          if stat.status == rdf_flows.GrrStatus.ReturnedStatus.CLIENT_KILLED:\n            # A client crashed while performing an action, fire an event.\n            crash_details = rdf_client.ClientCrash(\n                client_id=client_id,\n                session_id=msg.session_id,\n                backtrace=stat.backtrace,\n                crash_message=stat.error_message,\n                nanny_status=stat.nanny_status,\n                timestamp=rdfvalue.RDFDatetime.Now())\n            events.Events.PublishEvent(\n                \"ClientCrash\", crash_details, token=self.token)\n\n    if message_handler_requests:\n      data_store.REL_DB.WriteMessageHandlerRequests(message_handler_requests)\n\n    logging.debug(\"Received %s messages from %s in %s sec\", len(messages),\n                  client_id,\n                  time.time() - now)", "response": "Receives and processes messages for flows stored in the relational db."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef ReceiveMessages(self, client_id, messages):\n    if data_store.RelationalDBEnabled():\n      return self.ReceiveMessagesRelationalFlows(client_id, messages)\n\n    now = time.time()\n    with queue_manager.QueueManager(token=self.token) as manager:\n      for session_id, msgs in iteritems(\n          collection.Group(messages, operator.attrgetter(\"session_id\"))):\n\n        # Remove and handle messages to WellKnownFlows\n        leftover_msgs = self.HandleWellKnownFlows(msgs)\n\n        unprocessed_msgs = []\n        for msg in leftover_msgs:\n          if (msg.auth_state == msg.AuthorizationState.AUTHENTICATED or\n              msg.session_id == self.unauth_allowed_session_id):\n            unprocessed_msgs.append(msg)\n\n        if len(unprocessed_msgs) < len(leftover_msgs):\n          logging.info(\"Dropped %d unauthenticated messages for %s\",\n                       len(leftover_msgs) - len(unprocessed_msgs), client_id)\n\n        if not unprocessed_msgs:\n          continue\n\n        for msg in unprocessed_msgs:\n          manager.QueueResponse(msg)\n\n        for msg in unprocessed_msgs:\n          # Messages for well known flows should notify even though they don't\n          # have a status.\n          if msg.request_id == 0:\n            manager.QueueNotification(session_id=msg.session_id)\n            # Those messages are all the same, one notification is enough.\n            break\n          elif msg.type == rdf_flows.GrrMessage.Type.STATUS:\n            # If we receive a status message from the client it means the client\n            # has finished processing this request. We therefore can de-queue it\n            # from the client queue. msg.task_id will raise if the task id is\n            # not set (message originated at the client, there was no request on\n            # the server), so we have to check .HasTaskID() first.\n            if msg.HasTaskID():\n              manager.DeQueueClientRequest(msg)\n\n            manager.QueueNotification(\n                session_id=msg.session_id, last_status=msg.request_id)\n\n            stat = rdf_flows.GrrStatus(msg.payload)\n            if stat.status == rdf_flows.GrrStatus.ReturnedStatus.CLIENT_KILLED:\n              # A client crashed while performing an action, fire an event.\n              crash_details = rdf_client.ClientCrash(\n                  client_id=client_id,\n                  session_id=session_id,\n                  backtrace=stat.backtrace,\n                  crash_message=stat.error_message,\n                  nanny_status=stat.nanny_status,\n                  timestamp=rdfvalue.RDFDatetime.Now())\n              events.Events.PublishEvent(\n                  \"ClientCrash\", crash_details, token=self.token)\n\n    logging.debug(\"Received %s messages from %s in %s sec\", len(messages),\n                  client_id,\n                  time.time() - now)", "response": "Receives and processes the messages from the source."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nhandle well known flows.", "response": "def HandleWellKnownFlows(self, messages):\n    \"\"\"Hands off messages to well known flows.\"\"\"\n    msgs_by_wkf = {}\n    result = []\n    for msg in messages:\n      # Regular message - queue it.\n      if msg.response_id != 0:\n        result.append(msg)\n        continue\n\n      # Well known flows:\n      flow_name = msg.session_id.FlowName()\n\n      if flow_name in self.well_known_flows:\n        # This message should be processed directly on the front end.\n        msgs_by_wkf.setdefault(flow_name, []).append(msg)\n\n        # TODO(user): Deprecate in favor of 'well_known_flow_requests'\n        # metric.\n        stats_collector_instance.Get().IncrementCounter(\n            \"grr_well_known_flow_requests\")\n\n        stats_collector_instance.Get().IncrementCounter(\n            \"well_known_flow_requests\", fields=[str(msg.session_id)])\n      else:\n        # Message should be queued to be processed in the backend.\n\n        # Well known flows have a response_id==0, but if we queue up the state\n        # as that it will overwrite some other message that is queued. So we\n        # change it to a random number here.\n        msg.response_id = random.UInt32()\n\n        # Queue the message in the data store.\n        result.append(msg)\n\n    for flow_name, msg_list in iteritems(msgs_by_wkf):\n      wkf = self.well_known_flows[flow_name]\n      wkf.ProcessMessages(msg_list)\n\n    return result"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef ExecuteCommandFromClient(command):\n  cmd = command.cmd\n  args = command.args\n  time_limit = command.time_limit\n\n  res = client_utils_common.Execute(cmd, args, time_limit)\n  (stdout, stderr, status, time_used) = res\n\n  # Limit output to 10MB so our response doesn't get too big.\n  stdout = stdout[:10 * 1024 * 1024]\n  stderr = stderr[:10 * 1024 * 1024]\n\n  yield rdf_client_action.ExecuteResponse(\n      request=command,\n      stdout=stdout,\n      stderr=stderr,\n      exit_status=status,\n      # We have to return microseconds.\n      time_used=int(1e6 * time_used))", "response": "Executes one of the predefined commands."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef StatFSFromClient(args):\n  if platform.system() == \"Windows\":\n    raise RuntimeError(\"os.statvfs not available on Windows\")\n\n  for path in args.path_list:\n\n    try:\n      fd = vfs.VFSOpen(rdf_paths.PathSpec(path=path, pathtype=args.pathtype))\n      st = fd.StatFS()\n      mount_point = fd.GetMountPoint()\n    except (IOError, OSError):\n      continue\n\n    unix = rdf_client_fs.UnixVolume(mount_point=mount_point)\n\n    # On linux pre 2.6 kernels don't have frsize, so we fall back to bsize.\n    # The actual_available_allocation_units attribute is set to blocks\n    # available to the unprivileged user, root may have some additional\n    # reserved space.\n    yield rdf_client_fs.Volume(\n        bytes_per_sector=(st.f_frsize or st.f_bsize),\n        sectors_per_allocation_unit=1,\n        total_allocation_units=st.f_blocks,\n        actual_available_allocation_units=st.f_bavail,\n        unixvolume=unix)", "response": "A generator function that returns a list of rdf_client_fs. UnixVolume objects."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreading a buffer on the client and sends it to the server.", "response": "def Run(self, args):\n    \"\"\"Reads a buffer on the client and sends it to the server.\"\"\"\n    # Make sure we limit the size of our output\n    if args.length > constants.CLIENT_MAX_BUFFER_SIZE:\n      raise RuntimeError(\"Can not read buffers this large.\")\n\n    try:\n      fd = vfs.VFSOpen(args.pathspec, progress_callback=self.Progress)\n\n      fd.Seek(args.offset)\n      offset = fd.Tell()\n\n      data = fd.Read(args.length)\n\n    except (IOError, OSError) as e:\n      self.SetStatus(rdf_flows.GrrStatus.ReturnedStatus.IOERROR, e)\n      return\n\n    # Now return the data to the server\n    self.SendReply(\n        rdf_client.BufferReference(\n            offset=offset, data=data, length=len(data), pathspec=fd.pathspec))"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef Run(self, args):\n    # Make sure we limit the size of our output\n    if args.length > constants.CLIENT_MAX_BUFFER_SIZE:\n      raise RuntimeError(\"Can not read buffers this large.\")\n\n    data = vfs.ReadVFS(\n        args.pathspec,\n        args.offset,\n        args.length,\n        progress_callback=self.Progress)\n    result = rdf_protodict.DataBlob(\n        data=zlib.compress(data),\n        compression=rdf_protodict.DataBlob.CompressionType.ZCOMPRESSION)\n\n    digest = hashlib.sha256(data).digest()\n\n    # Ensure that the buffer is counted against this response. Check network\n    # send limit.\n    self.ChargeBytesToSession(len(data))\n\n    # Now return the data to the server into the special TransferStore well\n    # known flow.\n    self.grr_worker.SendReply(\n        result, session_id=rdfvalue.SessionID(flow_name=\"TransferStore\"))\n\n    # Now report the hash of this blob to our flow as well as the offset and\n    # length.\n    self.SendReply(\n        rdf_client.BufferReference(\n            offset=args.offset, length=len(data), data=digest))", "response": "Reads a buffer on the client and sends it to the server."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef Run(self, args):\n    # Make sure we limit the size of our output\n    if args.length > constants.CLIENT_MAX_BUFFER_SIZE:\n      raise RuntimeError(\"Can not read buffers this large.\")\n\n    data = vfs.ReadVFS(args.pathspec, args.offset, args.length)\n\n    digest = hashlib.sha256(data).digest()\n\n    # Now report the hash of this blob to our flow as well as the offset and\n    # length.\n    self.SendReply(\n        rdf_client.BufferReference(\n            offset=args.offset, length=len(data), data=digest))", "response": "Reads a buffer on the client and sends it to the server."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nwrites the blob to a file and returns its path.", "response": "def WriteBlobToFile(self, request):\n    \"\"\"Writes the blob to a file and returns its path.\"\"\"\n    # First chunk truncates the file, later chunks append.\n    if request.offset == 0:\n      mode = \"w+b\"\n    else:\n      mode = \"r+b\"\n\n    temp_file = tempfiles.CreateGRRTempFile(\n        filename=request.write_path, mode=mode)\n    with temp_file:\n      path = temp_file.name\n      temp_file.seek(0, 2)\n      if temp_file.tell() != request.offset:\n        raise IOError(\"Chunks out of order Error.\")\n\n      # Write the new chunk.\n      temp_file.write(request.executable.data)\n\n    return path"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef CleanUp(self, path):\n    try:\n      if os.path.exists(path):\n        os.remove(path)\n    except (OSError, IOError) as e:\n      logging.info(\"Failed to remove temporary file %s. Err: %s\", path, e)", "response": "Removes the temp file."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef Run(self, args):\n    # Verify the executable blob.\n    args.executable.Verify(\n        config.CONFIG[\"Client.executable_signing_public_key\"])\n\n    path = self.WriteBlobToFile(args)\n\n    # Only actually run the file on the last chunk.\n    if not args.more_data:\n      self.ProcessFile(path, args)\n      self.CleanUp(path)", "response": "Write the blob to disk and process the file."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef Run(self, args):\n    time_start = time.time()\n\n    args.python_code.Verify(\n        config.CONFIG[\"Client.executable_signing_public_key\"])\n\n    # The execed code can assign to this variable if it wants to return data.\n    logging.debug(\"exec for python code %s\", args.python_code.data[0:100])\n\n    context = globals().copy()\n    context[\"py_args\"] = args.py_args.ToDict()\n    context[\"magic_return_str\"] = \"\"\n    # Export the Progress function to allow python hacks to call it.\n    context[\"Progress\"] = self.Progress\n\n    stdout = io.StringIO()\n    with utils.Stubber(sys, \"stdout\", StdOutHook(stdout)):\n      exec(args.python_code.data, context)  # pylint: disable=exec-used\n\n    stdout_output = stdout.getvalue()\n    magic_str_output = context.get(\"magic_return_str\")\n\n    if stdout_output and magic_str_output:\n      output = \"Stdout: %s\\nMagic Str:%s\\n\" % (stdout_output, magic_str_output)\n    else:\n      output = stdout_output or magic_str_output\n\n    time_used = time.time() - time_start\n    # We have to return microseconds.\n    self.SendReply(\n        rdf_client_action.ExecutePythonResponse(\n            time_used=int(1e6 * time_used), return_val=output))", "response": "Runs the python code."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn an iterator over the readable regions for this process.", "response": "def Regions(self,\n              skip_mapped_files=False,\n              skip_shared_regions=False,\n              skip_executable_regions=False,\n              skip_readonly_regions=False):\n    \"\"\"Returns an iterator over the readable regions for this process.\"\"\"\n    try:\n      maps_file = open(\"/proc/\" + str(self.pid) + \"/maps\", \"r\")\n    except OSError as e:\n      raise process_error.ProcessError(e)\n\n    with maps_file:\n      for line in maps_file:\n        m = self.maps_re.match(line)\n        if not m:\n          continue\n        start = int(m.group(1), 16)\n        end = int(m.group(2), 16)\n        region_protec = m.group(3)\n        inode = int(m.group(6))\n\n        if \"r\" in region_protec:\n          if skip_mapped_files and inode != 0:\n            continue\n          if skip_shared_regions and \"s\" in region_protec:\n            continue\n          if skip_executable_regions and \"x\" in region_protec:\n            continue\n          if skip_readonly_regions and \"w\" not in region_protec:\n            continue\n          yield start, end - start"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef CopyFileInZip(from_zip, from_name, to_zip, to_name=None, signer=None):\n  data = from_zip.read(from_name)\n  if to_name is None:\n    to_name = from_name\n  if signer:\n    logging.debug(\"Signing %s\", from_name)\n    data = signer.SignBuffer(data)\n  to_zip.writestr(to_name, data)", "response": "Copy a file from a ZipFile and write it to a new ZipFile."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef CreateNewZipWithSignedLibs(z_in,\n                               z_out,\n                               ignore_files=None,\n                               signer=None,\n                               skip_signing_files=None):\n  \"\"\"Copies files from one zip to another, signing all qualifying files.\"\"\"\n  ignore_files = ignore_files or []\n  skip_signing_files = skip_signing_files or []\n  extensions_to_sign = [\".sys\", \".exe\", \".dll\", \".pyd\"]\n  to_sign = []\n  for template_file in z_in.namelist():\n    if template_file not in ignore_files:\n      extension = os.path.splitext(template_file)[1].lower()\n      if (signer and template_file not in skip_signing_files and\n          extension in extensions_to_sign):\n        to_sign.append(template_file)\n      else:\n        CopyFileInZip(z_in, template_file, z_out)\n\n  temp_files = {}\n  for filename in to_sign:\n    fd, path = tempfile.mkstemp()\n    with os.fdopen(fd, \"wb\") as temp_fd:\n      temp_fd.write(z_in.read(filename))\n    temp_files[filename] = path\n\n  try:\n    signer.SignFiles(itervalues(temp_files))\n  except AttributeError:\n    for f in itervalues(temp_files):\n      signer.SignFile(f)\n\n  for filename, tempfile_path in iteritems(temp_files):\n    z_out.writestr(filename, open(tempfile_path, \"rb\").read())", "response": "Copies files from one zip to another with all signed libraries."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nset the PE subsystem.", "response": "def SetPeSubsystem(fd, console=True):\n  \"\"\"Takes file like obj and returns (offset, value) for the PE subsystem.\"\"\"\n  current_pos = fd.tell()\n  fd.seek(0x3c)  # _IMAGE_DOS_HEADER.e_lfanew\n  header_offset = struct.unpack(\"<I\", fd.read(4))[0]\n  # _IMAGE_NT_HEADERS.OptionalHeader.Subsystem ( 0x18 + 0x44)\n  subsystem_offset = header_offset + 0x5c\n  fd.seek(subsystem_offset)\n  if console:\n    fd.write(b\"\\x03\")\n  else:\n    fd.write(b\"\\x02\")\n  fd.seek(current_pos)"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ngenerates a file from a template.", "response": "def GenerateFile(self, input_filename=None, output_filename=None):\n    \"\"\"Generates a file from a template, interpolating config values.\"\"\"\n    if input_filename is None:\n      input_filename = output_filename + \".in\"\n    if output_filename[-3:] == \".in\":\n      output_filename = output_filename[:-3]\n    logging.debug(\"Generating file %s from %s\", output_filename, input_filename)\n\n    with io.open(input_filename, \"r\") as fd:\n      data = fd.read()\n\n    with io.open(output_filename, \"w\") as fd:\n      fd.write(config.CONFIG.InterpolateValue(data, context=self.context))"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nprepares the build directory.", "response": "def MakeBuildDirectory(self):\n    \"\"\"Prepares the build directory.\"\"\"\n    # Create the build directory and let pyinstaller loose on it.\n    self.build_dir = config.CONFIG.Get(\n        \"PyInstaller.build_dir\", context=self.context)\n    self.work_path = config.CONFIG.Get(\n        \"PyInstaller.workpath_dir\", context=self.context)\n\n    self.CleanDirectory(self.build_dir)\n    self.CleanDirectory(self.work_path)"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nuses pyinstaller to build a client package.", "response": "def BuildWithPyInstaller(self):\n    \"\"\"Use pyinstaller to build a client package.\"\"\"\n    self.CleanDirectory(\n        config.CONFIG.Get(\"PyInstaller.distpath\", context=self.context))\n\n    logging.info(\"Copying pyinstaller support files\")\n    self.spec_file = os.path.join(self.build_dir, \"grr.spec\")\n\n    with open(self.spec_file, \"wb\") as fd:\n      fd.write(config.CONFIG.Get(\"PyInstaller.spec\", context=self.context))\n\n    with open(os.path.join(self.build_dir, \"version.txt\"), \"wb\") as fd:\n      fd.write(config.CONFIG.Get(\"PyInstaller.version\", context=self.context))\n\n    shutil.copy(\n        src=config.CONFIG.Get(\"PyInstaller.icon_path\", context=self.context),\n        dst=os.path.join(self.build_dir, u\"grr.ico\"))\n\n    # We expect the onedir output at this location.\n    self.output_dir = os.path.join(\n        config.CONFIG.Get(\"PyInstaller.distpath\", context=self.context),\n        \"grr-client\")\n\n    # Pyinstaller doesn't handle unicode strings.\n    args = [\n        \"--distpath\",\n        str(config.CONFIG.Get(\"PyInstaller.distpath\", context=self.context)),\n        \"--workpath\",\n        str(\n            config.CONFIG.Get(\"PyInstaller.workpath_dir\",\n                              context=self.context)),\n        str(self.spec_file)\n    ]\n    logging.info(\"Running pyinstaller: %s\", args)\n    PyInstallerMain.run(pyi_args=[utils.SmartStr(x) for x in args])\n\n    # Clear out some crud that pyinstaller includes.\n    for path in [\"tcl\", \"tk\", \"pytz\"]:\n      dir_path = os.path.join(self.output_dir, path)\n      try:\n        shutil.rmtree(dir_path)\n      except OSError:\n        logging.error(\"Unable to remove directory: %s\", dir_path)\n\n      try:\n        os.mkdir(dir_path)\n      except OSError:\n        logging.error(\"Unable to create directory: %s\", dir_path)\n\n      file_path = os.path.join(dir_path, path)\n      try:\n        # Create an empty file so the directories get put in the installers.\n        with open(file_path, \"wb\"):\n          pass\n      except IOError:\n        logging.error(\"Unable to create file: %s\", file_path)\n\n    version_ini = version.VersionPath()\n    shutil.copy(version_ini, os.path.join(self.output_dir, \"version.ini\"))\n\n    with open(os.path.join(self.output_dir, \"build.yaml\"), \"wb\") as fd:\n      self.WriteBuildYaml(fd)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nwriting build. yaml to fd.", "response": "def WriteBuildYaml(self, fd, build_timestamp=True):\n    \"\"\"Write build spec to fd.\"\"\"\n    output = {\n        \"Client.build_environment\":\n            rdf_client.Uname.FromCurrentSystem().signature(),\n        \"Template.build_type\":\n            config.CONFIG.Get(\"ClientBuilder.build_type\", context=self.context),\n        \"Template.version_major\":\n            config.CONFIG.Get(\"Source.version_major\", context=self.context),\n        \"Template.version_minor\":\n            config.CONFIG.Get(\"Source.version_minor\", context=self.context),\n        \"Template.version_revision\":\n            config.CONFIG.Get(\"Source.version_revision\", context=self.context),\n        \"Template.version_release\":\n            config.CONFIG.Get(\"Source.version_release\", context=self.context),\n        \"Template.arch\":\n            config.CONFIG.Get(\"Client.arch\", context=self.context)\n    }\n\n    if build_timestamp:\n      output[\"Client.build_time\"] = rdfvalue.RDFDatetime.Now()\n    else:\n      self.REQUIRED_BUILD_YAML_KEYS.remove(\"Client.build_time\")\n\n    for key, value in iteritems(output):\n      output[key] = str(value)\n\n    output[\"Template.build_context\"] = self.context\n\n    output_keys = set(iterkeys(output))\n    if output_keys != self.REQUIRED_BUILD_YAML_KEYS:\n      raise RuntimeError(\"Bad build.yaml: expected %s, got %s\" %\n                         (self.REQUIRED_BUILD_YAML_KEYS, output_keys))\n    fd.write(yaml.Dump(output).encode(\"utf-8\"))"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncreates the executable template.", "response": "def MakeExecutableTemplate(self, output_file=None):\n    \"\"\"Create the executable template.\n\n    The client is built in two phases. First an executable template is created\n    with the client binaries contained inside a zip file. Then the installation\n    package is created by appending the SFX extractor to this template and\n    writing a config file into the zip file.  This technique allows the\n    client build to be carried out once on the supported platform (e.g.\n    windows with MSVS), but the deployable installer can be build on any\n    platform which supports python.  Subclasses for each OS do the actual\n    work, we just make sure the output directory is set up correctly here.\n\n    Args:\n      output_file: string filename where we will write the template.\n    \"\"\"\n    self.template_file = output_file or config.CONFIG.Get(\n        \"ClientBuilder.template_path\", context=self.context)\n    utils.EnsureDirExists(os.path.dirname(self.template_file))"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef MakeZip(self, input_dir, output_file):\n    logging.info(\"Generating zip template file at %s\", output_file)\n    basename, _ = os.path.splitext(output_file)\n    # TODO(user):pytype: incorrect make_archive() definition in typeshed.\n    # pytype: disable=wrong-arg-types\n    shutil.make_archive(\n        basename, \"zip\", base_dir=\".\", root_dir=input_dir, verbose=True)", "response": "Creates a ZIP archive of the files in the input directory."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef GetClientConfig(self, context, validate=True, deploy_timestamp=True):\n    with utils.TempDirectory() as tmp_dir:\n      # Make sure we write the file in yaml format.\n      filename = os.path.join(\n          tmp_dir,\n          config.CONFIG.Get(\"ClientBuilder.config_filename\", context=context))\n\n      new_config = config.CONFIG.MakeNewConfig()\n      new_config.Initialize(reset=True, data=\"\")\n      new_config.SetWriteBack(filename)\n\n      # Only copy certain sections to the client. We enumerate all\n      # defined options and then resolve those from the config in the\n      # client's context. The result is the raw option as if the\n      # client read our config file.\n      client_context = context[:]\n      while contexts.CLIENT_BUILD_CONTEXT in client_context:\n        client_context.remove(contexts.CLIENT_BUILD_CONTEXT)\n      for descriptor in sorted(config.CONFIG.type_infos, key=lambda x: x.name):\n        if descriptor.name in self.SKIP_OPTION_LIST:\n          continue\n\n        if descriptor.section in self.CONFIG_SECTIONS:\n          value = config.CONFIG.GetRaw(\n              descriptor.name, context=client_context, default=None)\n\n          if value is not None:\n            logging.debug(\"Copying config option to client: %s\",\n                          descriptor.name)\n\n            new_config.SetRaw(descriptor.name, value)\n\n      if deploy_timestamp:\n        deploy_time_string = str(rdfvalue.RDFDatetime.Now())\n        new_config.Set(\"Client.deploy_time\", deploy_time_string)\n      new_config.Write()\n\n      if validate:\n        self.ValidateEndConfig(new_config)\n\n      private_validator = config.CONFIG.Get(\n          \"ClientBuilder.private_config_validator_class\", context=context)\n      if private_validator:\n        try:\n          validator = config_validator_base.PrivateConfigValidator.classes[\n              private_validator]()\n        except KeyError:\n          logging.error(\n              \"Couldn't find config validator class %s, \"\n              \"you probably need to copy it into lib/local\", private_validator)\n          raise\n        validator.ValidateEndConfig(new_config, self.context)\n\n      return io.open(filename, \"r\").read()", "response": "Generates the client config file for inclusion in deployable binaries."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nvalidate the end of a client config.", "response": "def ValidateEndConfig(self, config_obj, errors_fatal=True):\n    \"\"\"Given a generated client config, attempt to check for common errors.\"\"\"\n    errors = []\n\n    if not config.CONFIG[\"Client.fleetspeak_enabled\"]:\n      location = config_obj.Get(\"Client.server_urls\", context=self.context)\n      if not location:\n        errors.append(\"Empty Client.server_urls\")\n\n      for url in location:\n        if not url.startswith(\"http\"):\n          errors.append(\"Bad Client.server_urls specified %s\" % url)\n\n    key_data = config_obj.GetRaw(\n        \"Client.executable_signing_public_key\",\n        default=None,\n        context=self.context)\n    if key_data is None:\n      errors.append(\"Missing Client.executable_signing_public_key.\")\n    elif not key_data.startswith(\"-----BEGIN PUBLIC\"):\n      errors.append(\n          \"Invalid Client.executable_signing_public_key: %s\" % key_data)\n    else:\n      rsa_key = rdf_crypto.RSAPublicKey()\n      rsa_key.ParseFromHumanReadable(key_data)\n\n    if not config.CONFIG[\"Client.fleetspeak_enabled\"]:\n      certificate = config_obj.GetRaw(\n          \"CA.certificate\", default=None, context=self.context)\n      if certificate is None or not certificate.startswith(\"-----BEGIN CERTIF\"):\n        errors.append(\"CA certificate missing from config.\")\n\n    for bad_opt in [\"Client.private_key\"]:\n      if config_obj.Get(bad_opt, context=self.context, default=\"\"):\n        errors.append(\"Client cert in conf, this should be empty at deployment\"\n                      \" %s\" % bad_opt)\n\n    if errors_fatal and errors:\n      for error in errors:\n        logging.error(\"Build Config Error: %s\", error)\n      raise RuntimeError(\"Bad configuration generated. Terminating.\")\n    else:\n      return errors"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nwindow specific config validations.", "response": "def ValidateEndConfig(self, config_obj, errors_fatal=True):\n    \"\"\"Windows specific config validations.\"\"\"\n    errors = super(WindowsClientRepacker, self).ValidateEndConfig(\n        config_obj, errors_fatal=errors_fatal)\n\n    install_dir = config_obj[\"Client.install_path\"]\n    for path in config_obj[\"Client.tempdir_roots\"]:\n      if path.startswith(\"/\"):\n        errors.append(\n            \"Client.tempdir_root %s starts with /, probably has Unix path.\" %\n            path)\n      if not path.startswith(install_dir):\n        errors.append(\n            \"Client.tempdir_root %s is not inside the install_dir %s, this is \"\n            \"a security risk\" % ((path, install_dir)))\n\n    if config_obj.Get(\"Logging.path\").startswith(\"/\"):\n      errors.append(\"Logging.path starts with /, probably has Unix path. %s\" %\n                    config_obj[\"Logging.path\"])\n\n    if \"Windows\\\\\" in config_obj.GetRaw(\"Logging.path\"):\n      errors.append(\"Windows in Logging.path, you probably want \"\n                    \"%(WINDIR|env) instead\")\n\n    if not config_obj[\"Client.binary_name\"].endswith(\".exe\"):\n      errors.append(\"Missing .exe extension on binary_name %s\" %\n                    config_obj[\"Client.binary_name\"])\n\n    if not config_obj[\"Nanny.binary\"].endswith(\".exe\"):\n      errors.append(\"Missing .exe extension on nanny_binary\")\n\n    if errors_fatal and errors:\n      for error in errors:\n        logging.error(\"Build Config Error: %s\", error)\n      raise RuntimeError(\"Bad configuration generated. Terminating.\")\n    else:\n      return errors"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nrepackage the template zip with the installer.", "response": "def MakeDeployableBinary(self, template_path, output_path):\n    \"\"\"Repackage the template zip with the installer.\"\"\"\n    context = self.context + [\"Client Context\"]\n\n    zip_data = io.BytesIO()\n    output_zip = zipfile.ZipFile(\n        zip_data, mode=\"w\", compression=zipfile.ZIP_DEFLATED)\n\n    z_template = zipfile.ZipFile(open(template_path, \"rb\"))\n\n    # Track which files we've copied already.\n    completed_files = [\n        \"grr-client.exe\", \"GRRservice.exe\", \"dbg_grr-client.exe\",\n        \"dbg_GRRservice.exe\"\n    ]\n\n    # Change the name of the main binary to the configured name.\n    client_bin_name = config.CONFIG.Get(\"Client.binary_name\", context=context)\n\n    console_build = config.CONFIG.Get(\"ClientBuilder.console\", context=context)\n    if console_build:\n      client_filename = \"dbg_grr-client.exe\"\n      service_filename = \"dbg_GRRservice.exe\"\n    else:\n      client_filename = \"grr-client.exe\"\n      service_filename = \"GRRservice.exe\"\n\n    bin_name = z_template.getinfo(client_filename)\n    output_zip.writestr(client_bin_name, z_template.read(bin_name))\n\n    CopyFileInZip(z_template, \"grr-client.exe.manifest\", output_zip,\n                  \"%s.manifest\" % client_bin_name)\n    completed_files.append(\"grr-client.exe.manifest\")\n\n    # Change the name of the service binary to the configured name.\n    service_template = z_template.getinfo(service_filename)\n\n    service_bin_name = config.CONFIG.Get(\n        \"Nanny.service_binary_name\", context=context)\n    output_zip.writestr(service_bin_name, z_template.read(service_template))\n\n    if config.CONFIG[\"Client.fleetspeak_enabled\"]:\n      self._GenerateFleetspeakServiceConfig(output_zip)\n\n    if self.signed_template:\n      # If the template libs were already signed we can skip signing\n      CreateNewZipWithSignedLibs(\n          z_template, output_zip, ignore_files=completed_files)\n    else:\n      CreateNewZipWithSignedLibs(\n          z_template,\n          output_zip,\n          ignore_files=completed_files,\n          signer=self.signer)\n    output_zip.close()\n\n    return self.MakeSelfExtractingZip(zip_data.getvalue(), output_path)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _ValidateFleetspeakServiceConfig(self, config_path):\n    with open(config_path, \"rb\") as f:\n      pool = descriptor_pool.DescriptorPool()\n      pool.AddDescriptor(fs_config_pb2.Config.DESCRIPTOR)\n      parsed_config = text_format.Parse(\n          f.read(), fs_system_pb2.ClientServiceConfig(), descriptor_pool=pool)\n      if parsed_config.factory != \"Daemon\":\n        raise BuildError(\n            \"Fleetspeak config does not have the expected factory type.\")\n      daemon_cfg = fs_config_pb2.Config()\n      parsed_config.config.Unpack(daemon_cfg)\n      if not daemon_cfg.argv:\n        raise BuildError(\n            \"Fleetspeak daemon service config does not specify command line \"\n            \"args.\")", "response": "Validates a Fleetspeak service config."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef MakeSelfExtractingZip(self, payload_data, output_path):\n    context = self.context + [\"Client Context\"]\n\n    src_zip = zipfile.ZipFile(io.BytesIO(payload_data), mode=\"r\")\n    zip_data = io.BytesIO()\n    output_zip = zipfile.ZipFile(\n        zip_data, mode=\"w\", compression=zipfile.ZIP_DEFLATED)\n\n    config_file_name = config.CONFIG.Get(\n        \"ClientBuilder.config_filename\", context=context)\n    # Copy the rest of the files from the package to the new zip.\n    for template_file in src_zip.namelist():\n      if template_file != config_file_name:\n        # Avoid writing the config file twice if we're repacking a binary that\n        # has already been run through deployment. We write it in the next step,\n        # so no need to copy over from the original here.\n        CopyFileInZip(src_zip, template_file, output_zip)\n\n    client_config_content = self.GetClientConfig(context)\n\n    output_zip.writestr(\n        config_file_name,\n        client_config_content.encode(\"utf-8\"),\n        compress_type=zipfile.ZIP_STORED)\n\n    # The zip file comment is used by the self extractor to run the installation\n    # script. Comment has to be `bytes` object because `zipfile` module is not\n    # smart enough to properly handle `unicode` objects. We use the `encode`\n    # method instead of `SmartStr` because we expect this option to be an\n    # `unicode` object and in case it is not, we want it to blow up.\n    output_zip.comment = b\"$AUTORUN$>%s\" % config.CONFIG.Get(\n        \"ClientBuilder.autorun_command_line\", context=context).encode(\"utf-8\")\n\n    output_zip.close()\n\n    utils.EnsureDirExists(os.path.dirname(output_path))\n    with open(output_path, \"wb\") as fd:\n      # First write the installer stub\n      stub_data = io.BytesIO()\n      unzipsfx_stub = config.CONFIG.Get(\n          \"ClientBuilder.unzipsfx_stub\", context=context)\n      stub_raw = open(unzipsfx_stub, \"rb\").read()\n\n      # Check stub has been compiled with the requireAdministrator manifest.\n      if b\"level=\\\"requireAdministrator\" not in stub_raw:\n        raise RuntimeError(\"Bad unzip binary in use. Not compiled with the\"\n                           \"requireAdministrator manifest option.\")\n\n      stub_data.write(stub_raw)\n\n      # If in verbose mode, modify the unzip bins PE header to run in console\n      # mode for easier debugging.\n      SetPeSubsystem(\n          stub_data,\n          console=config.CONFIG.Get(\"ClientBuilder.console\", context=context))\n\n      # Now patch up the .rsrc section to contain the payload.\n      end_of_file = zip_data.tell() + stub_data.tell()\n\n      # This is the IMAGE_SECTION_HEADER.Name which is also the start of\n      # IMAGE_SECTION_HEADER.\n      offset_to_rsrc = stub_data.getvalue().find(b\".rsrc\")\n\n      # IMAGE_SECTION_HEADER.PointerToRawData is a 32 bit int.\n      stub_data.seek(offset_to_rsrc + 20)\n      start_of_rsrc_section = struct.unpack(\"<I\", stub_data.read(4))[0]\n\n      # Adjust IMAGE_SECTION_HEADER.SizeOfRawData to span from the old start to\n      # the end of file.\n      stub_data.seek(offset_to_rsrc + 16)\n      stub_data.write(struct.pack(\"<I\", end_of_file - start_of_rsrc_section))\n\n      # Concatenate stub and zip file.\n      out_data = io.BytesIO()\n      out_data.write(stub_data.getvalue())\n      out_data.write(zip_data.getvalue())\n\n      # Then write the actual output file.\n      fd.write(out_data.getvalue())\n\n    if self.signer:\n      self.signer.SignFile(output_path)\n\n    logging.info(\"Deployable binary generated at %s\", output_path)\n\n    return output_path", "response": "Repack the installer into the payload."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef GenerateDPKGFiles(self, template_path):\n\n    # Rename the generated binaries to the correct name.\n    template_binary_dir = os.path.join(template_path, \"dist/debian/grr-client\")\n    package_name = config.CONFIG.Get(\n        \"ClientBuilder.package_name\", context=self.context)\n    target_binary_dir = os.path.join(\n        template_path, \"dist/debian/%s%s\" %\n        (package_name,\n         config.CONFIG.Get(\"ClientBuilder.target_dir\", context=self.context)))\n    if package_name == \"grr-client\":\n      # Need to rename the template path or the move will fail.\n      shutil.move(template_binary_dir, \"%s-template\" % template_binary_dir)\n      template_binary_dir = \"%s-template\" % template_binary_dir\n\n    utils.EnsureDirExists(os.path.dirname(target_binary_dir))\n    shutil.move(template_binary_dir, target_binary_dir)\n\n    shutil.move(\n        os.path.join(target_binary_dir, \"grr-client\"),\n        os.path.join(\n            target_binary_dir,\n            config.CONFIG.Get(\"Client.binary_name\", context=self.context)))\n\n    deb_in_dir = os.path.join(template_path, \"dist/debian/debian.in/\")\n\n    self.GenerateDirectory(deb_in_dir, os.path.join(\n        template_path, \"dist/debian\"), [(\"grr-client\", package_name)])\n\n    # Generate directories for the /usr/sbin link.\n    utils.EnsureDirExists(\n        os.path.join(template_path, \"dist/debian/%s/usr/sbin\" % package_name))\n\n    if config.CONFIG.Get(\"Client.fleetspeak_enabled\", context=self.context):\n      self._GenerateFleetspeakConfig(template_path)\n      shutil.rmtree(deb_in_dir)\n      shutil.rmtree(os.path.join(template_path, \"dist\", \"fleetspeak\"))\n      return\n\n    # Generate the nanny template. This only exists from client version 3.1.2.5\n    # onwards.\n    if config.CONFIG[\"Template.version_numeric\"] >= 3125:\n      self.GenerateFile(\n          os.path.join(target_binary_dir, \"nanny.sh.in\"),\n          os.path.join(target_binary_dir, \"nanny.sh\"))\n\n    # Generate the upstart template.\n    self.GenerateFile(\n        os.path.join(template_path, \"dist/debian/upstart.in/grr-client.conf\"),\n        os.path.join(template_path, \"dist/debian/%s.upstart\" % package_name))\n\n    # Generate the initd template. The init will not run if it detects upstart\n    # is present.\n    self.GenerateFile(\n        os.path.join(template_path, \"dist/debian/initd.in/grr-client\"),\n        os.path.join(template_path, \"dist/debian/%s.init\" % package_name))\n\n    # Generate the systemd unit file.\n    self.GenerateFile(\n        os.path.join(template_path,\n                     \"dist/debian/systemd.in/grr-client.service\"),\n        os.path.join(template_path, \"dist/debian/%s.service\" % package_name))\n\n    # Clean up the template dirs.\n    shutil.rmtree(deb_in_dir)\n    shutil.rmtree(os.path.join(template_path, \"dist/debian/upstart.in\"))\n    shutil.rmtree(os.path.join(template_path, \"dist/debian/initd.in\"))\n    shutil.rmtree(os.path.join(template_path, \"dist/debian/systemd.in\"))", "response": "Generates the files needed by dpkg - buildpackage."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef MakeDeployableBinary(self, template_path, output_path):\n\n    rpmbuild_binary = \"/usr/bin/rpmbuild\"\n    if not os.path.exists(rpmbuild_binary):\n      logging.error(\"rpmbuild not found, unable to repack client.\")\n      return\n\n    with utils.TempDirectory() as tmp_dir:\n      template_dir = os.path.join(tmp_dir, \"dist\")\n      utils.EnsureDirExists(template_dir)\n\n      zf = zipfile.ZipFile(template_path)\n      for name in zf.namelist():\n        dirname = os.path.dirname(name)\n        utils.EnsureDirExists(os.path.join(template_dir, dirname))\n        with open(os.path.join(template_dir, name), \"wb\") as fd:\n          fd.write(zf.read(name))\n\n      # Set up a RPM building environment.\n\n      rpm_root_dir = os.path.join(tmp_dir, \"rpmbuild\")\n\n      rpm_build_dir = os.path.join(rpm_root_dir, \"BUILD\")\n      utils.EnsureDirExists(rpm_build_dir)\n\n      rpm_buildroot_dir = os.path.join(rpm_root_dir, \"BUILDROOT\")\n      utils.EnsureDirExists(rpm_buildroot_dir)\n\n      rpm_rpms_dir = os.path.join(rpm_root_dir, \"RPMS\")\n      utils.EnsureDirExists(rpm_rpms_dir)\n\n      rpm_specs_dir = os.path.join(rpm_root_dir, \"SPECS\")\n      utils.EnsureDirExists(rpm_specs_dir)\n\n      template_binary_dir = os.path.join(tmp_dir, \"dist/rpmbuild/grr-client\")\n\n      target_binary_dir = \"%s%s\" % (\n          rpm_build_dir,\n          config.CONFIG.Get(\"ClientBuilder.target_dir\", context=self.context))\n\n      utils.EnsureDirExists(os.path.dirname(target_binary_dir))\n      try:\n        shutil.rmtree(target_binary_dir)\n      except OSError:\n        pass\n      # TODO(user):pytype: incorrect move() definition in typeshed.\n      # pytype: disable=wrong-arg-types\n      shutil.move(template_binary_dir, target_binary_dir)\n      # pytype: enable=wrong-arg-types\n\n      client_name = config.CONFIG.Get(\"Client.name\", context=self.context)\n      client_binary_name = config.CONFIG.Get(\n          \"Client.binary_name\", context=self.context)\n      if client_binary_name != \"grr-client\":\n        # TODO(user):pytype: incorrect move() definition in typeshed.\n        # pytype: disable=wrong-arg-types\n        shutil.move(\n            os.path.join(target_binary_dir, \"grr-client\"),\n            os.path.join(target_binary_dir, client_binary_name))\n        # pytype: enable=wrong-arg-types\n\n      if config.CONFIG.Get(\"Client.fleetspeak_enabled\", context=self.context):\n        self._GenerateFleetspeakConfig(template_dir, rpm_build_dir)\n        if not config.CONFIG.Get(\n            \"Client.fleetspeak_service_name\", context=self.context):\n          # The Fleetspeak service name is required when generating the RPM\n          # spec file.\n          raise BuildError(\"Client.fleetspeak_service_name is not set.\")\n      else:\n        self._GenerateInitConfigs(template_dir, rpm_build_dir)\n\n      # Generate spec\n      spec_filename = os.path.join(rpm_specs_dir, \"%s.spec\" % client_name)\n      self.GenerateFile(\n          os.path.join(tmp_dir, \"dist/rpmbuild/grr.spec.in\"), spec_filename)\n\n      # Generate prelinking blacklist file\n      prelink_target_filename = os.path.join(rpm_build_dir,\n                                             \"etc/prelink.conf.d\",\n                                             \"%s.conf\" % client_name)\n\n      utils.EnsureDirExists(os.path.dirname(prelink_target_filename))\n      self.GenerateFile(\n          os.path.join(tmp_dir, \"dist/rpmbuild/prelink_blacklist.conf.in\"),\n          prelink_target_filename)\n\n      # Create a client config.\n      client_context = [\"Client Context\"] + self.context\n      client_config_content = self.GetClientConfig(client_context)\n\n      with open(\n          os.path.join(\n              target_binary_dir,\n              config.CONFIG.Get(\n                  \"ClientBuilder.config_filename\", context=self.context)),\n          \"wb\") as fd:\n        fd.write(client_config_content)\n\n      # Set the daemon to executable.\n      os.chmod(os.path.join(target_binary_dir, client_binary_name), 0o755)\n\n      client_arch = config.CONFIG.Get(\"Template.arch\", context=self.context)\n      if client_arch == \"amd64\":\n        client_arch = \"x86_64\"\n\n      command = [\n          rpmbuild_binary, \"--define\", \"_topdir \" + rpm_root_dir, \"--target\",\n          client_arch, \"--buildroot\", rpm_buildroot_dir, \"-bb\", spec_filename\n      ]\n      try:\n        subprocess.check_output(command, stderr=subprocess.STDOUT)\n      except subprocess.CalledProcessError as e:\n        logging.error(\"Error calling %s.\", command)\n        logging.error(e.output)\n        raise\n\n      client_version = config.CONFIG.Get(\n          \"Template.version_string\", context=self.context)\n      rpm_filename = os.path.join(\n          rpm_rpms_dir, client_arch,\n          \"%s-%s-1.%s.rpm\" % (client_name, client_version, client_arch))\n\n      utils.EnsureDirExists(os.path.dirname(output_path))\n      shutil.move(rpm_filename, output_path)\n\n      logging.info(\"Created package %s\", output_path)\n      self.Sign(output_path)\n      return output_path", "response": "This method will create a. rpm file for the client."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _GenerateFleetspeakConfig(self, template_dir, rpm_build_dir):\n    source_config = os.path.join(\n        template_dir, \"fleetspeak\",\n        os.path.basename(\n            config.CONFIG.Get(\n                \"ClientBuilder.fleetspeak_config_path\", context=self.context)))\n    fleetspeak_service_dir = config.CONFIG.Get(\n        \"ClientBuilder.fleetspeak_service_dir\", context=self.context)\n    dest_config_dir = os.path.join(rpm_build_dir, fleetspeak_service_dir[1:])\n    utils.EnsureDirExists(dest_config_dir)\n    dest_config_path = os.path.join(\n        dest_config_dir,\n        config.CONFIG.Get(\n            \"Client.fleetspeak_unsigned_config_fname\", context=self.context))\n    self.GenerateFile(\n        input_filename=source_config, output_filename=dest_config_path)", "response": "Generates a Fleetspeak config for GRR."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ngenerate init - system configs.", "response": "def _GenerateInitConfigs(self, template_dir, rpm_build_dir):\n    \"\"\"Generates init-system configs.\"\"\"\n    client_name = config.CONFIG.Get(\"Client.name\", context=self.context)\n    initd_target_filename = os.path.join(rpm_build_dir, \"etc/init.d\",\n                                         client_name)\n\n    # Generate init.d\n    utils.EnsureDirExists(os.path.dirname(initd_target_filename))\n    self.GenerateFile(\n        os.path.join(template_dir, \"rpmbuild/grr-client.initd.in\"),\n        initd_target_filename)\n\n    # Generate systemd unit\n    if config.CONFIG[\"Template.version_numeric\"] >= 3125:\n      systemd_target_filename = os.path.join(rpm_build_dir,\n                                             \"usr/lib/systemd/system/\",\n                                             \"%s.service\" % client_name)\n\n      utils.EnsureDirExists(os.path.dirname(systemd_target_filename))\n      self.GenerateFile(\n          os.path.join(template_dir, \"rpmbuild/grr-client.service.in\"),\n          systemd_target_filename)"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nbuilds necessary assets from sources.", "response": "def compile_protos():\n  \"\"\"Builds necessary assets from sources.\"\"\"\n  # If there's no makefile, we're likely installing from an sdist,\n  # so there's no need to compile the protos (they should be already\n  # compiled).\n  if not os.path.exists(os.path.join(THIS_DIRECTORY, \"makefile.py\")):\n    return\n\n  # Only compile protobufs if we're inside GRR source tree.\n  subprocess.check_call(\n      [\"python\", \"makefile.py\", \"--clean\"], cwd=THIS_DIRECTORY)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreading a cronjob from the database.", "response": "def ReadCronJobs(self, cronjob_ids=None):\n    \"\"\"Reads a cronjob from the database.\"\"\"\n    if cronjob_ids is None:\n      res = [job.Copy() for job in itervalues(self.cronjobs)]\n\n    else:\n      res = []\n      for job_id in cronjob_ids:\n        try:\n          res.append(self.cronjobs[job_id].Copy())\n        except KeyError:\n          raise db.UnknownCronJobError(\"Cron job with id %s not found.\" %\n                                       job_id)\n\n    for job in res:\n      lease = self.cronjob_leases.get(job.cron_job_id)\n      if lease:\n        job.leased_until, job.leased_by = lease\n    return res"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef UpdateCronJob(self,\n                    cronjob_id,\n                    last_run_status=db.Database.unchanged,\n                    last_run_time=db.Database.unchanged,\n                    current_run_id=db.Database.unchanged,\n                    state=db.Database.unchanged,\n                    forced_run_requested=db.Database.unchanged):\n    \"\"\"Updates run information for an existing cron job.\"\"\"\n    job = self.cronjobs.get(cronjob_id)\n    if job is None:\n      raise db.UnknownCronJobError(\"Cron job %s not known.\" % cronjob_id)\n\n    if last_run_status != db.Database.unchanged:\n      job.last_run_status = last_run_status\n    if last_run_time != db.Database.unchanged:\n      job.last_run_time = last_run_time\n    if current_run_id != db.Database.unchanged:\n      job.current_run_id = current_run_id\n    if state != db.Database.unchanged:\n      job.state = state\n    if forced_run_requested != db.Database.unchanged:\n      job.forced_run_requested = forced_run_requested", "response": "Updates the last_run_status current_run_id and state of an existing cronjob."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ndelete a cronjob along with all its runs.", "response": "def DeleteCronJob(self, cronjob_id):\n    \"\"\"Deletes a cronjob along with all its runs.\"\"\"\n    if cronjob_id not in self.cronjobs:\n      raise db.UnknownCronJobError(\"Cron job %s not known.\" % cronjob_id)\n    del self.cronjobs[cronjob_id]\n    try:\n      del self.cronjob_leases[cronjob_id]\n    except KeyError:\n      pass\n    for job_run in self.ReadCronJobRuns(cronjob_id):\n      del self.cronjob_runs[(cronjob_id, job_run.run_id)]"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef LeaseCronJobs(self, cronjob_ids=None, lease_time=None):\n    leased_jobs = []\n\n    now = rdfvalue.RDFDatetime.Now()\n    expiration_time = now + lease_time\n\n    for job in itervalues(self.cronjobs):\n      if cronjob_ids and job.cron_job_id not in cronjob_ids:\n        continue\n      existing_lease = self.cronjob_leases.get(job.cron_job_id)\n      if existing_lease is None or existing_lease[0] < now:\n        self.cronjob_leases[job.cron_job_id] = (expiration_time,\n                                                utils.ProcessIdString())\n        job = job.Copy()\n        job.leased_until, job.leased_by = self.cronjob_leases[job.cron_job_id]\n        leased_jobs.append(job)\n\n    return leased_jobs", "response": "Leases all available cron jobs."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nmake leased cron jobs available for leasing again.", "response": "def ReturnLeasedCronJobs(self, jobs):\n    \"\"\"Makes leased cron jobs available for leasing again.\"\"\"\n    errored_jobs = []\n\n    for returned_job in jobs:\n      existing_lease = self.cronjob_leases.get(returned_job.cron_job_id)\n      if existing_lease is None:\n        errored_jobs.append(returned_job)\n        continue\n\n      if (returned_job.leased_until != existing_lease[0] or\n          returned_job.leased_by != existing_lease[1]):\n        errored_jobs.append(returned_job)\n        continue\n\n      del self.cronjob_leases[returned_job.cron_job_id]\n\n    if errored_jobs:\n      raise ValueError(\"Some jobs could not be returned: %s\" %\n                       \",\".join(job.cron_job_id for job in errored_jobs))"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef WriteCronJobRun(self, run_object):\n    if run_object.cron_job_id not in self.cronjobs:\n      raise db.UnknownCronJobError(\"Job with id %s not found.\" %\n                                   run_object.cron_job_id)\n\n    clone = run_object.Copy()\n    clone.timestamp = rdfvalue.RDFDatetime.Now()\n    self.cronjob_runs[(clone.cron_job_id, clone.run_id)] = clone", "response": "Writes a cron job run object to the database."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreading all cron job runs for a given job id.", "response": "def ReadCronJobRuns(self, job_id):\n    \"\"\"Reads all cron job runs for a given job id.\"\"\"\n    runs = [\n        run for run in itervalues(self.cronjob_runs)\n        if run.cron_job_id == job_id\n    ]\n    return sorted(runs, key=lambda run: run.started_at, reverse=True)"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreads a single cron job run from the db.", "response": "def ReadCronJobRun(self, job_id, run_id):\n    \"\"\"Reads a single cron job run from the db.\"\"\"\n    for run in itervalues(self.cronjob_runs):\n      if run.cron_job_id == job_id and run.run_id == run_id:\n        return run\n    raise db.UnknownCronJobRunError(\n        \"Run with job id %s and run id %s not found.\" % (job_id, run_id))"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef DeleteOldCronJobRuns(self, cutoff_timestamp):\n    deleted = 0\n    for run in list(itervalues(self.cronjob_runs)):\n      if run.timestamp < cutoff_timestamp:\n        del self.cronjob_runs[(run.cron_job_id, run.run_id)]\n        deleted += 1\n\n    return deleted", "response": "Deletes old cron job runs for a given job id."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nuploads a signed blob into the datastore.", "response": "def UploadSignedConfigBlob(content,\n                           aff4_path,\n                           client_context=None,\n                           limit=None,\n                           token=None):\n  \"\"\"Upload a signed blob into the datastore.\n\n  Args:\n    content: File content to upload.\n    aff4_path: aff4 path to upload to.\n    client_context: The configuration contexts to use.\n    limit: The maximum size of the chunk to use.\n    token: A security token.\n\n  Raises:\n    IOError: On failure to write.\n  \"\"\"\n  if limit is None:\n    limit = config.CONFIG[\"Datastore.maximum_blob_size\"]\n\n  # Get the values of these parameters which apply to the client running on the\n  # target platform.\n  if client_context is None:\n    # Default to the windows client.\n    client_context = [\"Platform:Windows\", \"Client Context\"]\n\n  config.CONFIG.Validate(\n      parameters=\"PrivateKeys.executable_signing_private_key\")\n\n  signing_key = config.CONFIG.Get(\n      \"PrivateKeys.executable_signing_private_key\", context=client_context)\n\n  verification_key = config.CONFIG.Get(\n      \"Client.executable_signing_public_key\", context=client_context)\n\n  signed_binary_utils.WriteSignedBinary(\n      rdfvalue.RDFURN(aff4_path),\n      content,\n      signing_key,\n      public_key=verification_key,\n      chunk_size=limit,\n      token=token)\n\n  logging.info(\"Uploaded to %s\", aff4_path)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _CheckAccess(self, username, subject_id, approval_type):\n    precondition.AssertType(subject_id, Text)\n\n    cache_key = (username, subject_id, approval_type)\n    try:\n      self.acl_cache.Get(cache_key)\n      stats_collector_instance.Get().IncrementCounter(\n          \"approval_searches\", fields=[\"-\", \"cache\"])\n      return True\n    except KeyError:\n      stats_collector_instance.Get().IncrementCounter(\n          \"approval_searches\", fields=[\"-\", \"reldb\"])\n\n    approvals = data_store.REL_DB.ReadApprovalRequests(\n        username, approval_type, subject_id=subject_id, include_expired=False)\n\n    errors = []\n    for approval in approvals:\n      try:\n        approval_checks.CheckApprovalRequest(approval)\n        self.acl_cache.Put(cache_key, True)\n        return\n      except access_control.UnauthorizedAccess as e:\n        errors.append(e)\n\n    subject = approval_checks.BuildLegacySubject(subject_id, approval_type)\n    if not errors:\n      raise access_control.UnauthorizedAccess(\n          \"No approval found.\", subject=subject)\n    else:\n      raise access_control.UnauthorizedAccess(\n          \" \".join(utils.SmartStr(e) for e in errors), subject=subject)", "response": "Checks access to a given subject by a given user."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nchecks whether a given user can access given client.", "response": "def CheckClientAccess(self, username, client_id):\n    \"\"\"Checks whether a given user can access given client.\"\"\"\n    self._CheckAccess(\n        username, str(client_id),\n        rdf_objects.ApprovalRequest.ApprovalType.APPROVAL_TYPE_CLIENT)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef CheckHuntAccess(self, username, hunt_id):\n\n    self._CheckAccess(\n        username, str(hunt_id),\n        rdf_objects.ApprovalRequest.ApprovalType.APPROVAL_TYPE_HUNT)", "response": "Checks whether a given user can access given hunt."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef CheckCronJobAccess(self, username, cron_job_id):\n\n    self._CheckAccess(\n        username, str(cron_job_id),\n        rdf_objects.ApprovalRequest.ApprovalType.APPROVAL_TYPE_CRON_JOB)", "response": "Checks whether a given user can access given cron job."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef CheckIfCanStartClientFlow(self, username, flow_name):\n    del username  # Unused.\n\n    flow_cls = flow.GRRFlow.GetPlugin(flow_name)\n\n    if not flow_cls.category:\n      raise access_control.UnauthorizedAccess(\n          \"Flow %s can't be started via the API.\" % flow_name)", "response": "Checks whether a given user can start a given flow."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nchecks whether the user is an admin.", "response": "def CheckIfUserIsAdmin(self, username):\n    \"\"\"Checks whether the user is an admin.\"\"\"\n\n    user_obj = data_store.REL_DB.ReadGRRUser(username)\n    if user_obj.user_type != user_obj.UserType.USER_TYPE_ADMIN:\n      raise access_control.UnauthorizedAccess(\"User %s is not an admin.\" %\n                                              username)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef Artifacts(self, os_name=None, cpe=None, label=None):\n    hit = lambda x: x[0] == x[1] or not x[0]\n    seq = [(self.os_name, os_name), (self.cpe, cpe), (self.label, label)]\n    return all(map(hit, seq))", "response": "Determines if the conditions applies to the host data."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nsearches the condition for the specified values.", "response": "def Search(self, artifact, os_name=None, cpe=None, label=None):\n    \"\"\"Whether the condition contains the specified values.\n\n    Args:\n      artifact: A string identifier for the artifact.\n      os_name: An OS string.\n      cpe: A CPE string.\n      label: A label string.\n\n    Returns:\n      True if the values match the non-empty query attributes.\n      Empty query attributes are ignored in the comparison.\n    \"\"\"\n    hit = lambda x: x[0] == x[1] or not x[0]\n    seq = [(artifact, self.artifact), (os_name, self.os_name), (cpe, self.cpe),\n           (label, self.label)]\n    return all(map(hit, seq))"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nregister a callback for the given conditions.", "response": "def _Register(self, conditions, callback):\n    \"\"\"Map functions that should be called if the condition applies.\"\"\"\n    for condition in conditions:\n      registered = self._registry.setdefault(condition, [])\n      if callback and callback not in registered:\n        registered.append(callback)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef Add(self, artifact=None, target=None, callback=None):\n    # Cases where a target field is undefined or empty need special handling.\n    # Repeated field helper in target yields results, so expand this out into a\n    # list. If the attribute doesn't exist, default to an empty list.\n    # Then, in either case, replace the empty list with one containing a single\n    # None value.\n    if target is None:\n      target = Target()\n    os_name = target.Get(\"os\") or [None]\n    cpe = target.Get(\"cpe\") or [None]\n    label = target.Get(\"label\") or [None]\n    attributes = itertools.product(os_name, cpe, label)\n    new_conditions = [Condition(artifact, *attr) for attr in attributes]\n    self.conditions.update(new_conditions)\n    self._Register(new_conditions, callback)", "response": "Adds a condition to the check list."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef Update(self, other, callback):\n    self.conditions.update(other.conditions)\n    self._Register(other.conditions, callback)", "response": "Updates the set of existing triggers with the contents of the other set."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ntests if host data should trigger a check.", "response": "def Match(self, artifact=None, os_name=None, cpe=None, label=None):\n    \"\"\"Test if host data should trigger a check.\n\n    Args:\n      artifact: An artifact name.\n      os_name: An OS string.\n      cpe: A CPE string.\n      label: A label string.\n\n    Returns:\n      A list of conditions that match.\n    \"\"\"\n    return [\n        c for c in self.conditions if c.Match(artifact, os_name, cpe, label)\n    ]"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nsearch the host attributes that trigger data collection.", "response": "def Search(self, artifact=None, os_name=None, cpe=None, label=None):\n    \"\"\"Find the host attributes that trigger data collection.\n\n    Args:\n      artifact: An artifact name.\n      os_name: An OS string.\n      cpe: A CPE string.\n      label: A label string.\n\n    Returns:\n      A list of conditions that contain the specified attributes.\n    \"\"\"\n    return [\n        c for c in self.conditions if c.Search(artifact, os_name, cpe, label)\n    ]"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef Artifacts(self, os_name=None, cpe=None, label=None):\n    return [\n        c.artifact for c in self.conditions if c.Artifacts(os_name, cpe, label)\n    ]", "response": "Returns a list of artifacts that correspond with other trigger conditions."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nfind the methods that evaluate the data that meets the condition.", "response": "def Calls(self, conditions=None):\n    \"\"\"Find the methods that evaluate data that meets this condition.\n\n    Args:\n      conditions: A tuple of (artifact, os_name, cpe, label)\n\n    Returns:\n      A list of methods that evaluate the data.\n    \"\"\"\n    results = set()\n    if conditions is None:\n      conditions = [None]\n    for condition in conditions:\n      for c in self.Match(*condition):\n        results.update(self._registry.get(c, []))\n    return results"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn plugins metadata in ApiReportDescriptor.", "response": "def GetReportDescriptor(cls):\n    \"\"\"Returns plugins' metadata in ApiReportDescriptor.\"\"\"\n    if cls.TYPE is None:\n      raise ValueError(\"%s.TYPE is unintialized.\" % cls)\n\n    if cls.TITLE is None:\n      raise ValueError(\"%s.TITLE is unintialized.\" % cls)\n\n    if cls.SUMMARY is None:\n      raise ValueError(\"%s.SUMMARY is unintialized.\" % cls)\n\n    return rdf_report_plugins.ApiReportDescriptor(\n        type=cls.TYPE,\n        name=cls.__name__,\n        title=cls.TITLE,\n        summary=cls.SUMMARY,\n        requires_time_range=cls.REQUIRES_TIME_RANGE)"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nmakes a certificate and sign it with the CA s private key.", "response": "def MakeCASignedCert(common_name,\n                     private_key,\n                     ca_cert,\n                     ca_private_key,\n                     serial_number=2):\n  \"\"\"Make a cert and sign it with the CA's private key.\"\"\"\n  public_key = private_key.GetPublicKey()\n\n  builder = x509.CertificateBuilder()\n\n  builder = builder.issuer_name(ca_cert.GetIssuer())\n\n  subject = x509.Name(\n      [x509.NameAttribute(oid.NameOID.COMMON_NAME, common_name)])\n  builder = builder.subject_name(subject)\n\n  valid_from = rdfvalue.RDFDatetime.Now() - rdfvalue.Duration(\"1d\")\n  valid_until = rdfvalue.RDFDatetime.Now() + rdfvalue.Duration(\"3650d\")\n  builder = builder.not_valid_before(valid_from.AsDatetime())\n  builder = builder.not_valid_after(valid_until.AsDatetime())\n\n  builder = builder.serial_number(serial_number)\n  builder = builder.public_key(public_key.GetRawPublicKey())\n\n  builder = builder.add_extension(\n      x509.BasicConstraints(ca=False, path_length=None), critical=True)\n  certificate = builder.sign(\n      private_key=ca_private_key.GetRawPrivateKey(),\n      algorithm=hashes.SHA256(),\n      backend=openssl.backend)\n  return rdf_crypto.RDFX509Cert(certificate)"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ngenerates a CA certificate.", "response": "def MakeCACert(private_key,\n               common_name=u\"grr\",\n               issuer_cn=u\"grr_test\",\n               issuer_c=u\"US\"):\n  \"\"\"Generate a CA certificate.\n\n  Args:\n    private_key: The private key to use.\n    common_name: Name for cert.\n    issuer_cn: Name for issuer.\n    issuer_c: Country for issuer.\n\n  Returns:\n    The certificate.\n  \"\"\"\n  public_key = private_key.GetPublicKey()\n\n  builder = x509.CertificateBuilder()\n\n  issuer = x509.Name([\n      x509.NameAttribute(oid.NameOID.COMMON_NAME, issuer_cn),\n      x509.NameAttribute(oid.NameOID.COUNTRY_NAME, issuer_c)\n  ])\n  subject = x509.Name(\n      [x509.NameAttribute(oid.NameOID.COMMON_NAME, common_name)])\n  builder = builder.subject_name(subject)\n  builder = builder.issuer_name(issuer)\n\n  valid_from = rdfvalue.RDFDatetime.Now() - rdfvalue.Duration(\"1d\")\n  valid_until = rdfvalue.RDFDatetime.Now() + rdfvalue.Duration(\"3650d\")\n  builder = builder.not_valid_before(valid_from.AsDatetime())\n  builder = builder.not_valid_after(valid_until.AsDatetime())\n\n  builder = builder.serial_number(1)\n  builder = builder.public_key(public_key.GetRawPublicKey())\n\n  builder = builder.add_extension(\n      x509.BasicConstraints(ca=True, path_length=None), critical=True)\n  builder = builder.add_extension(\n      x509.SubjectKeyIdentifier.from_public_key(public_key.GetRawPublicKey()),\n      critical=False)\n  certificate = builder.sign(\n      private_key=private_key.GetRawPrivateKey(),\n      algorithm=hashes.SHA256(),\n      backend=openssl.backend)\n  return rdf_crypto.RDFX509Cert(certificate)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _LoadAuditEvents(handlers,\n                     get_report_args,\n                     actions=None,\n                     token=None,\n                     transformers=None):\n  \"\"\"Returns AuditEvents for given handlers, actions, and timerange.\"\"\"\n  if transformers is None:\n    transformers = {}\n\n  if data_store.RelationalDBEnabled():\n    entries = data_store.REL_DB.ReadAPIAuditEntries(\n        min_timestamp=get_report_args.start_time,\n        max_timestamp=get_report_args.start_time + get_report_args.duration,\n        router_method_names=list(handlers.keys()))\n    rows = [_EntryToEvent(entry, handlers, transformers) for entry in entries]\n  else:\n    entries = report_utils.GetAuditLogEntries(\n        offset=get_report_args.duration,\n        now=get_report_args.start_time + get_report_args.duration,\n        token=token)\n    if actions is None:\n      actions = set(handlers.values())\n    rows = [entry for entry in entries if entry.action in actions]\n  rows.sort(key=lambda row: row.timestamp, reverse=True)\n  return rows", "response": "Loads audit events for given handlers actions and timerange."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nconvert an APIAuditEntry to an AuditEvent.", "response": "def _EntryToEvent(entry, handlers, transformers):\n  \"\"\"Converts an APIAuditEntry to a legacy AuditEvent.\"\"\"\n  event = rdf_events.AuditEvent(\n      timestamp=entry.timestamp,\n      user=entry.username,\n      action=handlers[entry.router_method_name])\n\n  for fn in transformers:\n    fn(entry, event)\n\n  return event"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nextract a Client ID from an APIAuditEntry s HTTP request path.", "response": "def _ExtractClientIdFromPath(entry, event):\n  \"\"\"Extracts a Client ID from an APIAuditEntry's HTTP request path.\"\"\"\n  match = re.match(r\".*(C\\.[0-9a-fA-F]{16}).*\", entry.http_request_path)\n  if match:\n    event.client = match.group(1)"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nextracts a CronJob ID from an APIAuditEntry s HTTP request path.", "response": "def _ExtractCronJobIdFromPath(entry, event):\n  \"\"\"Extracts a CronJob ID from an APIAuditEntry's HTTP request path.\"\"\"\n  match = re.match(r\".*cron-job/([^/]+).*\", entry.http_request_path)\n  if match:\n    event.urn = \"aff4:/cron/{}\".format(match.group(1))"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nextracting a Hunt ID from an APIAuditEntry s HTTP request path.", "response": "def _ExtractHuntIdFromPath(entry, event):\n  \"\"\"Extracts a Hunt ID from an APIAuditEntry's HTTP request path.\"\"\"\n  match = re.match(r\".*hunt/([^/]+).*\", entry.http_request_path)\n  if match:\n    event.urn = \"aff4:/hunts/{}\".format(match.group(1))"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning the data for the given timerange.", "response": "def GetReportData(self, get_report_args, token=None):\n    \"\"\"Filter the cron job approvals in the given timerange.\"\"\"\n    ret = rdf_report_plugins.ApiReportData(\n        representation_type=RepresentationType.AUDIT_CHART,\n        audit_chart=rdf_report_plugins.ApiAuditChartReportData(\n            used_fields=self.USED_FIELDS))\n\n    ret.audit_chart.rows = _LoadAuditEvents(\n        self.HANDLERS,\n        get_report_args,\n        transformers=[_ExtractClientIdFromPath],\n        token=token)\n    return ret"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nfilter the last week of user actions.", "response": "def GetReportData(self, get_report_args, token):\n    \"\"\"Filter the last week of user actions.\"\"\"\n    ret = rdf_report_plugins.ApiReportData(\n        representation_type=RepresentationType.PIE_CHART)\n\n    counts = self._GetUserCounts(get_report_args, token)\n    for username in aff4_users.GRRUser.SYSTEM_USERS:\n      del counts[username]\n\n    ret.pie_chart.data = [\n        rdf_report_plugins.ApiReportDataPoint1D(x=count, label=user)\n        for user, count in sorted(iteritems(counts))\n    ]\n\n    return ret"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef GetReportData(self, get_report_args, token):\n    ret = rdf_report_plugins.ApiReportData(\n        representation_type=RepresentationType.STACK_CHART)\n\n    week_duration = rdfvalue.Duration(\"7d\")\n    num_weeks = math.ceil(get_report_args.duration.seconds /\n                          week_duration.seconds)\n    weeks = range(0, num_weeks)\n    start_time = get_report_args.start_time\n    end_time = start_time + num_weeks * week_duration\n    user_activity = collections.defaultdict(lambda: {week: 0 for week in weeks})\n\n    entries = self._LoadUserActivity(\n        start_time=get_report_args.start_time, end_time=end_time, token=token)\n\n    for username, timestamp, count in entries:\n      week = (timestamp - start_time).seconds // week_duration.seconds\n      if week in user_activity[username]:\n        user_activity[username][week] += count\n\n    user_activity = sorted(iteritems(user_activity))\n    user_activity = [(user, data)\n                     for user, data in user_activity\n                     if user not in aff4_users.GRRUser.SYSTEM_USERS]\n\n    ret.stack_chart.data = [\n        rdf_report_plugins.ApiReportDataSeries2D(\n            label=user,\n            points=(rdf_report_plugins.ApiReportDataPoint2D(x=x, y=y)\n                    for x, y in sorted(data.items())))\n        for user, data in user_activity\n    ]\n\n    return ret", "response": "Returns the last 7 days of the user actions."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _GetAFF4AttributeForReportType(report_type\n                                  ):\n  \"\"\"Returns the corresponding AFF4 attribute for the given report type.\"\"\"\n  if report_type == rdf_stats.ClientGraphSeries.ReportType.GRR_VERSION:\n    return aff4_stats.ClientFleetStats.SchemaCls.GRRVERSION_HISTOGRAM\n  elif report_type == rdf_stats.ClientGraphSeries.ReportType.OS_TYPE:\n    return aff4_stats.ClientFleetStats.SchemaCls.OS_HISTOGRAM\n  elif report_type == rdf_stats.ClientGraphSeries.ReportType.OS_RELEASE:\n    return aff4_stats.ClientFleetStats.SchemaCls.RELEASE_HISTOGRAM\n  elif report_type == rdf_stats.ClientGraphSeries.ReportType.N_DAY_ACTIVE:\n    return aff4_stats.ClientFleetStats.SchemaCls.LAST_CONTACTED_HISTOGRAM\n  else:\n    raise ValueError(\"Unknown report type %s.\" % report_type)", "response": "Returns the corresponding AFF4 attribute for the given report type."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nwrite graph series for a particular client label to the DB.", "response": "def WriteGraphSeries(graph_series,\n                     label,\n                     token = None):\n  \"\"\"Writes graph series for a particular client label to the DB.\n\n  Args:\n    graph_series: A series of rdf_stats.Graphs containing aggregated data for a\n      particular report-type.\n    label: Client label by which data in the graph_series was aggregated.\n    token: ACL token to use for writing to the legacy (non-relational)\n      datastore.\n\n  Raises:\n    AFF4AttributeTypeError: If, when writing to the legacy DB, an unexpected\n    report-data type is encountered.\n  \"\"\"\n  if data_store.RelationalDBEnabled():\n    data_store.REL_DB.WriteClientGraphSeries(graph_series, label)\n\n  if _ShouldUseLegacyDatastore():\n    # We need to use the __call__() method of the aff4.Attribute class\n    # to instantiate Graph and GraphSeries objects, or AFF4Object.AddAttribute()\n    # won't work.\n    aff4_attr = _GetAFF4AttributeForReportType(graph_series.report_type)()\n\n    if isinstance(aff4_attr, rdf_stats.GraphSeries):\n      for graph in graph_series.graphs:\n        aff4_attr.Append(graph)\n    elif isinstance(aff4_attr, rdf_stats.Graph):\n      for sample in graph_series.graphs[0]:\n        aff4_attr.Append(x_value=sample.x_value, y_value=sample.y_value)\n    else:\n      raise AFF4AttributeTypeError(aff4_attr.__class__)\n\n    with aff4.FACTORY.Create(\n        GetAFF4ClientReportsURN().Add(label),\n        aff4_type=aff4_stats.ClientFleetStats,\n        mode=\"w\",\n        token=token) as stats_for_label:\n      stats_for_label.AddAttribute(aff4_attr)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nfetch all graph series for the given label and report - type from the DB.", "response": "def FetchAllGraphSeries(\n    label,\n    report_type,\n    period = None,\n    token = None\n):\n  \"\"\"Fetches graph series for the given label and report-type from the DB.\n\n  Args:\n    label: Client label to fetch data for.\n    report_type: rdf_stats.ClientGraphSeries.ReportType to fetch data for.\n    period: rdfvalue.Duration specifying how far back in time to fetch data. If\n      not provided, all data for the given label and report-type will be\n      returned.\n    token: ACL token to use for reading from the legacy (non-relational)\n      datastore.\n\n  Raises:\n    AFF4AttributeTypeError: If, when reading to the legacy DB, an unexpected\n    report-data type is encountered.\n\n  Returns:\n    A dict mapping timestamps to graph-series. The timestamps\n    represent when the graph-series were written to the datastore.\n  \"\"\"\n  if _ShouldUseLegacyDatastore():\n    return _FetchAllGraphSeriesFromTheLegacyDB(\n        label, report_type, period=period, token=token)\n\n  if period is None:\n    time_range = None\n  else:\n    range_end = rdfvalue.RDFDatetime.Now()\n    time_range = time_utils.TimeRange(range_end - period, range_end)\n  return data_store.REL_DB.ReadAllClientGraphSeries(\n      label, report_type, time_range=time_range)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _FetchAllGraphSeriesFromTheLegacyDB(\n    label,\n    report_type,\n    period = None,\n    token = None\n):\n  \"\"\"Fetches graph-series from the legacy DB [see FetchAllGraphSeries()].\"\"\"\n  if period is None:\n    time_range = aff4.ALL_TIMES\n  else:\n    range_end = rdfvalue.RDFDatetime.Now()\n    time_range = (range_end - period, range_end)\n  series_with_timestamps = {}\n  try:\n    stats_for_label = aff4.FACTORY.Open(\n        GetAFF4ClientReportsURN().Add(label),\n        aff4_type=aff4_stats.ClientFleetStats,\n        mode=\"r\",\n        age=time_range,\n        token=token)\n  except aff4.InstantiationError:\n    # Nothing to return for the given label and report-type.\n    return series_with_timestamps\n  aff4_attr = _GetAFF4AttributeForReportType(report_type)\n  if aff4_attr.attribute_type == rdf_stats.GraphSeries:\n    for graphs in stats_for_label.GetValuesForAttribute(aff4_attr):\n      graph_series = rdf_stats.ClientGraphSeries(report_type=report_type)\n      for graph in graphs:\n        graph_series.graphs.Append(graph)\n      series_with_timestamps[graphs.age] = graph_series\n  elif aff4_attr.attribute_type == rdf_stats.Graph:\n    for graph in stats_for_label.GetValuesForAttribute(aff4_attr):\n      graph_series = rdf_stats.ClientGraphSeries(report_type=report_type)\n      graph_series.graphs.Append(graph)\n      series_with_timestamps[graph.age] = graph_series\n  else:\n    raise AFF4AttributeTypeError(aff4_attr.attribute_type)\n  return series_with_timestamps", "response": "Fetches all graph - series from the legacy DB [ see FetchAllGraphSeries() ]"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef FetchMostRecentGraphSeries(label,\n                               report_type,\n                               token = None\n                              ):\n  \"\"\"Fetches the latest graph series for a client label from the DB.\n\n  Args:\n    label: Client label to fetch data for.\n    report_type: rdf_stats.ClientGraphSeries.ReportType to fetch data for.\n    token: ACL token to use for reading from the legacy (non-relational)\n      datastore.\n\n  Raises:\n    AFF4AttributeTypeError: If, when reading to the legacy DB, an unexpected\n    report-data type is encountered.\n\n  Returns:\n    The graph series for the given label and report type that was last\n    written to the DB, or None if no series for that label and report-type\n    exist.\n  \"\"\"\n  if _ShouldUseLegacyDatastore():\n    return _FetchMostRecentGraphSeriesFromTheLegacyDB(\n        label, report_type, token=token)\n\n  return data_store.REL_DB.ReadMostRecentClientGraphSeries(label, report_type)", "response": "Fetches the latest graph series for a given label and report type from the DB."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _FetchMostRecentGraphSeriesFromTheLegacyDB(\n    label,\n    report_type,\n    token = None\n):\n  \"\"\"Fetches the latest graph-series for a client label from the legacy DB.\n\n  Args:\n    label: Client label to fetch data for.\n    report_type: rdf_stats.ClientGraphSeries.ReportType to fetch data for.\n    token: ACL token to use for reading from the DB.\n\n  Raises:\n    AFF4AttributeTypeError: If an unexpected report-data type is encountered.\n\n  Returns:\n    The graph series for the given label and report type that was last\n    written to the DB, or None if no series for that label and report-type\n    exist.\n  \"\"\"\n  try:\n    stats_for_label = aff4.FACTORY.Open(\n        GetAFF4ClientReportsURN().Add(label),\n        aff4_type=aff4_stats.ClientFleetStats,\n        mode=\"r\",\n        token=token)\n  except aff4.InstantiationError:\n    # Nothing to return for the given label and report-type.\n    return None\n  aff4_attr = _GetAFF4AttributeForReportType(report_type)\n  graph_series = rdf_stats.ClientGraphSeries(report_type=report_type)\n  if aff4_attr.attribute_type == rdf_stats.GraphSeries:\n    graphs = stats_for_label.Get(aff4_attr)\n    if graphs is None:\n      return None\n    for graph in graphs:\n      graph_series.graphs.Append(graph)\n  elif aff4_attr.attribute_type == rdf_stats.Graph:\n    graph = stats_for_label.Get(aff4_attr)\n    if graph is None:\n      return None\n    graph_series.graphs.Append(graph)\n  else:\n    raise AFF4AttributeTypeError(aff4_attr.attribute_type)\n  return graph_series", "response": "Fetches the latest graph - series for a given label and report - type from the legacy DB."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ngenerate a new unique task_id.", "response": "def GenerateTaskID(self):\n    \"\"\"Generates a new, unique task_id.\"\"\"\n    # Random number can not be zero since next_id_base must increment.\n    random_number = random.PositiveUInt16()\n\n    # 16 bit random numbers\n    with GrrMessage.lock:\n      next_id_base = GrrMessage.next_id_base\n\n      id_base = (next_id_base + random_number) & 0xFFFFFFFF\n      if id_base < next_id_base:\n        time.sleep(0.001)\n\n      GrrMessage.next_id_base = id_base\n\n    # 32 bit timestamp (in 1/1000 second resolution)\n    time_base = (int(time.time() * 1000) & 0x1FFFFFFF) << 32\n\n    task_id = time_base | id_base\n\n    self.Set(\"task_id\", task_id)\n    return task_id"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef AddFilesWithUnknownHashes(\n    client_path_blob_refs,\n    use_external_stores = True\n):\n  \"\"\"Adds new files consisting of given blob references.\n\n  Args:\n    client_path_blob_refs: A dictionary mapping `db.ClientPath` instances to\n      lists of blob references.\n    use_external_stores: A flag indicating if the files should also be added to\n      external file stores.\n\n  Returns:\n    A dictionary mapping `db.ClientPath` to hash ids of the file.\n\n  Raises:\n    BlobNotFoundError: If one of the referenced blobs cannot be found.\n  \"\"\"\n  hash_id_blob_refs = dict()\n  client_path_hash_id = dict()\n  metadatas = dict()\n\n  all_client_path_blob_refs = list()\n  for client_path, blob_refs in iteritems(client_path_blob_refs):\n    # In the special case where there is only one blob, we don't need to go to\n    # the data store to read said blob and rehash it, we have all that\n    # information already available. For empty files without blobs, we can just\n    # hash the empty string instead.\n    if len(blob_refs) <= 1:\n      if blob_refs:\n        hash_id = rdf_objects.SHA256HashID.FromBytes(\n            blob_refs[0].blob_id.AsBytes())\n      else:\n        hash_id = rdf_objects.SHA256HashID.FromData(b\"\")\n\n      client_path_hash_id[client_path] = hash_id\n      hash_id_blob_refs[hash_id] = blob_refs\n      metadatas[hash_id] = FileMetadata(\n          client_path=client_path, blob_refs=blob_refs)\n    else:\n      for blob_ref in blob_refs:\n        all_client_path_blob_refs.append((client_path, blob_ref))\n\n  client_path_offset = collections.defaultdict(lambda: 0)\n  client_path_sha256 = collections.defaultdict(hashlib.sha256)\n  verified_client_path_blob_refs = collections.defaultdict(list)\n\n  client_path_blob_ref_batches = collection.Batch(\n      items=all_client_path_blob_refs, size=_BLOBS_READ_BATCH_SIZE)\n\n  for client_path_blob_ref_batch in client_path_blob_ref_batches:\n    blob_id_batch = set(\n        blob_ref.blob_id for _, blob_ref in client_path_blob_ref_batch)\n    blobs = data_store.BLOBS.ReadBlobs(blob_id_batch)\n\n    for client_path, blob_ref in client_path_blob_ref_batch:\n      blob = blobs[blob_ref.blob_id]\n      if blob is None:\n        message = \"Could not find one of referenced blobs: {}\".format(\n            blob_ref.blob_id)\n        raise BlobNotFoundError(message)\n\n      offset = client_path_offset[client_path]\n      if blob_ref.size != len(blob):\n        raise ValueError(\n            \"Got conflicting size information for blob %s: %d vs %d.\" %\n            (blob_ref.blob_id, blob_ref.size, len(blob)))\n      if blob_ref.offset != offset:\n        raise ValueError(\n            \"Got conflicting offset information for blob %s: %d vs %d.\" %\n            (blob_ref.blob_id, blob_ref.offset, offset))\n\n      verified_client_path_blob_refs[client_path].append(blob_ref)\n      client_path_offset[client_path] = offset + len(blob)\n      client_path_sha256[client_path].update(blob)\n\n  for client_path in iterkeys(client_path_sha256):\n    sha256 = client_path_sha256[client_path].digest()\n    hash_id = rdf_objects.SHA256HashID.FromBytes(sha256)\n\n    client_path_hash_id[client_path] = hash_id\n    hash_id_blob_refs[hash_id] = verified_client_path_blob_refs[client_path]\n\n  data_store.REL_DB.WriteHashBlobReferences(hash_id_blob_refs)\n\n  if use_external_stores:\n    for client_path in iterkeys(verified_client_path_blob_refs):\n      metadatas[client_path_hash_id[client_path]] = FileMetadata(\n          client_path=client_path,\n          blob_refs=verified_client_path_blob_refs[client_path])\n\n    EXTERNAL_FILE_STORE.AddFiles(metadatas)\n\n  return client_path_hash_id", "response": "Adds new files consisting of given blobs."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef AddFileWithUnknownHash(client_path, blob_refs, use_external_stores=True):\n  precondition.AssertType(client_path, db.ClientPath)\n  precondition.AssertIterableType(blob_refs, rdf_objects.BlobReference)\n  return AddFilesWithUnknownHashes(\n      {client_path: blob_refs},\n      use_external_stores=use_external_stores)[client_path]", "response": "Adds a new file consisting of given blob IDs."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef CheckHashes(hash_ids):\n  return {\n      k: bool(v)\n      for k, v in data_store.REL_DB.ReadHashBlobReferences(hash_ids).items()\n  }", "response": "Checks if files with given hashes are present in the file store."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nopen the latest content of a given file for reading.", "response": "def OpenFile(client_path, max_timestamp=None):\n  \"\"\"Opens latest content of a given file for reading.\n\n  Args:\n    client_path: A db.ClientPath object describing path to a file.\n    max_timestamp: If specified, will open the last collected version with a\n      timestamp equal or lower than max_timestamp. If not specified, will simply\n      open the latest version.\n\n  Returns:\n    A file like object with random access support.\n\n  Raises:\n    FileHasNoContentError: if the file was never collected.\n    MissingBlobReferencesError: if one of the blobs was not found.\n  \"\"\"\n\n  path_info = data_store.REL_DB.ReadLatestPathInfosWithHashBlobReferences(\n      [client_path], max_timestamp=max_timestamp)[client_path]\n\n  if path_info is None:\n    raise FileHasNoContentError(client_path)\n\n  hash_id = rdf_objects.SHA256HashID.FromBytes(\n      path_info.hash_entry.sha256.AsBytes())\n  blob_references = data_store.REL_DB.ReadHashBlobReferences([hash_id])[hash_id]\n\n  if blob_references is None:\n    raise MissingBlobReferencesError(\n        \"File hash was expected to have corresponding \"\n        \"blob references, but they were not found: %r\" % hash_id)\n\n  return BlobStream(client_path, blob_references, hash_id)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nstream contents of given files.", "response": "def StreamFilesChunks(client_paths, max_timestamp=None, max_size=None):\n  \"\"\"Streams contents of given files.\n\n  Args:\n    client_paths: db.ClientPath objects describing paths to files.\n    max_timestamp: If specified, then for every requested file will open the\n      last collected version of the file with a timestamp equal or lower than\n      max_timestamp. If not specified, will simply open a latest version for\n      each file.\n    max_size: If specified, only the chunks covering max_size bytes will be\n      returned.\n\n  Yields:\n    StreamedFileChunk objects for every file read. Chunks will be returned\n    sequentially, their order will correspond to the client_paths order.\n    Files having no content will simply be ignored.\n  \"\"\"\n\n  path_infos_by_cp = (\n      data_store.REL_DB.ReadLatestPathInfosWithHashBlobReferences(\n          client_paths, max_timestamp=max_timestamp))\n\n  hash_ids_by_cp = {\n      cp: rdf_objects.SHA256HashID.FromBytes(pi.hash_entry.sha256.AsBytes())\n      for cp, pi in iteritems(path_infos_by_cp)\n      if pi\n  }\n\n  blob_refs_by_hash_id = data_store.REL_DB.ReadHashBlobReferences(\n      hash_ids_by_cp.values())\n\n  all_chunks = []\n  for cp in client_paths:\n    try:\n      hash_id = hash_ids_by_cp[cp]\n    except KeyError:\n      continue\n\n    try:\n      blob_refs = blob_refs_by_hash_id[hash_id]\n    except KeyError:\n      continue\n\n    num_blobs = len(blob_refs)\n    total_size = 0\n    for ref in blob_refs:\n      total_size += ref.size\n\n    cur_size = 0\n    for i, ref in enumerate(blob_refs):\n      all_chunks.append((cp, ref.blob_id, i, num_blobs, ref.offset, total_size))\n\n      cur_size += ref.size\n      if max_size is not None and cur_size >= max_size:\n        break\n\n  for batch in collection.Batch(all_chunks, STREAM_CHUNKS_READ_AHEAD):\n    blobs = data_store.BLOBS.ReadBlobs(\n        [blob_id for cp, blob_id, i, num_blobs, offset, total_size in batch])\n    for cp, blob_id, i, num_blobs, offset, total_size in batch:\n      yield StreamedFileChunk(cp, blobs[blob_id], i, num_blobs, offset,\n                              total_size)"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nadds multiple files to the file store.", "response": "def AddFiles(self, hash_id_metadatas):\n    \"\"\"Adds multiple files to the file store.\n\n    Args:\n      hash_id_metadatas: A dictionary mapping hash ids to file metadata (a tuple\n        of hash client path and blob references).\n    \"\"\"\n    for hash_id, metadata in iteritems(hash_id_metadatas):\n      self.AddFile(hash_id, metadata)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nfetching a chunk corresponding to the current offset.", "response": "def _GetChunk(self):\n    \"\"\"Fetches a chunk corresponding to the current offset.\"\"\"\n\n    found_ref = None\n    for ref in self._blob_refs:\n      if self._offset >= ref.offset and self._offset < (ref.offset + ref.size):\n        found_ref = ref\n        break\n\n    if not found_ref:\n      return None, None\n\n    # If self._current_ref == found_ref, then simply return previously found\n    # chunk. Otherwise, update self._current_chunk value.\n    if self._current_ref != found_ref:\n      self._current_ref = found_ref\n\n      data = data_store.BLOBS.ReadBlobs([found_ref.blob_id])\n      self._current_chunk = data[found_ref.blob_id]\n\n    return self._current_chunk, self._current_ref"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nread data from the server.", "response": "def Read(self, length=None):\n    \"\"\"Reads data.\"\"\"\n\n    if length is None:\n      length = self._length - self._offset\n\n    if length > self._max_unbound_read:\n      raise OversizedReadError(\"Attempted to read %d bytes when \"\n                               \"Server.max_unbound_read_size is %d\" %\n                               (length, self._max_unbound_read))\n\n    result = io.BytesIO()\n    while result.tell() < length:\n      chunk, ref = self._GetChunk()\n      if not chunk:\n        break\n\n      part = chunk[self._offset - ref.offset:]\n      if not part:\n        break\n\n      result.write(part)\n      self._offset += min(length, len(part))\n\n    return result.getvalue()[:length]"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nmove the reading cursor.", "response": "def Seek(self, offset, whence=os.SEEK_SET):\n    \"\"\"Moves the reading cursor.\"\"\"\n\n    if whence == os.SEEK_SET:\n      self._offset = offset\n    elif whence == os.SEEK_CUR:\n      self._offset += offset\n    elif whence == os.SEEK_END:\n      self._offset = self._length + offset\n    else:\n      raise ValueError(\"Invalid whence argument: %s\" % whence)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nuploading chunks of a file on a given path to the transfer store flow.", "response": "def UploadFilePath(self, filepath, offset=0, amount=None):\n    \"\"\"Uploads chunks of a file on a given path to the transfer store flow.\n\n    Args:\n      filepath: A path to the file to upload.\n      offset: An integer offset at which the file upload should start on.\n      amount: An upper bound on number of bytes to stream. If it is `None` then\n        the whole file is uploaded.\n\n    Returns:\n      A `BlobImageDescriptor` object.\n    \"\"\"\n    return self._UploadChunkStream(\n        self._streamer.StreamFilePath(filepath, offset=offset, amount=amount))"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef UploadFile(self, fd, offset=0, amount=None):\n    return self._UploadChunkStream(\n        self._streamer.StreamFile(fd, offset=offset, amount=amount))", "response": "Uploads a chunk of a given file descriptor to the transfer store flow."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nuploading a single chunk to the transfer store flow.", "response": "def _UploadChunk(self, chunk):\n    \"\"\"Uploads a single chunk to the transfer store flow.\n\n    Args:\n      chunk: A chunk to upload.\n\n    Returns:\n      A `BlobImageChunkDescriptor` object.\n    \"\"\"\n    blob = _CompressedDataBlob(chunk)\n\n    self._action.ChargeBytesToSession(len(chunk.data))\n    self._action.SendReply(blob, session_id=self._TRANSFER_STORE_SESSION_ID)\n\n    return rdf_client_fs.BlobImageChunkDescriptor(\n        digest=hashlib.sha256(chunk.data).digest(),\n        offset=chunk.offset,\n        length=len(chunk.data))"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef Parse(self, stat, file_object, knowledge_base):\n    _ = knowledge_base\n    entries = []\n\n    crondata = file_object.read().decode(\"utf-8\")\n    jobs = crontab.CronTab(tab=crondata)\n\n    for job in jobs:\n      entries.append(\n          rdf_cronjobs.CronTabEntry(\n              minute=str(job.minute),\n              hour=str(job.hour),\n              dayofmonth=str(job.dom),\n              month=str(job.month),\n              dayofweek=str(job.dow),\n              command=str(job.command),\n              comment=str(job.comment)))\n\n    try:\n      source_urn = file_object.urn\n    except AttributeError:\n      source_urn = None\n\n    yield rdf_cronjobs.CronTabFile(aff4path=source_urn, jobs=entries)", "response": "Parse the crontab file."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nparse the file finder condition types into the condition objects.", "response": "def Parse(conditions):\n    \"\"\"Parses the file finder condition types into the condition objects.\n\n    Args:\n      conditions: An iterator over `FileFinderCondition` objects.\n\n    Yields:\n      `MetadataCondition` objects that correspond to the file-finder conditions.\n    \"\"\"\n    kind = rdf_file_finder.FileFinderCondition.Type\n    classes = {\n        kind.MODIFICATION_TIME: ModificationTimeCondition,\n        kind.ACCESS_TIME: AccessTimeCondition,\n        kind.INODE_CHANGE_TIME: InodeChangeTimeCondition,\n        kind.SIZE: SizeCondition,\n        kind.EXT_FLAGS: ExtFlagsCondition,\n    }\n\n    for condition in conditions:\n      try:\n        yield classes[condition.condition_type](condition)\n      except KeyError:\n        pass"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef Parse(conditions):\n    kind = rdf_file_finder.FileFinderCondition.Type\n    classes = {\n        kind.CONTENTS_LITERAL_MATCH: LiteralMatchCondition,\n        kind.CONTENTS_REGEX_MATCH: RegexMatchCondition,\n    }\n\n    for condition in conditions:\n      try:\n        yield classes[condition.condition_type](condition)\n      except KeyError:\n        pass", "response": "Parses the file finder condition types into the condition objects."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef Scan(self, fd,\n           matcher):\n    \"\"\"Scans given file searching for occurrences of given pattern.\n\n    Args:\n      fd: A file descriptor of the file that needs to be searched.\n      matcher: A matcher object specifying a pattern to search for.\n\n    Yields:\n      `BufferReference` objects pointing to file parts with matching content.\n    \"\"\"\n    streamer = streaming.Streamer(\n        chunk_size=self.CHUNK_SIZE, overlap_size=self.OVERLAP_SIZE)\n\n    offset = self.params.start_offset\n    amount = self.params.length\n    for chunk in streamer.StreamFile(fd, offset=offset, amount=amount):\n      for span in chunk.Scan(matcher):\n        ctx_begin = max(span.begin - self.params.bytes_before, 0)\n        ctx_end = min(span.end + self.params.bytes_after, len(chunk.data))\n        ctx_data = chunk.data[ctx_begin:ctx_end]\n\n        yield rdf_client.BufferReference(\n            offset=chunk.offset + ctx_begin,\n            length=len(ctx_data),\n            data=ctx_data)\n\n        if self.params.mode == self.params.Mode.FIRST_HIT:\n          return", "response": "Scans given file for occurrences of given pattern."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ninitializing the server logging configuration.", "response": "def ServerLoggingStartupInit():\n  \"\"\"Initialize the server logging configuration.\"\"\"\n  global LOGGER\n  if local_log:\n    logging.debug(\"Using local LogInit from %s\", local_log)\n    local_log.LogInit()\n    logging.debug(\"Using local AppLogInit from %s\", local_log)\n    LOGGER = local_log.AppLogInit()\n  else:\n    LogInit()\n    LOGGER = AppLogInit()"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef GetNewEventId(self, event_time=None):\n    if event_time is None:\n      event_time = int(time.time() * 1e6)\n\n    return \"%s:%s:%s\" % (event_time, socket.gethostname(), os.getpid())", "response": "Return a unique Event ID string."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef LogHttpAdminUIAccess(self, request, response):\n    # TODO(user): generate event_id elsewhere and use it for all the log\n    # messages that have to do with handling corresponding request.\n    event_id = self.GetNewEventId()\n\n    api_method = response.headers.get(\"X-API-Method\", \"unknown\")\n    api_reason = response.headers.get(\"X-GRR-Reason\", \"none\")\n    log_msg = \"%s API call [%s] by %s (reason: %s): %s [%d]\" % (\n        event_id, api_method, request.user, api_reason, request.full_path,\n        response.status_code)\n    logging.info(log_msg)\n\n    if response.headers.get(\"X-No-Log\") != \"True\":\n      if data_store.RelationalDBEnabled():\n        entry = rdf_objects.APIAuditEntry.FromHttpRequestResponse(\n            request, response)\n        data_store.REL_DB.WriteAPIAuditEntry(entry)", "response": "Logs an http based API call."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef LogHttpFrontendAccess(self, request, source=None, message_count=None):\n    # TODO(user): generate event_id elsewhere and use it for all the log\n    # messages that have to do with handling corresponding request.\n    event_id = self.GetNewEventId()\n\n    log_msg = \"%s-%s [%s]: %s %s %s %s (%d)\" % (\n        event_id, request.source_ip, source or \"<unknown>\", request.method,\n        request.url, request.user_agent, request.user, message_count or 0)\n    logging.info(log_msg)", "response": "Writes a log entry for a Frontend or UI Request."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _GetPkgResources(package_name, filepath):\n  requirement = pkg_resources.Requirement.parse(package_name)\n  try:\n    return pkg_resources.resource_filename(requirement, filepath)\n  except pkg_resources.DistributionNotFound:\n    # It may be that the working set is not in sync (e.g. if sys.path was\n    # manipulated). Try to reload it just in case.\n    pkg_resources.working_set = pkg_resources.WorkingSet()\n    try:\n      return pkg_resources.resource_filename(requirement, filepath)\n    except pkg_resources.DistributionNotFound:\n      logging.error(\"Distribution %s not found. Is it installed?\", package_name)\n      return None", "response": "A wrapper for the pkg_resources. resource_filename function."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef ResourcePath(package_name, filepath):\n  # If we are running a pyinstaller-built binary we rely on the sys.prefix\n  # code below and avoid running this which will generate confusing error\n  # messages.\n  if not getattr(sys, \"frozen\", None):\n    target = _GetPkgResources(package_name, filepath)\n    if target and os.access(target, os.R_OK):\n      return target\n\n  # Installing from wheel places data_files relative to sys.prefix and not\n  # site-packages. If we can not find in site-packages, check sys.prefix\n  # instead.\n  # https://python-packaging-user-guide.readthedocs.io/en/latest/distributing/#data-files\n  target = os.path.join(sys.prefix, filepath)\n  if target and os.access(target, os.R_OK):\n    return target\n\n  return None", "response": "Computes a path to the specified resource in the specified package."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ncomputing a path to the specified module.", "response": "def ModulePath(module_name):\n  \"\"\"Computes a path to the specified module.\n\n  Args:\n    module_name: A name of the module to get the path for.\n\n  Returns:\n    A path to the specified module.\n\n  Raises:\n    ImportError: If specified module cannot be imported.\n  \"\"\"\n  module = importlib.import_module(module_name)\n  path = inspect.getfile(module)\n  # TODO: In Python 2 `inspect.getfile` returns a byte string, so\n  # we have to decode that in order to be consistent with Python 3.\n  if compatibility.PY2:\n    path = path.decode(\"utf-8\")\n\n  # In case of modules with want a path to the directory rather than to the\n  # `__init__.py` file itself.\n  if os.path.basename(path).startswith(\"__init__.\"):\n    path = os.path.dirname(path)\n\n  # Sometimes __file__ points at a .pyc file, when we really mean the .py.\n  if path.endswith(\".pyc\"):\n    path = path[:-4] + \".py\"\n\n  return path"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef WithLimitedCallFrequency(min_time_between_calls):\n\n  def Decorated(f):\n    \"\"\"Actual decorator implementation.\"\"\"\n\n    lock = threading.RLock()\n    prev_times = {}\n    prev_results = {}\n    result_locks = {}\n\n    @functools.wraps(f)\n    def Fn(*args, **kwargs):\n      \"\"\"Wrapper around the decorated function.\"\"\"\n\n      if WITH_LIMITED_CALL_FREQUENCY_PASS_THROUGH:\n        # This effectively turns off the caching.\n        min_time = rdfvalue.Duration(0)\n      else:\n        min_time = min_time_between_calls\n\n      key = (args, tuple(sorted(kwargs.items())))\n      now = rdfvalue.RDFDatetime.Now()\n\n      with lock:\n        for k, prev_time in list(prev_times.items()):\n          if now - prev_time >= min_time:\n            prev_times.pop(k)\n            prev_results.pop(k, None)\n            result_locks.pop(k, None)\n\n        try:\n          # We eliminated all the old entries, so if the key is present\n          # in the cache, it means that the data is fresh enough to be used.\n          prev_time = prev_times[key]\n          return prev_results[key]\n        except KeyError:\n          prev_time = None\n          should_call = True\n\n        if not should_call:\n          return prev_results[key]\n\n        try:\n          result_lock = result_locks[key]\n        except KeyError:\n          result_lock = threading.RLock()\n          result_locks[key] = result_lock\n\n      with result_lock:\n        t = prev_times.get(key)\n\n        if t == prev_time:\n          result = f(*args, **kwargs)\n          with lock:\n            prev_times[key] = rdfvalue.RDFDatetime.Now()\n            prev_results[key] = result\n\n          return result\n        else:\n          return prev_results[key]\n\n    return Fn\n\n  return Decorated", "response": "A Python function decorator that returns a cached version of the passed - in function f."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nconvert tuple field - name type to MetricFieldDefinition protos.", "response": "def FieldDefinitionProtosFromTuples(field_def_tuples):\n  \"\"\"Converts (field-name, type) tuples to MetricFieldDefinition protos.\"\"\"\n  # TODO: This needs fixing for Python 3.\n  field_def_protos = []\n  for field_name, field_type in field_def_tuples:\n    if field_type in (int, long):\n      field_type = rdf_stats.MetricFieldDefinition.FieldType.INT\n    elif issubclass(field_type, Text):\n      field_type = rdf_stats.MetricFieldDefinition.FieldType.STR\n    else:\n      raise ValueError(\"Invalid field type: %s\" % field_type)\n    field_def_protos.append(\n        rdf_stats.MetricFieldDefinition(\n            field_name=field_name, field_type=field_type))\n  return field_def_protos"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nconvert a list of MetricFieldDefinition protos to a list of tuples.", "response": "def FieldDefinitionTuplesFromProtos(field_def_protos):\n  \"\"\"Converts MetricFieldDefinition protos to (field-name, type) tuples.\"\"\"\n  # TODO: This needs fixing for Python 3.\n  field_def_tuples = []\n  for proto in field_def_protos:\n    if proto.field_type == rdf_stats.MetricFieldDefinition.FieldType.INT:\n      field_type = int\n    elif proto.field_type == rdf_stats.MetricFieldDefinition.FieldType.STR:\n      # Use old style str in Python 2 here or the streamz library will break.\n      field_type = compatibility.builtins.str\n    else:\n      raise ValueError(\"Unknown field type: %s\" % proto.field_type)\n    field_def_tuples.append((proto.field_name, field_type))\n  return field_def_tuples"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nconvert Python types to MetricMetadata. ValueType enum values.", "response": "def MetricValueTypeFromPythonType(python_type):\n  \"\"\"Converts Python types to MetricMetadata.ValueType enum values.\"\"\"\n  if python_type in (int, long):\n    return rdf_stats.MetricMetadata.ValueType.INT\n  elif python_type == float:\n    return rdf_stats.MetricMetadata.ValueType.FLOAT\n  else:\n    raise ValueError(\"Invalid value type: %s\" % python_type)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nconvert MetricMetadata. ValueType enums to corresponding Python types.", "response": "def PythonTypeFromMetricValueType(value_type):\n  \"\"\"Converts MetricMetadata.ValueType enums to corresponding Python types.\"\"\"\n  if value_type == rdf_stats.MetricMetadata.ValueType.INT:\n    return int\n  elif value_type == rdf_stats.MetricMetadata.ValueType.FLOAT:\n    return float\n  else:\n    raise ValueError(\"Unknown value type: %s\" % value_type)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef CreateCounterMetadata(metric_name, fields=None, docstring=None, units=None):\n  return rdf_stats.MetricMetadata(\n      varname=metric_name,\n      metric_type=rdf_stats.MetricMetadata.MetricType.COUNTER,\n      value_type=rdf_stats.MetricMetadata.ValueType.INT,\n      fields_defs=FieldDefinitionProtosFromTuples(fields or []),\n      docstring=docstring,\n      units=units)", "response": "Helper function for creating MetricMetadata for counter metrics."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef CreateGaugeMetadata(metric_name,\n                        value_type,\n                        fields=None,\n                        docstring=None,\n                        units=None):\n  \"\"\"Helper function for creating MetricMetadata for gauge metrics.\"\"\"\n  return rdf_stats.MetricMetadata(\n      varname=metric_name,\n      metric_type=rdf_stats.MetricMetadata.MetricType.GAUGE,\n      value_type=MetricValueTypeFromPythonType(value_type),\n      fields_defs=FieldDefinitionProtosFromTuples(fields or []),\n      docstring=docstring,\n      units=units)", "response": "Helper function for creating MetricMetadata for gauge metrics."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nimport hashes from filename into store.", "response": "def ImportFile(store, filename, start):\n  \"\"\"Import hashes from 'filename' into 'store'.\"\"\"\n  with io.open(filename, \"r\") as fp:\n    reader = csv.Reader(fp.read())\n    i = 0\n    current_row = None\n    product_code_list = []\n    op_system_code_list = []\n    for row in reader:\n      # Skip first row.\n      i += 1\n      if i and i % 5000 == 0:\n        data_store.DB.Flush()\n        print(\"Imported %d hashes\" % i)\n      if i > 1:\n        if len(row) != 8:\n          continue\n        try:\n          if i < start:\n            continue\n          if current_row:\n            if current_row[0] == row[0]:\n              # Same hash, add product/system\n              product_code_list.append(int(row[5]))\n              op_system_code_list.append(row[6])\n              continue\n            # Fall through and add current row.\n          else:\n            # First row.\n            current_row = row\n            product_code_list = [int(row[5])]\n            op_system_code_list = [row[6]]\n            continue\n          _ImportRow(store, current_row, product_code_list, op_system_code_list)\n          # Set new hash.\n          current_row = row\n          product_code_list = [int(row[5])]\n          op_system_code_list = [row[6]]\n        except Exception as e:  # pylint: disable=broad-except\n          print(\"Failed at %d with %s\" % (i, str(e)))\n          return i - 1\n    if current_row:\n      _ImportRow(store, current_row, product_code_list, op_system_code_list)\n    return i"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef main(argv):\n  del argv  # Unused.\n  server_startup.Init()\n\n  filename = flags.FLAGS.filename\n  if not os.path.exists(filename):\n    print(\"File %s does not exist\" % filename)\n    return\n\n  with aff4.FACTORY.Create(\n      filestore.NSRLFileStore.PATH,\n      filestore.NSRLFileStore,\n      mode=\"rw\",\n      token=aff4.FACTORY.root_token) as store:\n    imported = ImportFile(store, filename, flags.FLAGS.start)\n    data_store.DB.Flush()\n    print(\"Imported %d hashes\" % imported)", "response": "Main function. Import a file into the file store."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nconverting the canonical path as used by GRR to the local path.", "response": "def CanonicalPathToLocalPath(path):\n  r\"\"\"Converts the canonical paths as used by GRR to OS specific paths.\n\n  Due to the inconsistencies between handling paths in windows we need to\n  convert a path to an OS specific version prior to using it. This function\n  should be called just before any OS specific functions.\n\n  Canonical paths on windows have:\n    - / instead of \\.\n    - Begin with /X:// where X is the drive letter.\n\n  Args:\n    path: A canonical path specification.\n\n  Returns:\n    A windows specific path.\n  \"\"\"\n  # Account for raw devices\n  path = path.replace(\"/\\\\\", \"\\\\\")\n  path = path.replace(\"/\", \"\\\\\")\n  m = re.match(r\"\\\\([a-zA-Z]):(.*)$\", path)\n  if m:\n    path = \"%s:\\\\%s\" % (m.group(1), m.group(2).lstrip(\"\\\\\"))\n\n  return path"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef LocalPathToCanonicalPath(path):\n  path_components = path.split(\"/\")\n  result = []\n  for component in path_components:\n    # Devices must maintain their \\\\ so they do not get broken up.\n    m = re.match(r\"\\\\\\\\.\\\\\", component)\n\n    # The component is not special and can be converted as normal\n    if not m:\n      component = component.replace(\"\\\\\", \"/\")\n\n    result.append(component)\n\n  return utils.JoinPath(*result)", "response": "Converts a local path from the local system s convention to the canonical path."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nproviding chmod - like functionality for windows.", "response": "def WinChmod(filename, acl_list, user=None):\n  \"\"\"Provide chmod-like functionality for windows.\n\n  Doco links:\n    goo.gl/n7YR1\n    goo.gl/rDv81\n    goo.gl/hDobb\n\n  Args:\n    filename: target filename for acl\n\n    acl_list: list of ntsecuritycon acl strings to be applied with bitwise OR.\n              e.g. [\"FILE_GENERIC_READ\", \"FILE_GENERIC_WRITE\"]\n\n    user: username string. If not specified we use the user we are running as.\n\n  Raises:\n    AttributeError: if a bad permission is passed\n    RuntimeError: if filename doesn't exist\n  \"\"\"\n  if user is None:\n    user = win32api.GetUserName()\n\n  if not os.path.exists(filename):\n    raise RuntimeError(\"filename %s does not exist\" % filename)\n\n  acl_bitmask = 0\n  for acl in acl_list:\n    acl_bitmask |= getattr(ntsecuritycon, acl)\n\n  dacl = win32security.ACL()\n  win_user, _, _ = win32security.LookupAccountName(\"\", user)\n\n  dacl.AddAccessAllowedAce(win32security.ACL_REVISION, acl_bitmask, win_user)\n\n  security_descriptor = win32security.GetFileSecurity(\n      filename, win32security.DACL_SECURITY_INFORMATION)\n\n  # Tell windows to set the acl and mark it as explicitly set\n  security_descriptor.SetSecurityDescriptorDacl(DACL_PRESENT, dacl,\n                                                DACL_DEFAULT)\n  win32security.SetFileSecurity(\n      filename, win32security.DACL_SECURITY_INFORMATION, security_descriptor)"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ntries to find proxies by interrogating all the user's settings. This function is a modified urillib.getproxies_registry() from the standard library. We just store the proxy value in the environment for urllib to find it. TODO(user): Iterate through all the possible values if one proxy fails, in case more than one proxy is specified in different users profiles. Returns: A list of proxies.", "response": "def FindProxies():\n  \"\"\"Tries to find proxies by interrogating all the user's settings.\n\n  This function is a modified urillib.getproxies_registry() from the\n  standard library. We just store the proxy value in the environment\n  for urllib to find it.\n\n  TODO(user): Iterate through all the possible values if one proxy\n  fails, in case more than one proxy is specified in different users\n  profiles.\n\n  Returns:\n    A list of proxies.\n  \"\"\"\n\n  proxies = []\n  for i in range(0, 100):\n    try:\n      sid = winreg.EnumKey(winreg.HKEY_USERS, i)\n    except OSError:\n      break\n\n    try:\n      subkey = (\n          sid + \"\\\\Software\\\\Microsoft\\\\Windows\"\n          \"\\\\CurrentVersion\\\\Internet Settings\")\n\n      internet_settings = winreg.OpenKey(winreg.HKEY_USERS, subkey)\n\n      proxy_enable = winreg.QueryValueEx(internet_settings, \"ProxyEnable\")[0]\n\n      if proxy_enable:\n        # Returned as Unicode but problems if not converted to ASCII\n        proxy_server = str(\n            winreg.QueryValueEx(internet_settings, \"ProxyServer\")[0])\n        if \"=\" in proxy_server:\n          # Per-protocol settings\n          for p in proxy_server.split(\";\"):\n            protocol, address = p.split(\"=\", 1)\n            # See if address has a type:// prefix\n\n            if not re.match(\"^([^/:]+)://\", address):\n              address = \"%s://%s\" % (protocol, address)\n\n            proxies.append(address)\n        else:\n          # Use one setting for all protocols\n          if proxy_server[:5] == \"http:\":\n            proxies.append(proxy_server)\n          else:\n            proxies.append(\"http://%s\" % proxy_server)\n\n      internet_settings.Close()\n\n    except (OSError, ValueError, TypeError):\n      continue\n\n  logging.debug(\"Found proxy servers: %s\", proxies)\n\n  return proxies"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef GetRawDevice(path):\n  path = CanonicalPathToLocalPath(path)\n  # Try to expand the shortened paths\n  try:\n    path = win32file.GetLongPathName(path)\n  except pywintypes.error:\n    pass\n\n  try:\n    mount_point = win32file.GetVolumePathName(path)\n  except pywintypes.error as details:\n    logging.info(\"path not found. %s\", details)\n    raise IOError(\"No mountpoint for path: %s\" % path)\n\n  if not path.startswith(mount_point):\n    stripped_mp = mount_point.rstrip(\"\\\\\")\n    if not path.startswith(stripped_mp):\n      raise IOError(\"path %s is not mounted under %s\" % (path, mount_point))\n\n  corrected_path = LocalPathToCanonicalPath(path[len(mount_point):])\n  corrected_path = utils.NormalizePath(corrected_path)\n\n  volume = win32file.GetVolumeNameForVolumeMountPoint(mount_point).rstrip(\"\\\\\")\n  volume = LocalPathToCanonicalPath(volume)\n\n  # The pathspec for the raw volume\n  result = rdf_paths.PathSpec(\n      path=volume,\n      pathtype=rdf_paths.PathSpec.PathType.OS,\n      mount_point=mount_point.rstrip(\"\\\\\"))\n\n  return result, corrected_path", "response": "Resolves the raw device that contains the path."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _GetServiceKey():\n  global _service_key\n\n  if _service_key is None:\n    hive = getattr(winreg, config.CONFIG[\"Client.config_hive\"])\n    path = config.CONFIG[\"Client.config_key\"]\n\n    # Don't use winreg.KEY_WOW64_64KEY since it breaks on Windows 2000\n    _service_key = winreg.CreateKeyEx(hive, path, 0, winreg.KEY_ALL_ACCESS)\n\n  return _service_key", "response": "Returns the service key."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef RtlGetVersion(os_version_info_struct):\n  rc = ctypes.windll.Ntdll.RtlGetVersion(ctypes.byref(os_version_info_struct))\n  if rc != 0:\n    raise OSError(\"Getting Windows version failed.\")", "response": "Wraps the lowlevel RtlGetVersion routine."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ngetting the kernel version as string eg. 5. 1. 2600.", "response": "def KernelVersion():\n  \"\"\"Gets the kernel version as string, eg. \"5.1.2600\".\n\n  Returns:\n    The kernel version, or \"unknown\" in the case of failure.\n  \"\"\"\n  rtl_osversioninfoexw = RtlOSVersionInfoExw()\n  try:\n    RtlGetVersion(rtl_osversioninfoexw)\n  except OSError:\n    return \"unknown\"\n\n  return \"%d.%d.%d\" % (rtl_osversioninfoexw.dwMajorVersion,\n                       rtl_osversioninfoexw.dwMinorVersion,\n                       rtl_osversioninfoexw.dwBuildNumber)"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nwrite a heartbeat to the registry.", "response": "def Heartbeat(self):\n    \"\"\"Writes a heartbeat to the registry.\"\"\"\n    service_key = _GetServiceKey()\n    try:\n      winreg.SetValueEx(service_key, \"Nanny.heartbeat\", 0, winreg.REG_DWORD,\n                        int(time.time()))\n    except OSError as e:\n      logging.debug(\"Failed to heartbeat nanny at %s: %s\", service_key, e)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nwriting the message into the transaction log.", "response": "def Write(self, grr_message):\n    \"\"\"Write the message into the transaction log.\n\n    Args:\n      grr_message: A GrrMessage instance.\n    \"\"\"\n    grr_message = grr_message.SerializeToString()\n    try:\n      winreg.SetValueEx(_GetServiceKey(), \"Transaction\", 0, winreg.REG_BINARY,\n                        grr_message)\n      self._synced = False\n    except OSError:\n      pass"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn a GrrMessage instance from the transaction log or None.", "response": "def Get(self):\n    \"\"\"Return a GrrMessage instance from the transaction log or None.\"\"\"\n    try:\n      value, reg_type = winreg.QueryValueEx(_GetServiceKey(), \"Transaction\")\n    except OSError:\n      return\n\n    if reg_type != winreg.REG_BINARY:\n      return\n\n    try:\n      return rdf_flows.GrrMessage.FromSerializedString(value)\n    except message.Error:\n      return"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef FromPath(cls, path, follow_symlink = True):\n    # Note that we do not add type assertion for `path` here. The reason is that\n    # many of the existing system calls (e.g. `os.listdir`) return results as\n    # bytestrings in Python 2. This is fine because it also means that they also\n    # accept bytestring paths as arguments in Python 2 (e.g. `os.stat`). Having\n    # consistent types in both versions is certainly desired but it might be too\n    # much work for too little benefit.\n    precondition.AssertType(follow_symlink, bool)\n\n    if follow_symlink:\n      stat_obj = os.stat(path)\n    else:\n      stat_obj = os.lstat(path)\n\n    return cls(path=path, stat_obj=stat_obj)", "response": "Creates a new stat object from a given path."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nfetching Linux extended file flags.", "response": "def _FetchLinuxFlags(self):\n    \"\"\"Fetches Linux extended file flags.\"\"\"\n    if platform.system() != \"Linux\":\n      return 0\n\n    # Since we open a file in the next step we do not want to open a symlink.\n    # `lsattr` returns an error when trying to check flags of a symlink, so we\n    # assume that symlinks cannot have them.\n    if self.IsSymlink():\n      return 0\n\n    # Some files (e.g. sockets) cannot be opened. For these we do not really\n    # care about extended flags (they should have none). `lsattr` does not seem\n    # to support such cases anyway. It is also possible that a file has been\n    # deleted (because this method is used lazily).\n    try:\n      fd = os.open(self._path, os.O_RDONLY)\n    except (IOError, OSError):\n      return 0\n\n    try:\n      # This import is Linux-specific.\n      import fcntl  # pylint: disable=g-import-not-at-top\n      # TODO: On Python 2.7.6 `array.array` accepts only byte\n      # strings as an argument. On Python 2.7.12 and 2.7.13 unicodes are\n      # supported as well. On Python 3, only unicode strings are supported. This\n      # is why, as a temporary hack, we wrap the literal with `str` call that\n      # will convert it to whatever is the default on given Python version. This\n      # should be changed to raw literal once support for Python 2 is dropped.\n      buf = array.array(compatibility.NativeStr(\"l\"), [0])\n      # TODO(user):pytype: incorrect type spec for fcntl.ioctl\n      # pytype: disable=wrong-arg-types\n      fcntl.ioctl(fd, self.FS_IOC_GETFLAGS, buf)\n      # pytype: enable=wrong-arg-types\n      return buf[0]\n    except (IOError, OSError):\n      # File system does not support extended attributes.\n      return 0\n    finally:\n      os.close(fd)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef Get(self, path, follow_symlink = True):\n    key = self._Key(path=path, follow_symlink=follow_symlink)\n    try:\n      return self._cache[key]\n    except KeyError:\n      value = Stat.FromPath(path, follow_symlink=follow_symlink)\n      self._cache[key] = value\n\n      # If we are not following symlinks and the file is a not symlink then\n      # the stat result for this file stays the same even if we want to follow\n      # symlinks.\n      if not follow_symlink and not value.IsSymlink():\n        self._cache[self._Key(path=path, follow_symlink=True)] = value\n\n      return value", "response": "Get a stat object corresponding to the given path."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef GetClassesByArtifact(cls, artifact_name):\n    return [\n        cls.classes[c]\n        for c in cls.classes\n        if artifact_name in cls.classes[c].supported_artifacts\n    ]", "response": "Get the classes that support parsing a given artifact."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nenumerate all the configs in the given stats and file_objects.", "response": "def EnumerateAllConfigs(self, stats, file_objects):\n    \"\"\"Generate RDFs for the fully expanded configs.\n\n    Args:\n      stats: A list of RDF StatEntries corresponding to the file_objects.\n      file_objects: A list of file handles.\n\n    Returns:\n      A tuple of a list of RDFValue PamConfigEntries found & a list of strings\n      which are the external config references found.\n    \"\"\"\n    # Convert the stats & file_objects into a cache of a\n    # simple path keyed dict of file contents.\n    cache = {}\n    for stat_obj, file_obj in zip(stats, file_objects):\n      cache[stat_obj.pathspec.path] = utils.ReadFileBytesAsUnicode(file_obj)\n\n    result = []\n    external = []\n    # Check to see if we have the old pam config file laying around.\n    if self.OLD_PAMCONF_FILENAME in cache:\n      # The PAM documentation says if it contains config data, then\n      # it takes precedence over the rest of the config.\n      # If it doesn't, the rest of the PAMDIR config counts.\n      result, external = self.EnumerateConfig(None, self.OLD_PAMCONF_FILENAME,\n                                              cache)\n      if result:\n        return result, external\n\n    # If we made it here, there isn't a old-style pam.conf file worth\n    # speaking of, so process everything!\n    for path in cache:\n      # PAM uses the basename as the 'service' id.\n      service = os.path.basename(path)\n      r, e = self.EnumerateConfig(service, path, cache)\n      result.extend(r)\n      external.extend(e)\n    return result, external"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef EnumerateConfig(self, service, path, cache, filter_type=None):\n\n    result = []\n    external = []\n    path = self._FixPath(path)\n\n    # Make sure we only look at files under PAMDIR.\n    # Check we have the file in our artifact/cache. If not, our artifact\n    # didn't give it to us, and that's a problem.\n    # Note: This should only ever happen if it was referenced\n    # from /etc/pam.conf so we can assume that was the file.\n    if path not in cache:\n      external.append(\"%s -> %s\", self.OLD_PAMCONF_FILENAME, path)\n      return result, external\n\n    for tokens in self.ParseEntries(cache[path]):\n      if path == self.OLD_PAMCONF_FILENAME:\n        # We are processing the old style PAM conf file. It's a special case.\n        # It's format is \"service type control module-path module-arguments\"\n        # i.e. the 'service' is the first arg, the rest is line\n        # is like everything else except for that addition.\n        try:\n          service = tokens[0]  # Grab the service from the start line.\n          tokens = tokens[1:]  # Make the rest of the line look like \"normal\".\n        except IndexError:\n          continue  # It's a blank line, skip it.\n\n      # Process any inclusions in the line.\n      new_path = None\n      filter_request = None\n      try:\n        # If a line starts with @include, then include the entire referenced\n        # file.\n        # e.g. \"@include common-auth\"\n        if tokens[0] == \"@include\":\n          new_path = tokens[1]\n        # If a line's second arg is an include/substack, then filter the\n        # referenced file only including entries that match the 'type'\n        # requested.\n        # e.g. \"auth include common-auth-screensaver\"\n        elif tokens[1] in [\"include\", \"substack\"]:\n          new_path = tokens[2]\n          filter_request = tokens[0]\n      except IndexError:\n        # It's not a valid include line, so keep processing as normal.\n        pass\n\n      # If we found an include file, enumerate that file now, and\n      # included it where we are in this config file.\n      if new_path:\n        # Preemptively check to see if we have a problem where the config\n        # is referencing a file outside of the expected/defined artifact.\n        # Doing it here allows us to produce a better context for the\n        # problem. Hence the slight duplication of code.\n\n        new_path = self._FixPath(new_path)\n        if new_path not in cache:\n          external.append(\"%s -> %s\" % (path, new_path))\n          continue  # Skip to the next line of the file.\n        r, e = self.EnumerateConfig(service, new_path, cache, filter_request)\n        result.extend(r)\n        external.extend(e)\n      else:\n        # If we have been asked to filter on types, skip over any types\n        # we are not interested in.\n        if filter_type and tokens[0] != filter_type:\n          continue  # We can skip this line.\n\n        # If we got here, then we want to include this line in this service's\n        # config.\n\n        # Reform the line and break into the correct fields as best we can.\n        # Note: ParseEntries doesn't cope with what we need to do.\n        match = self.PAMCONF_RE.match(\" \".join(tokens))\n        if match:\n          p_type, control, module_path, module_args = match.group(1, 2, 3, 4)\n          # Trim a leading \"-\" from the type field if present.\n          if p_type.startswith(\"-\"):\n            p_type = p_type[1:]\n          result.append(\n              rdf_config_file.PamConfigEntry(\n                  service=service,\n                  type=p_type,\n                  control=control,\n                  module_path=module_path,\n                  module_args=module_args))\n    return result, external", "response": "Return PamConfigEntries it finds as it recursively follows PAM configs.\n\n    Args:\n      service: A string containing the service name we are processing.\n      path: A string containing the file path name we want.\n      cache: A dictionary keyed on path, with the file contents (list of str).\n      filter_type: A string containing type name of the results we want.\n\n    Returns:\n      A tuple of a list of RDFValue PamConfigEntries found & a list of strings\n      which are the external config references found."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nvalidates an AFF4 type.", "response": "def _ValidateAFF4Type(aff4_type):\n  \"\"\"Validates an AFF4 type.\"\"\"\n  if aff4_type is None:\n    return\n\n  if not isinstance(aff4_type, type):\n    raise TypeError(\"aff4_type=%s must be a type\" % aff4_type)\n  if not issubclass(aff4_type, AFF4Object):\n    raise TypeError(\"aff4_type=%s must be a subclass of AFF4Object.\" %\n                    aff4_type)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef Open(self, urn, aff4_type=None, mode=\"r\"):\n    key = self._ObjectKey(urn, mode)\n\n    try:\n      obj = self._objects_cache[key]\n    except KeyError:\n      obj = FACTORY.Open(\n          urn, mode=mode, follow_symlinks=False, token=self._token)\n      self._objects_cache[key] = obj\n\n    if aff4_type is not None and not isinstance(obj, aff4_type):\n      raise InstantiationError(\n          \"Object %s is of type %s, but required_type is %s\" %\n          (urn, obj.__class__.__name__, aff4_type.__name__))\n\n    return obj", "response": "Opens the named object in the pool."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nopening many urns efficiently returning cached objects when possible.", "response": "def MultiOpen(self, urns, aff4_type=None, mode=\"r\"):\n    \"\"\"Opens many urns efficiently, returning cached objects when possible.\"\"\"\n    not_opened_urns = []\n    _ValidateAFF4Type(aff4_type)\n\n    for urn in urns:\n      key = self._ObjectKey(urn, mode)\n      try:\n        result = self._objects_cache[key]\n        if aff4_type is not None and not isinstance(result, aff4_type):\n          continue\n        yield result\n\n      except KeyError:\n        not_opened_urns.append(urn)\n\n    if not_opened_urns:\n      for obj in FACTORY.MultiOpen(\n          not_opened_urns, follow_symlinks=False, mode=mode, token=self._token):\n        key = self._ObjectKey(obj.urn, mode)\n        self._objects_cache[key] = obj\n\n        if aff4_type is not None and not isinstance(obj, aff4_type):\n          continue\n\n        yield obj"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef ListChildren(self, urn):\n    result = self.MultiListChildren([urn])\n    try:\n      return result[urn]\n    except KeyError:\n      return []", "response": "Lists children of a given urn."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef MultiListChildren(self, urns):\n    result = {}\n    not_listed_urns = []\n\n    for urn in urns:\n      try:\n        result[urn] = self._children_lists_cache[urn]\n      except KeyError:\n        not_listed_urns.append(urn)\n\n    if not_listed_urns:\n      for urn, children in FACTORY.MultiListChildren(not_listed_urns):\n        result[urn] = self._children_lists_cache[urn] = children\n\n      for urn in not_listed_urns:\n        self._children_lists_cache.setdefault(urn, [])\n        result.setdefault(urn, [])\n\n    return result", "response": "Lists children of a bunch of given urns. Results are cached."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef RecursiveMultiListChildren(self, urns):\n    result = {}\n\n    checked_urns = set()\n    not_cached_urns = []\n    urns_to_check = urns\n    while True:\n      found_children = []\n\n      for urn in urns_to_check:\n        try:\n          children = result[urn] = self._children_lists_cache[urn]\n          found_children.extend(children)\n        except KeyError:\n          not_cached_urns.append(urn)\n\n      checked_urns.update(urns_to_check)\n      urns_to_check = set(found_children) - checked_urns\n\n      if not urns_to_check:\n        break\n\n    for urn, children in FACTORY.RecursiveMultiListChildren(not_cached_urns):\n      result[urn] = self._children_lists_cache[urn] = children\n\n    return result", "response": "Recursively lists given urns. Results are cached."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef MultiMarkForDeletion(self, urns):\n    all_children_urns = self.RecursiveMultiListChildren(urns)\n\n    urns.extend(collection.Flatten(itervalues(all_children_urns)))\n    self._urns_for_deletion.update(urns)\n\n    for obj in self.MultiOpen(urns):\n      obj.OnDelete(deletion_pool=self)", "response": "Marks multiple urns for deletion."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef root_urns_for_deletion(self):\n    roots = set()\n    for urn in self._urns_for_deletion:\n      new_root = True\n\n      str_urn = utils.SmartUnicode(urn)\n      fake_roots = []\n      for root in roots:\n        str_root = utils.SmartUnicode(root)\n\n        if str_urn.startswith(str_root):\n          new_root = False\n          break\n        elif str_root.startswith(str_urn):\n          fake_roots.append(root)\n\n      if new_root:\n        roots -= set(fake_roots)\n        roots.add(urn)\n\n    return roots", "response": "Returns a set of all the root urns marked for deletion."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nparses an aff4 age and returns a datastore age specification.", "response": "def ParseAgeSpecification(cls, age):\n    \"\"\"Parses an aff4 age and returns a datastore age specification.\"\"\"\n    try:\n      return (0, int(age))\n    except (ValueError, TypeError):\n      pass\n\n    if age == NEWEST_TIME:\n      return data_store.DB.NEWEST_TIMESTAMP\n    elif age == ALL_TIMES:\n      return data_store.DB.ALL_TIMESTAMPS\n    elif len(age) == 2:\n      start, end = age\n\n      return (int(start), int(end))\n\n    raise ValueError(\"Unknown age specification: %s\" % age)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef GetAttributes(self, urns, age=NEWEST_TIME):\n    urns = set([utils.SmartUnicode(u) for u in urns])\n    to_read = {urn: self._MakeCacheInvariant(urn, age) for urn in urns}\n\n    # Urns not present in the cache we need to get from the database.\n    if to_read:\n      for subject, values in data_store.DB.MultiResolvePrefix(\n          to_read,\n          AFF4_PREFIXES,\n          timestamp=self.ParseAgeSpecification(age),\n          limit=None):\n\n        # Ensure the values are sorted.\n        values.sort(key=lambda x: x[-1], reverse=True)\n\n        yield utils.SmartUnicode(subject), values", "response": "Retrieves all the attributes for all the urns."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nsetting the attributes in the data store.", "response": "def SetAttributes(self,\n                    urn,\n                    attributes,\n                    to_delete,\n                    add_child_index=True,\n                    mutation_pool=None):\n    \"\"\"Sets the attributes in the data store.\"\"\"\n\n    attributes[AFF4Object.SchemaCls.LAST] = [\n        rdfvalue.RDFDatetime.Now().SerializeToDataStore()\n    ]\n    to_delete.add(AFF4Object.SchemaCls.LAST)\n    if mutation_pool:\n      pool = mutation_pool\n    else:\n      pool = data_store.DB.GetMutationPool()\n\n    pool.MultiSet(urn, attributes, replace=False, to_delete=to_delete)\n\n    if add_child_index:\n      self._UpdateChildIndex(urn, pool)\n    if mutation_pool is None:\n      pool.Flush()"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nupdating the child indexes.", "response": "def _UpdateChildIndex(self, urn, mutation_pool):\n    \"\"\"Update the child indexes.\n\n    This function maintains the index for direct child relations. When we set\n    an AFF4 path, we always add an attribute like\n    index:dir/%(childname)s to its parent. This is written\n    asynchronously to its parent.\n\n    In order to query for all direct children of an AFF4 object, we then simple\n    get the attributes which match the regex index:dir/.+ which are the\n    direct children.\n\n    Args:\n      urn: The AFF4 object for which we update the index.\n      mutation_pool: A MutationPool object to write to.\n    \"\"\"\n    try:\n      # Create navigation aids by touching intermediate subject names.\n      while urn.Path() != \"/\":\n        basename = urn.Basename()\n        dirname = rdfvalue.RDFURN(urn.Dirname())\n\n        try:\n          self.intermediate_cache.Get(urn)\n          return\n        except KeyError:\n          extra_attributes = None\n          # This is a performance optimization. On the root there is no point\n          # setting the last access time since it gets accessed all the time.\n          # TODO(amoser): Can we get rid of the index in the root node entirely?\n          # It's too big to query anyways...\n          if dirname != u\"/\":\n            extra_attributes = {\n                AFF4Object.SchemaCls.LAST: [\n                    rdfvalue.RDFDatetime.Now().SerializeToDataStore()\n                ]\n            }\n\n          mutation_pool.AFF4AddChild(\n              dirname, basename, extra_attributes=extra_attributes)\n\n          self.intermediate_cache.Put(urn, 1)\n\n          urn = dirname\n\n    except access_control.UnauthorizedAccess:\n      pass"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning an invariant key for an AFF4 object.", "response": "def _MakeCacheInvariant(self, urn, age):\n    \"\"\"Returns an invariant key for an AFF4 object.\n\n    The object will be cached based on this key. This function is specifically\n    extracted to ensure that we encapsulate all security critical aspects of the\n    AFF4 object so that objects do not leak across security boundaries.\n\n    Args:\n       urn: The urn of the object.\n       age: The age policy used to build this object. Should be one of\n         ALL_TIMES, NEWEST_TIME or a range.\n\n    Returns:\n       A key into the cache.\n    \"\"\"\n    precondition.AssertType(urn, Text)\n    return \"%s:%s\" % (urn, self.ParseAgeSpecification(age))"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncreate a new object and locks it.", "response": "def CreateWithLock(self,\n                     urn,\n                     aff4_type,\n                     token=None,\n                     age=NEWEST_TIME,\n                     force_new_version=True,\n                     blocking=True,\n                     blocking_lock_timeout=10,\n                     blocking_sleep_interval=1,\n                     lease_time=100):\n    \"\"\"Creates a new object and locks it.\n\n    Similar to OpenWithLock below, this creates a locked object. The difference\n    is that when you call CreateWithLock, the object does not yet have to exist\n    in the data store.\n\n    Args:\n      urn: The object to create.\n      aff4_type: The desired type for this object.\n      token: The Security Token to use for opening this item.\n      age: The age policy used to build this object. Only makes sense when mode\n        has \"r\".\n      force_new_version: Forces the creation of a new object in the data_store.\n      blocking: When True, wait and repeatedly try to grab the lock.\n      blocking_lock_timeout: Maximum wait time when sync is True.\n      blocking_sleep_interval: Sleep time between lock grabbing attempts. Used\n        when blocking is True.\n      lease_time: Maximum time the object stays locked. Lock will be considered\n        released when this time expires.\n\n    Returns:\n      An AFF4 object of the desired type and mode.\n\n    Raises:\n      AttributeError: If the mode is invalid.\n    \"\"\"\n\n    if not data_store.AFF4Enabled():\n      raise NotImplementedError(\"AFF4 data store has been disabled.\")\n\n    transaction = self._AcquireLock(\n        urn,\n        blocking=blocking,\n        blocking_lock_timeout=blocking_lock_timeout,\n        blocking_sleep_interval=blocking_sleep_interval,\n        lease_time=lease_time)\n\n    # Since we now own the data store subject, we can simply create the aff4\n    # object in the usual way.\n    return self.Create(\n        urn,\n        aff4_type,\n        mode=\"rw\",\n        token=token,\n        age=age,\n        force_new_version=force_new_version,\n        transaction=transaction)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef OpenWithLock(self,\n                   urn,\n                   aff4_type=None,\n                   token=None,\n                   age=NEWEST_TIME,\n                   blocking=True,\n                   blocking_lock_timeout=10,\n                   blocking_sleep_interval=1,\n                   lease_time=100):\n    \"\"\"Open given urn and locks it.\n\n    Opens an object and locks it for 'lease_time' seconds. OpenWithLock can\n    only be used in 'with ...' statement. The lock is released when code\n    execution leaves 'with ...' block.\n\n    The urn is always opened in \"rw\" mode. Symlinks are not followed in\n    OpenWithLock() due to possible race conditions.\n\n    Args:\n      urn: The urn to open.\n      aff4_type: If this optional parameter is set, we raise an\n        InstantiationError if the object exists and is not an instance of this\n        type. This check is important when a different object can be stored in\n        this location.\n      token: The Security Token to use for opening this item.\n      age: The age policy used to build this object. Should be one of\n        NEWEST_TIME, ALL_TIMES or a time range given as a tuple (start, end) in\n        microseconds since Jan 1st, 1970.\n      blocking: When True, wait and repeatedly try to grab the lock.\n      blocking_lock_timeout: Maximum wait time when sync is True.\n      blocking_sleep_interval: Sleep time between lock grabbing attempts. Used\n        when blocking is True.\n      lease_time: Maximum time the object stays locked. Lock will be considered\n        released when this time expires.\n\n    Raises:\n      ValueError: The URN passed in is None.\n\n    Returns:\n      Context manager to be used in 'with ...' statement.\n    \"\"\"\n\n    if not data_store.AFF4Enabled():\n      raise NotImplementedError(\"AFF4 data store has been disabled.\")\n\n    transaction = self._AcquireLock(\n        urn,\n        blocking=blocking,\n        blocking_lock_timeout=blocking_lock_timeout,\n        blocking_sleep_interval=blocking_sleep_interval,\n        lease_time=lease_time)\n\n    # Since we now own the data store subject, we can simply read the aff4\n    # object in the usual way.\n    return self.Open(\n        urn,\n        aff4_type=aff4_type,\n        mode=\"rw\",\n        token=token,\n        age=age,\n        follow_symlinks=False,\n        transaction=transaction)", "response": "Opens an object and locks it for lease_time seconds."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nacquiring a lock for a given URN.", "response": "def _AcquireLock(self,\n                   urn,\n                   blocking=None,\n                   blocking_lock_timeout=None,\n                   lease_time=None,\n                   blocking_sleep_interval=None):\n    \"\"\"This actually acquires the lock for a given URN.\"\"\"\n    if urn is None:\n      raise ValueError(\"URN cannot be None\")\n\n    urn = rdfvalue.RDFURN(urn)\n\n    try:\n      return data_store.DB.LockRetryWrapper(\n          urn,\n          retrywrap_timeout=blocking_sleep_interval,\n          retrywrap_max_timeout=blocking_lock_timeout,\n          blocking=blocking,\n          lease_time=lease_time)\n    except data_store.DBSubjectLockError as e:\n      raise LockError(e)"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nmakes a copy of one AFF4 object to a different URN.", "response": "def Copy(self,\n           old_urn,\n           new_urn,\n           age=NEWEST_TIME,\n           limit=None,\n           update_timestamps=False):\n    \"\"\"Make a copy of one AFF4 object to a different URN.\"\"\"\n    new_urn = rdfvalue.RDFURN(new_urn)\n\n    if update_timestamps and age != NEWEST_TIME:\n      raise ValueError(\n          \"Can't update timestamps unless reading the latest version.\")\n\n    values = {}\n    for predicate, value, ts in data_store.DB.ResolvePrefix(\n        old_urn,\n        AFF4_PREFIXES,\n        timestamp=self.ParseAgeSpecification(age),\n        limit=limit):\n      if update_timestamps:\n        values.setdefault(predicate, []).append((value, None))\n      else:\n        values.setdefault(predicate, []).append((value, ts))\n\n    if values:\n      with data_store.DB.GetMutationPool() as pool:\n        pool.MultiSet(new_urn, values, replace=False)\n        self._UpdateChildIndex(new_urn, pool)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nchecking if an object with a given URN and type exists in the datastore.", "response": "def ExistsWithType(self,\n                     urn,\n                     aff4_type=None,\n                     follow_symlinks=True,\n                     age=NEWEST_TIME,\n                     token=None):\n    \"\"\"Checks if an object with a given URN and type exists in the datastore.\n\n    Args:\n      urn: The urn to check.\n      aff4_type: Expected object type.\n      follow_symlinks: If object opened is a symlink, follow it.\n      age: The age policy used to check this object. Should be either\n        NEWEST_TIME or a time range given as a tuple (start, end) in\n        microseconds since Jan 1st, 1970.\n      token: The Security Token to use for opening this item.\n\n    Raises:\n      ValueError: if aff4_type is not specified.\n\n    Returns:\n      True if there's an object with a matching type at a given URN, False\n      otherwise.\n    \"\"\"\n    if not aff4_type:\n      raise ValueError(\"aff4_type can't be None\")\n\n    try:\n      self.Open(\n          urn,\n          aff4_type=aff4_type,\n          follow_symlinks=follow_symlinks,\n          age=age,\n          token=token)\n      return True\n    except InstantiationError:\n      return False"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef Open(self,\n           urn,\n           aff4_type=None,\n           mode=\"r\",\n           token=None,\n           local_cache=None,\n           age=NEWEST_TIME,\n           follow_symlinks=True,\n           transaction=None):\n    \"\"\"Opens the named object.\n\n    This instantiates the object from the AFF4 data store.\n    Note that the root aff4:/ object is a container for all other\n    objects. Opening it for reading will instantiate a AFF4Volume instance, even\n    if the row does not exist.\n\n    The mode parameter specifies, how the object should be opened. A read only\n    mode will raise when calling Set() on it, while a write only object will\n    never read from the data store. Note that its impossible to open an object\n    with pure write support (since we have no idea what type it should be\n    without reading the data base) - use Create() instead for purely write mode.\n\n    Args:\n      urn: The urn to open.\n      aff4_type: If this parameter is set, we raise an IOError if the object is\n        not an instance of this type. This check is important when a different\n        object can be stored in this location. If mode is \"w\", this parameter\n        will determine the type of the object and is mandatory.\n      mode: The mode to open the file with.\n      token: The Security Token to use for opening this item.\n      local_cache: A dict containing a cache as returned by GetAttributes. If\n        set, this bypasses the factory cache.\n      age: The age policy used to build this object. Should be one of\n        NEWEST_TIME, ALL_TIMES or a time range given as a tuple (start, end) in\n        microseconds since Jan 1st, 1970.\n      follow_symlinks: If object opened is a symlink, follow it.\n      transaction: A lock in case this object is opened under lock.\n\n    Returns:\n      An AFF4Object instance.\n\n    Raises:\n      IOError: If the object is not of the required type.\n      AttributeError: If the requested mode is incorrect.\n    \"\"\"\n\n    if not data_store.AFF4Enabled():\n      raise NotImplementedError(\"AFF4 data store has been disabled.\")\n\n    _ValidateAFF4Type(aff4_type)\n\n    if mode not in [\"w\", \"r\", \"rw\"]:\n      raise AttributeError(\"Invalid mode %s\" % mode)\n\n    if mode == \"w\":\n      if aff4_type is None:\n        raise AttributeError(\"Need a type to open in write only mode.\")\n      return self.Create(\n          urn,\n          aff4_type,\n          mode=mode,\n          token=token,\n          age=age,\n          force_new_version=False,\n          transaction=transaction)\n\n    urn = rdfvalue.RDFURN(urn)\n\n    if token is None:\n      token = data_store.default_token\n\n    if \"r\" in mode and (local_cache is None or urn not in local_cache):\n      local_cache = dict(self.GetAttributes([urn], age=age))\n\n    # Read the row from the table. We know the object already exists if there is\n    # some data in the local_cache already for this object.\n    result = AFF4Object(\n        urn,\n        mode=mode,\n        token=token,\n        local_cache=local_cache,\n        age=age,\n        follow_symlinks=follow_symlinks,\n        object_exists=bool(local_cache.get(urn)),\n        transaction=transaction)\n\n    result.aff4_type = aff4_type\n\n    # Now we have a AFF4Object, turn it into the type it is currently supposed\n    # to be as specified by Schema.TYPE.\n    existing_type = result.Get(result.Schema.TYPE, default=\"AFF4Volume\")\n\n    if existing_type:\n      try:\n        result = result.Upgrade(AFF4Object.classes[existing_type])\n      except KeyError:\n        raise InstantiationError(\"Unable to open %s, type %s unknown.\" %\n                                 (urn, existing_type))\n\n    if aff4_type is not None and not isinstance(result, aff4_type):\n      raise InstantiationError(\n          \"Object %s is of type %s, but required_type is %s\" %\n          (urn, result.__class__.__name__, aff4_type.__name__))\n\n    return result", "response": "Opens the named object in the AFF4 data store."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef MultiOpen(self,\n                urns,\n                mode=\"rw\",\n                token=None,\n                aff4_type=None,\n                age=NEWEST_TIME,\n                follow_symlinks=True):\n    \"\"\"Opens a bunch of urns efficiently.\"\"\"\n\n    if not data_store.AFF4Enabled():\n      raise NotImplementedError(\"AFF4 data store has been disabled.\")\n\n    if token is None:\n      token = data_store.default_token\n\n    if mode not in [\"w\", \"r\", \"rw\"]:\n      raise ValueError(\"Invalid mode %s\" % mode)\n\n    symlinks = {}\n\n    _ValidateAFF4Type(aff4_type)\n\n    for urn, values in self.GetAttributes(urns, age=age):\n      try:\n        obj = self.Open(\n            urn,\n            mode=mode,\n            token=token,\n            local_cache={urn: values},\n            age=age,\n            follow_symlinks=False)\n        # We can't pass aff4_type to Open since it will raise on AFF4Symlinks.\n        # Setting it here, if needed, so that BadGetAttributeError checking\n        # works.\n        if aff4_type:\n          obj.aff4_type = aff4_type\n\n        if follow_symlinks and isinstance(obj, AFF4Symlink):\n          target = obj.Get(obj.Schema.SYMLINK_TARGET)\n          if target is not None:\n            symlinks.setdefault(target, []).append(obj.urn)\n        elif aff4_type:\n          if isinstance(obj, aff4_type):\n            yield obj\n        else:\n          yield obj\n      except IOError:\n        pass\n\n    if symlinks:\n      for obj in self.MultiOpen(\n          symlinks, mode=mode, token=token, aff4_type=aff4_type, age=age):\n        to_link = symlinks[obj.urn]\n        for additional_symlink in to_link[1:]:\n          clone = obj.__class__(obj.urn, clone=obj)\n          clone.symlink_urn = additional_symlink\n          yield clone\n\n        obj.symlink_urn = symlinks[obj.urn][0]\n        yield obj", "response": "Opens a bunch of urns efficiently."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nopening many URNs and returns handles in the same order.", "response": "def MultiOpenOrdered(self, urns, **kwargs):\n    \"\"\"Opens many URNs and returns handles in the same order.\n\n    `MultiOpen` can return file handles in arbitrary order. This makes it more\n    efficient and in most cases the order does not matter. However, there are\n    cases where order is important and this function should be used instead.\n\n    Args:\n      urns: A list of URNs to open.\n      **kwargs: Same keyword arguments as in `MultiOpen`.\n\n    Returns:\n      A list of file-like objects corresponding to the specified URNs.\n\n    Raises:\n      IOError: If one of the specified URNs does not correspond to the AFF4\n          object.\n    \"\"\"\n    precondition.AssertIterableType(urns, rdfvalue.RDFURN)\n\n    urn_filedescs = {}\n    for filedesc in self.MultiOpen(urns, **kwargs):\n      urn_filedescs[filedesc.urn] = filedesc\n\n    filedescs = []\n\n    for urn in urns:\n      try:\n        filedescs.append(urn_filedescs[urn])\n      except KeyError:\n        raise IOError(\"No associated AFF4 object for `%s`\" % urn)\n\n    return filedescs"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn all the versions of the object in the object list.", "response": "def OpenDiscreteVersions(self,\n                           urn,\n                           mode=\"r\",\n                           token=None,\n                           local_cache=None,\n                           age=ALL_TIMES,\n                           diffs_only=False,\n                           follow_symlinks=True):\n    \"\"\"Returns all the versions of the object as AFF4 objects.\n\n    This iterates through versions of an object, returning the newest version\n    first, then each older version until the beginning of time.\n\n    Note that versions are defined by changes to the TYPE attribute, and this\n    takes the version between two TYPE attributes.\n    In many cases as a user you don't want this, as you want to be returned an\n    object with as many attributes as possible, instead of the subset of them\n    that were Set between these two times.\n\n    Args:\n      urn: The urn to open.\n      mode: The mode to open the file with.\n      token: The Security Token to use for opening this item.\n      local_cache: A dict containing a cache as returned by GetAttributes. If\n        set, this bypasses the factory cache.\n      age: The age policy used to build this object. Should be one of ALL_TIMES\n        or a time range\n      diffs_only: If True, will return only diffs between the versions (instead\n        of full objects).\n      follow_symlinks: If object opened is a symlink, follow it.\n\n    Yields:\n      An AFF4Object for each version.\n\n    Raises:\n      IOError: On bad open or wrong time range specified.\n    \"\"\"\n    if age == NEWEST_TIME or len(age) == 1:\n      raise IOError(\"Bad age policy NEWEST_TIME for OpenDiscreteVersions.\")\n\n    aff4object = FACTORY.Open(\n        urn,\n        mode=mode,\n        token=token,\n        local_cache=local_cache,\n        age=age,\n        follow_symlinks=follow_symlinks)\n\n    # TYPE is always written last so we trust it to bound the version.\n    # Iterate from newest to oldest.\n    type_iter = aff4object.GetValuesForAttribute(aff4object.Schema.TYPE)\n    versions = sorted(t.age for t in type_iter)\n\n    additional_aff4object = None\n    # If we want full versions of the object, age != ALL_TIMES and we're\n    # dealing with a time range with a start != 0, then we need to fetch\n    # the version of the object corresponding to the start of the time\n    # range and use its attributes when doing filtering below.\n    #\n    # Same applies to fetching in DIFF mode and passing a time range in \"age\".\n    # Unversioned arguments will have a timestamp zero and won't be included\n    # into the diff. additional_aff4object is then used to account for them.\n    if len(age) == 2 and age[0]:\n      additional_aff4object = FACTORY.Open(\n          urn, mode=mode, token=token, local_cache=local_cache, age=age[0])\n\n    if diffs_only:\n      versions.insert(0, rdfvalue.RDFDatetime(0))\n      pairs = list(zip(versions, versions[1:]))\n    else:\n      pairs = [(rdfvalue.RDFDatetime(0), v) for v in versions]\n\n    if pairs:\n      right_end = pairs[-1][1]\n\n    for start, end in pairs:\n      # Create a subset of attributes for use in the new object that represents\n      # this version.\n      clone_attrs = {}\n\n      if additional_aff4object:\n        # Attributes are supposed to be in the decreasing timestamp\n        # order (see aff4.FACTORY.GetAttributes as a reference). Therefore\n        # appending additional_aff4object to the end (since it corresponds\n        # to the earliest version of the object).\n        attributes = itertools.chain(\n            iteritems(aff4object.synced_attributes),\n            iteritems(additional_aff4object.synced_attributes))\n      else:\n        attributes = iteritems(aff4object.synced_attributes)\n\n      for k, values in attributes:\n        reduced_v = []\n        for v in values:\n          if v.age > start and v.age <= end:\n            reduced_v.append(v)\n          elif v.age == 0 and end == right_end:\n            v.age = rdfvalue.RDFDatetime(end)\n            reduced_v.append(v)\n        clone_attrs.setdefault(k, []).extend(reduced_v)\n\n      cur_type = clone_attrs[aff4object.Schema.TYPE][0].ToRDFValue()\n      obj_cls = AFF4Object.classes[str(cur_type)]\n      new_obj = obj_cls(\n          urn,\n          mode=mode,\n          parent=aff4object.parent,\n          clone=clone_attrs,\n          token=token,\n          age=(start, end),\n          local_cache=local_cache,\n          follow_symlinks=follow_symlinks)\n      new_obj.Initialize()  # This is required to set local attributes.\n      yield new_obj"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning metadata about all objects in the urns.", "response": "def Stat(self, urns):\n    \"\"\"Returns metadata about all urns.\n\n    Currently the metadata include type, and last update time.\n\n    Args:\n      urns: The urns of the objects to open.\n\n    Yields:\n      A dict of metadata.\n\n    Raises:\n      ValueError: A string was passed instead of an iterable.\n    \"\"\"\n    if isinstance(urns, string_types):\n      raise ValueError(\"Expected an iterable, not string.\")\n    for subject, values in data_store.DB.MultiResolvePrefix(\n        urns, [\"aff4:type\", \"metadata:last\"]):\n      res = dict(urn=rdfvalue.RDFURN(subject))\n      for v in values:\n        if v[0] == \"aff4:type\":\n          res[\"type\"] = v\n        elif v[0] == \"metadata:last\":\n          res[\"last\"] = rdfvalue.RDFDatetime(v[1])\n      yield res"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncreate the object in the data store.", "response": "def Create(self,\n             urn,\n             aff4_type,\n             mode=\"w\",\n             token=None,\n             age=NEWEST_TIME,\n             force_new_version=True,\n             object_exists=False,\n             mutation_pool=None,\n             transaction=None):\n    \"\"\"Creates the urn if it does not already exist, otherwise opens it.\n\n    If the urn exists and is of a different type, this will also promote it to\n    the specified type.\n\n    Args:\n      urn: The object to create.\n      aff4_type: The desired type for this object.\n      mode: The desired mode for this object.\n      token: The Security Token to use for opening this item.\n      age: The age policy used to build this object. Only makes sense when mode\n        has \"r\".\n      force_new_version: Forces the creation of a new object in the data_store.\n      object_exists: If we know the object already exists we can skip index\n        creation.\n      mutation_pool: An optional MutationPool object to write to. If not given,\n        the data_store is used directly.\n      transaction: For locked objects, a lock is passed to the object.\n\n    Returns:\n      An AFF4 object of the desired type and mode.\n\n    Raises:\n      AttributeError: If the mode is invalid.\n    \"\"\"\n    if not data_store.AFF4Enabled():\n      raise NotImplementedError(\"AFF4 data store has been disabled.\")\n\n    if mode not in [\"w\", \"r\", \"rw\"]:\n      raise AttributeError(\"Invalid mode %s\" % mode)\n\n    if token is None:\n      token = data_store.default_token\n\n    if urn is not None:\n      urn = rdfvalue.RDFURN(urn)\n\n    _ValidateAFF4Type(aff4_type)\n\n    if \"r\" in mode:\n      # Check to see if an object already exists.\n      try:\n        existing = self.Open(\n            urn, mode=mode, token=token, age=age, transaction=transaction)\n\n        result = existing.Upgrade(aff4_type)\n\n        # We can't pass aff4_type into the Open call since it will raise with a\n        # type mismatch. We set it like this so BadGetAttributeError checking\n        # works.\n        if aff4_type:\n          result.aff4_type = aff4_type.__name__\n\n        if force_new_version and existing.Get(\n            result.Schema.TYPE) != aff4_type.__name__:\n          result.ForceNewVersion()\n        return result\n      except IOError:\n        pass\n\n    result = aff4_type(\n        urn,\n        mode=mode,\n        token=token,\n        age=age,\n        aff4_type=aff4_type.__name__,\n        object_exists=object_exists,\n        mutation_pool=mutation_pool,\n        transaction=transaction)\n    result.Initialize()\n    if force_new_version:\n      result.ForceNewVersion()\n\n    return result"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ndrop all the information about given objects. DANGEROUS! This recursively deletes all objects contained within the specified URN. Args: urns: Urns of objects to remove. token: The Security Token to use for opening this item. Raises: ValueError: If one of the urns is too short. This is a safety check to ensure the root is not removed.", "response": "def MultiDelete(self, urns, token=None):\n    \"\"\"Drop all the information about given objects.\n\n    DANGEROUS! This recursively deletes all objects contained within the\n    specified URN.\n\n    Args:\n      urns: Urns of objects to remove.\n      token: The Security Token to use for opening this item.\n\n    Raises:\n      ValueError: If one of the urns is too short. This is a safety check to\n      ensure the root is not removed.\n    \"\"\"\n    urns = [rdfvalue.RDFURN(urn) for urn in urns]\n\n    if token is None:\n      token = data_store.default_token\n\n    for urn in urns:\n      if urn.Path() == \"/\":\n        raise ValueError(\"Can't delete root URN. Please enter a valid URN\")\n\n    deletion_pool = DeletionPool(token=token)\n    deletion_pool.MultiMarkForDeletion(urns)\n\n    marked_root_urns = deletion_pool.root_urns_for_deletion\n    marked_urns = deletion_pool.urns_for_deletion\n\n    logging.debug(u\"Found %d objects to remove when removing %s\",\n                  len(marked_urns), urns)\n\n    logging.debug(u\"Removing %d root objects when removing %s: %s\",\n                  len(marked_root_urns), urns, marked_root_urns)\n\n    pool = data_store.DB.GetMutationPool()\n    for root in marked_root_urns:\n      # Only the index of the parent object should be updated. Everything\n      # below the target object (along with indexes) is going to be\n      # deleted.\n      self._DeleteChildFromIndex(root, mutation_pool=pool)\n\n    for urn_to_delete in marked_urns:\n      try:\n        self.intermediate_cache.ExpireObject(urn_to_delete.Path())\n      except KeyError:\n        pass\n\n    pool.DeleteSubjects(marked_urns)\n    pool.Flush()\n\n    # Ensure this is removed from the cache as well.\n    self.Flush()\n\n    logging.debug(\"Removed %d objects\", len(marked_urns))"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef MultiListChildren(self, urns, limit=None, age=NEWEST_TIME):\n    checked_subjects = set()\n\n    for subject, values in data_store.DB.AFF4MultiFetchChildren(\n        urns, timestamp=Factory.ParseAgeSpecification(age), limit=limit):\n\n      checked_subjects.add(subject)\n\n      subject_result = []\n      for child, timestamp in values:\n        urn = rdfvalue.RDFURN(subject).Add(child)\n        urn.age = rdfvalue.RDFDatetime(timestamp)\n        subject_result.append(urn)\n\n      yield subject, subject_result\n\n    for subject in set(urns) - checked_subjects:\n      yield subject, []", "response": "Lists bunch of directories efficiently."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef ListChildren(self, urn, limit=None, age=NEWEST_TIME):\n    _, children_urns = list(\n        self.MultiListChildren([urn], limit=limit, age=age))[0]\n    return children_urns", "response": "Lists the children of a given resource."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef RecursiveMultiListChildren(self, urns, limit=None, age=NEWEST_TIME):\n\n    checked_urns = set()\n    urns_to_check = urns\n    while True:\n      found_children = []\n\n      for subject, values in self.MultiListChildren(\n          urns_to_check, limit=limit, age=age):\n\n        found_children.extend(values)\n        yield subject, values\n\n      checked_urns.update(urns_to_check)\n\n      urns_to_check = set(found_children) - checked_urns\n      if not urns_to_check:\n        break", "response": "Recursively lists a bunch of directories."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning a copy without registering in the attribute registry.", "response": "def Copy(self):\n    \"\"\"Return a copy without registering in the attribute registry.\"\"\"\n    return Attribute(\n        self.predicate,\n        self.attribute_type,\n        self.description,\n        self.name,\n        _copy=True)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef GetRDFValueType(self):\n    result = self.attribute_type\n    for field_name in self.field_names:\n      # Support the new semantic protobufs.\n      try:\n        result = result.type_infos.get(field_name).type\n      except AttributeError:\n        raise AttributeError(\"Invalid attribute %s\" % field_name)\n\n    return result", "response": "Returns this attribute s RDFValue class."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef GetSubFields(self, fd, field_names):\n\n    if isinstance(fd, rdf_protodict.RDFValueArray):\n      for value in fd:\n        for res in self._GetSubField(value, field_names):\n          yield res\n    else:\n      for res in self._GetSubField(fd, field_names):\n        yield res", "response": "Gets all the subfields indicated by field_names."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn the values for this attribute as stored in an AFF4Object.", "response": "def GetValues(self, fd):\n    \"\"\"Return the values for this attribute as stored in an AFF4Object.\"\"\"\n    result = None\n    for result in fd.new_attributes.get(self, []):\n      # We need to interpolate sub fields in this rdfvalue.\n      if self.field_names:\n        for x in self.GetSubFields(result, self.field_names):\n          yield x\n\n      else:\n        yield result\n\n    for result in fd.synced_attributes.get(self, []):\n      result = result.ToRDFValue()\n\n      # We need to interpolate sub fields in this rdfvalue.\n      if result is not None:\n        if self.field_names:\n          for x in self.GetSubFields(result, self.field_names):\n            yield x\n\n        else:\n          yield result\n\n    if result is None:\n      default = self.GetDefault(fd)\n      if default is not None:\n        yield default"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn a default attribute if it is not set.", "response": "def GetDefault(self, fd=None, default=None):\n    \"\"\"Returns a default attribute if it is not set.\"\"\"\n    if callable(self.default):\n      return self.default(fd)\n\n    if self.default is not None:\n      # We can't return mutable objects here or the default might change for all\n      # objects of this class.\n      if isinstance(self.default, rdfvalue.RDFValue):\n        default = self.default.Copy()\n        default.attribute_instance = self\n        return self(default)\n      else:\n        return self(self.default)\n\n    if isinstance(default, rdfvalue.RDFValue):\n      default = default.Copy()\n      default.attribute_instance = self\n\n    return default"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ngive a serialized value decode the attribute.", "response": "def DecodeValueFromAttribute(self, attribute_name, value, ts):\n    \"\"\"Given a serialized value, decode the attribute.\n\n    Only attributes which have been previously defined are permitted.\n\n    Args:\n       attribute_name: The string name of the attribute.\n       value: The serialized attribute value.\n       ts: The timestamp of this attribute.\n    \"\"\"\n    try:\n      # Get the Attribute object from our schema.\n      attribute = Attribute.PREDICATES[attribute_name]\n      cls = attribute.attribute_type\n      self._AddAttributeToCache(attribute, LazyDecoder(cls, value, ts),\n                                self.synced_attributes)\n    except KeyError:\n      pass\n    except (ValueError, rdfvalue.DecodeError):\n      logging.debug(\"%s: %s invalid encoding. Skipping.\", self.urn,\n                    attribute_name)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef UpdateLease(self, duration):\n    if not self.locked:\n      raise LockError(\"Object must be locked to update the lease: %s.\" %\n                      self.urn)\n\n    if self.CheckLease() == 0:\n      self._RaiseLockError(\"UpdateLease\")\n\n    self.transaction.UpdateLease(duration)", "response": "Updates the lease and flushes the object."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nclosing and destroys the object.", "response": "def Close(self):\n    \"\"\"Close and destroy the object.\n\n    This is similar to Flush, but does not maintain object validity. Hence the\n    object should not be interacted with after Close().\n\n    Raises:\n       LockError: The lease for this object has expired.\n    \"\"\"\n    if self.locked and self.CheckLease() == 0:\n      raise LockError(\"Can not update lease that has already expired.\")\n\n    self._WriteAttributes()\n\n    # Releasing this lock allows another thread to own it.\n    if self.locked:\n      self.transaction.Release()\n\n    if self.parent:\n      self.parent.Close()\n\n    # Interacting with a closed object is a bug. We need to catch this ASAP so\n    # we remove all mode permissions from this object.\n    self.mode = \"\""}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nwrites the dirty attributes to the data store.", "response": "def _WriteAttributes(self):\n    \"\"\"Write the dirty attributes to the data store.\"\"\"\n    # If the object is not opened for writing we do not need to flush it to the\n    # data_store.\n    if \"w\" not in self.mode:\n      return\n\n    if self.urn is None:\n      raise ValueError(\"Storing of anonymous AFF4 objects not supported.\")\n\n    to_set = {}\n    for attribute_name, value_array in iteritems(self.new_attributes):\n      to_set_list = to_set.setdefault(attribute_name, [])\n      for value in value_array:\n        to_set_list.append((value.SerializeToDataStore(), value.age))\n\n    if self._dirty:\n      # We determine this object has a new version only if any of the versioned\n      # attributes have changed. Non-versioned attributes do not represent a new\n      # object version. The type of an object is versioned and represents a\n      # version point in the life of the object.\n      if self._new_version:\n        to_set[self.Schema.TYPE] = [\n            (utils.SmartUnicode(self.__class__.__name__),\n             rdfvalue.RDFDatetime.Now())\n        ]\n\n      # We only update indexes if the schema does not forbid it and we are not\n      # sure that the object already exists.\n      add_child_index = self.Schema.ADD_CHILD_INDEX\n      if self.object_exists:\n        add_child_index = False\n\n      # Write the attributes to the Factory cache.\n      FACTORY.SetAttributes(\n          self.urn,\n          to_set,\n          self._to_delete,\n          add_child_index=add_child_index,\n          mutation_pool=self.mutation_pool)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _SyncAttributes(self):\n    # This effectively moves all the values from the new_attributes to the\n    # synced_attributes caches.\n    for attribute, value_array in iteritems(self.new_attributes):\n      if not attribute.versioned or self.age_policy == NEWEST_TIME:\n        # Store the latest version if there are multiple unsynced versions.\n        value = value_array[-1]\n        self.synced_attributes[attribute] = [\n            LazyDecoder(decoded=value, age=value.age)\n        ]\n\n      else:\n        synced_value_array = self.synced_attributes.setdefault(attribute, [])\n        for value in value_array:\n          synced_value_array.append(LazyDecoder(decoded=value, age=value.age))\n\n        synced_value_array.sort(key=lambda x: x.age, reverse=True)\n\n    self.new_attributes = {}\n    self._to_delete.clear()\n    self._dirty = False\n    self._new_version = False", "response": "Sync the new attributes to the synced attribute cache."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncheck that the value for the given attribute is of the expected type.", "response": "def _CheckAttribute(self, attribute, value):\n    \"\"\"Check that the value is of the expected type.\n\n    Args:\n       attribute: An instance of Attribute().\n       value: An instance of RDFValue.\n\n    Raises:\n       ValueError: when the value is not of the expected type.\n       AttributeError: When the attribute is not of type Attribute().\n    \"\"\"\n    if not isinstance(attribute, Attribute):\n      raise AttributeError(\"Attribute %s must be of type aff4.Attribute()\" %\n                           attribute)\n\n    if not isinstance(value, attribute.attribute_type):\n      raise ValueError(\"Value for attribute %s must be of type %s()\" %\n                       (attribute, attribute.attribute_type.__name__))"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nadd an additional attribute to this object.", "response": "def AddAttribute(self, attribute, value=None, age=None):\n    \"\"\"Add an additional attribute to this object.\n\n    If value is None, attribute is expected to be already initialized with a\n    value. For example:\n\n    fd.AddAttribute(fd.Schema.CONTAINS(\"some data\"))\n\n    Args:\n       attribute: The attribute name or an RDFValue derived from the attribute.\n       value: The value the attribute will be set to.\n       age: Age (timestamp) of the attribute. If None, current time is used.\n\n    Raises:\n       IOError: If this object is read only.\n    \"\"\"\n    if \"w\" not in self.mode:\n      raise IOError(\"Writing attribute %s to read only object.\" % attribute)\n\n    if value is None:\n      value = attribute\n      attribute = value.attribute_instance\n\n    # Check if this object should be locked in order to add the attribute.\n    # NOTE: We don't care about locking when doing blind writes.\n    if self.mode != \"w\" and attribute.lock_protected and not self.transaction:\n      raise IOError(\"Object must be locked to write attribute %s.\" % attribute)\n\n    self._CheckAttribute(attribute, value)\n    # Does this represent a new version?\n    if attribute.versioned:\n      if attribute.creates_new_object_version:\n        self._new_version = True\n\n      # Update the time of this new attribute.\n      if age:\n        value.age = age\n      else:\n        value.age = rdfvalue.RDFDatetime.Now()\n\n    # Non-versioned attributes always replace previous versions and get written\n    # at the earliest timestamp (so they appear in all objects).\n    else:\n      self._to_delete.add(attribute)\n      self.synced_attributes.pop(attribute, None)\n      self.new_attributes.pop(attribute, None)\n      value.age = 0\n\n    self._AddAttributeToCache(attribute, value, self.new_attributes)\n    self._dirty = True"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nclearing the attribute from this object.", "response": "def DeleteAttribute(self, attribute):\n    \"\"\"Clears the attribute from this object.\"\"\"\n    if \"w\" not in self.mode:\n      raise IOError(\"Deleting attribute %s from read only object.\" % attribute)\n\n    # Check if this object should be locked in order to delete the attribute.\n    # NOTE: We don't care about locking when doing blind writes.\n    if self.mode != \"w\" and attribute.lock_protected and not self.transaction:\n      raise IOError(\"Object must be locked to delete attribute %s.\" % attribute)\n\n    if attribute in self.synced_attributes:\n      self._to_delete.add(attribute)\n      del self.synced_attributes[attribute]\n\n    if attribute in self.new_attributes:\n      del self.new_attributes[attribute]\n\n    # Does this represent a new version?\n    if attribute.versioned and attribute.creates_new_object_version:\n      self._new_version = True\n\n    self._dirty = True"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef Get(self, attribute, default=None):\n    if attribute is None:\n      return default\n\n    # Allow the user to specify the attribute by name.\n    elif isinstance(attribute, str):\n      attribute = Attribute.GetAttributeByName(attribute)\n\n    # We can't read attributes from the data_store unless read mode was\n    # specified. It is ok to read new attributes though.\n    if \"r\" not in self.mode and (attribute not in self.new_attributes and\n                                 attribute not in self.synced_attributes):\n      raise IOError(\"Fetching %s from object not opened for reading.\" %\n                    attribute)\n\n    for result in self.GetValuesForAttribute(attribute, only_one=True):\n      try:\n        # The attribute may be a naked string or int - i.e. not an RDFValue at\n        # all.\n        result.attribute_instance = attribute\n      except AttributeError:\n        pass\n\n      return result\n\n    return attribute.GetDefault(self, default)", "response": "Gets the attribute from this object."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns a list of values from this attribute.", "response": "def GetValuesForAttribute(self, attribute, only_one=False):\n    \"\"\"Returns a list of values from this attribute.\"\"\"\n    if not only_one and self.age_policy == NEWEST_TIME:\n      raise ValueError(\"Attempting to read all attribute versions for an \"\n                       \"object opened for NEWEST_TIME. This is probably \"\n                       \"not what you want.\")\n\n    if attribute is None:\n      return []\n\n    elif isinstance(attribute, string_types):\n      attribute = Attribute.GetAttributeByName(attribute)\n\n    return attribute.GetValues(self)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef Upgrade(self, aff4_class):\n    _ValidateAFF4Type(aff4_class)\n\n    # We are already of the required type\n    if self.__class__ == aff4_class:\n      return self\n\n    # Check that we have the right type.\n    if not isinstance(aff4_class, type):\n      raise InstantiationError(\"aff4_class=%s must be a type\" % aff4_class)\n    if not issubclass(aff4_class, AFF4Object):\n      raise InstantiationError(\n          \"aff4_class=%s must be a subclass of AFF4Object.\" % aff4_class)\n\n    # It's not allowed to downgrade the object\n    if isinstance(self, aff4_class):\n      # TODO(user): check what we should do here:\n      #                 1) Nothing\n      #                 2) raise\n      #                 3) return self\n      # Option 3) seems ok, but we need to be sure that we don't use\n      # Create(mode='r') anywhere where code actually expects the object to be\n      # downgraded.\n      return self\n\n    # NOTE: It is possible for attributes to become inaccessible here if the old\n    # object has an attribute which the new object does not have in its\n    # schema. The values of these attributes will not be available any longer in\n    # the new object - usually because old attributes do not make sense in the\n    # context of the new object.\n\n    # Instantiate the class\n    result = aff4_class(\n        self.urn,\n        mode=self.mode,\n        clone=self,\n        parent=self.parent,\n        token=self.token,\n        age=self.age_policy,\n        object_exists=self.object_exists,\n        follow_symlinks=self.follow_symlinks,\n        aff4_type=self.aff4_type,\n        mutation_pool=self.mutation_pool,\n        transaction=self.transaction)\n    result.symlink_urn = self.urn\n    result.Initialize()\n\n    return result", "response": "Upgrades this object to the type specified."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef AddLabels(self, labels_names, owner=None):\n    if owner is None and not self.token:\n      raise ValueError(\"Can't set label: No owner specified and \"\n                       \"no access token available.\")\n    if isinstance(labels_names, string_types):\n      raise ValueError(\"Label list can't be string.\")\n\n    owner = owner or self.token.username\n\n    current_labels = self.Get(self.Schema.LABELS, self.Schema.LABELS())\n    for label_name in labels_names:\n      label = rdf_aff4.AFF4ObjectLabel(\n          name=label_name, owner=owner, timestamp=rdfvalue.RDFDatetime.Now())\n      current_labels.AddLabel(label)\n\n    self.Set(current_labels)", "response": "Adds labels to the AFF4Object."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nremoving specified labels from the AFF4Object.", "response": "def RemoveLabels(self, labels_names, owner=None):\n    \"\"\"Remove specified labels from the AFF4Object.\"\"\"\n    if owner is None and not self.token:\n      raise ValueError(\"Can't remove label: No owner specified and \"\n                       \"no access token available.\")\n    if isinstance(labels_names, string_types):\n      raise ValueError(\"Label list can't be string.\")\n\n    owner = owner or self.token.username\n\n    current_labels = self.Get(self.Schema.LABELS)\n    for label_name in labels_names:\n      label = rdf_aff4.AFF4ObjectLabel(name=label_name, owner=owner)\n      current_labels.RemoveLabel(label)\n\n    self.Set(self.Schema.LABELS, current_labels)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nchecking that attribute is a valid Attribute instance.", "response": "def SetAttribute(self, attribute):\n    \"\"\"Checks that attribute is a valid Attribute() instance.\"\"\"\n    # Grab the attribute registered for this name\n    self.attribute = attribute\n    self.attribute_obj = Attribute.GetAttributeByName(attribute)\n    if self.attribute_obj is None:\n      raise lexer.ParseError(\"Attribute %s not defined\" % attribute)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef SetOperator(self, operator):\n    self.operator = operator\n    # Find the appropriate list of operators for this attribute\n    attribute_type = self.attribute_obj.GetRDFValueType()\n    operators = attribute_type.operators\n\n    # Do we have such an operator?\n    self.number_of_args, self.operator_method = operators.get(\n        operator, (0, None))\n\n    if self.operator_method is None:\n      raise lexer.ParseError(\"Operator %s not defined on attribute '%s'\" %\n                             (operator, self.attribute))\n\n    self.operator_method = getattr(attribute_type, self.operator_method)", "response": "Sets the operator for this expression."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn the data_store filter implementation from the attribute.", "response": "def Compile(self, filter_implemention):\n    \"\"\"Returns the data_store filter implementation from the attribute.\"\"\"\n    return self.operator_method(self.attribute_obj, filter_implemention,\n                                *self.args)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef ListChildren(self, limit=None, age=NEWEST_TIME):\n    # Just grab all the children from the index.\n    for predicate, timestamp in data_store.DB.AFF4FetchChildren(\n        self.urn, timestamp=Factory.ParseAgeSpecification(age), limit=limit):\n      urn = self.urn.Add(predicate)\n      urn.age = rdfvalue.RDFDatetime(timestamp)\n      yield urn", "response": "Returns an iterator over the children of this object."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn an AFF4 object for all our direct children.", "response": "def OpenChildren(self,\n                   children=None,\n                   mode=\"r\",\n                   limit=None,\n                   chunk_limit=100000,\n                   age=NEWEST_TIME):\n    \"\"\"Yields AFF4 Objects of all our direct children.\n\n    This method efficiently returns all attributes for our children directly, in\n    a few data store round trips. We use the directory indexes to query the data\n    store.\n\n    Args:\n      children: A list of children RDFURNs to open. If None open all our\n        children.\n      mode: The mode the files should be opened with.\n      limit: Total number of items we will attempt to retrieve.\n      chunk_limit: Maximum number of items to retrieve at a time.\n      age: The age of the items to retrieve. Should be one of ALL_TIMES,\n        NEWEST_TIME or a range.\n\n    Yields:\n      Instances for each direct child.\n    \"\"\"\n    if children is None:\n      # No age passed here to avoid ignoring indexes that were updated\n      # to a timestamp greater than the object's age.\n      subjects = list(self.ListChildren())\n    else:\n      subjects = list(children)\n    subjects.sort()\n    result_count = 0\n    # Read at most limit children at a time.\n    while subjects:\n      to_read = subjects[:chunk_limit]\n      subjects = subjects[chunk_limit:]\n      for child in FACTORY.MultiOpen(\n          to_read, mode=mode, token=self.token, age=age):\n        yield child\n        result_count += 1\n        if limit and result_count >= limit:\n          return"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a pathspec for an aff4 object even if there is none stored.", "response": "def real_pathspec(self):\n    \"\"\"Returns a pathspec for an aff4 object even if there is none stored.\"\"\"\n    pathspec = self.Get(self.Schema.PATHSPEC)\n\n    stripped_components = []\n    parent = self\n\n    # TODO(user): this code is potentially slow due to multiple separate\n    # aff4.FACTORY.Open() calls. OTOH the loop below is executed very rarely -\n    # only when we deal with deep files that got fetched alone and then\n    # one of the directories in their path gets updated.\n    while not pathspec and len(parent.urn.Split()) > 1:\n      # We try to recurse up the tree to get a real pathspec.\n      # These directories are created automatically without pathspecs when a\n      # deep directory is listed without listing the parents.\n      # Note /fs/os or /fs/tsk won't be updateable so we will raise IOError\n      # if we try.\n      stripped_components.append(parent.urn.Basename())\n      pathspec = parent.Get(parent.Schema.PATHSPEC)\n      parent = FACTORY.Open(parent.urn.Dirname(), token=self.token)\n\n    if pathspec:\n      if stripped_components:\n        # We stripped pieces of the URL, time to add them back.\n        new_path = utils.JoinPath(*reversed(stripped_components[:-1]))\n        pathspec.Append(\n            rdf_paths.PathSpec(path=new_path, pathtype=pathspec.last.pathtype))\n    else:\n      raise IOError(\"Item has no pathspec.\")\n\n    return pathspec"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _MultiStream(cls, fds):\n    for fd in fds:\n      fd.Seek(0)\n      while True:\n        chunk = fd.Read(cls.MULTI_STREAM_CHUNK_SIZE)\n        if not chunk:\n          break\n        yield fd, chunk, None", "response": "Method overriden by subclasses to optimize the MultiStream behavior."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef MultiStream(cls, fds):\n    for fd in fds:\n      if not isinstance(fd, AFF4Stream):\n        raise ValueError(\"All object to be streamed have to inherit from \"\n                         \"AFF4Stream (found one inheriting from %s).\" %\n                         (fd.__class__.__name__))\n\n    classes_map = {}\n    for fd in fds:\n      classes_map.setdefault(fd.__class__, []).append(fd)\n\n    for fd_class, fds in iteritems(classes_map):\n      # pylint: disable=protected-access\n      for fd, chunk, exception in fd_class._MultiStream(fds):\n        yield fd, chunk, exception", "response": "Effectively streams data from multiple opened AFF4Streams objects."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef Initialize(self):\n    super(AFF4MemoryStreamBase, self).Initialize()\n    contents = b\"\"\n\n    if \"r\" in self.mode:\n      contents = self.Get(self.Schema.CONTENT).AsBytes()\n      try:\n        if contents is not None:\n          contents = zlib.decompress(contents)\n      except zlib.error:\n        pass\n\n    self.fd = io.BytesIO(contents)\n    self.size = len(contents)\n    self.offset = 0", "response": "Try to load the data from the store."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _MultiStream(cls, fds):\n\n    missing_chunks_by_fd = {}\n    for chunk_fd_pairs in collection.Batch(\n        cls._GenerateChunkPaths(fds), cls.MULTI_STREAM_CHUNKS_READ_AHEAD):\n\n      chunks_map = dict(chunk_fd_pairs)\n      contents_map = {}\n      for chunk_fd in FACTORY.MultiOpen(\n          chunks_map, mode=\"r\", token=fds[0].token):\n        if isinstance(chunk_fd, AFF4Stream):\n          fd = chunks_map[chunk_fd.urn]\n          contents_map[chunk_fd.urn] = chunk_fd.read()\n\n      for chunk_urn, fd in chunk_fd_pairs:\n        if chunk_urn not in contents_map or not contents_map[chunk_urn]:\n          missing_chunks_by_fd.setdefault(fd, []).append(chunk_urn)\n\n      for chunk_urn, fd in chunk_fd_pairs:\n        if fd in missing_chunks_by_fd:\n          continue\n\n        yield fd, contents_map[chunk_urn], None\n\n    for fd, missing_chunks in iteritems(missing_chunks_by_fd):\n      e = MissingChunksError(\n          \"%d missing chunks (multi-stream).\" % len(missing_chunks),\n          missing_chunks=missing_chunks)\n      yield fd, None, e", "response": "Effectively streams data from multiple opened AFF4ImageBase objects."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef Initialize(self):\n    super(AFF4ImageBase, self).Initialize()\n    self.offset = 0\n    # A cache for segments.\n    self.chunk_cache = ChunkCache(self._WriteChunk, 100)\n\n    if \"r\" in self.mode:\n      self.size = int(self.Get(self.Schema.SIZE))\n      # pylint: disable=protected-access\n      self.chunksize = int(self.Get(self.Schema._CHUNKSIZE))\n      # pylint: enable=protected-access\n      self.content_last = self.Get(self.Schema.CONTENT_LAST)\n    else:\n      self.size = 0\n      self.content_last = None", "response": "Initialize the AFF4ImageBase object."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreading as much as possible but not more than length.", "response": "def _ReadPartial(self, length):\n    \"\"\"Read as much as possible, but not more than length.\"\"\"\n    chunk = self.offset // self.chunksize\n    chunk_offset = self.offset % self.chunksize\n\n    available_to_read = min(length, self.chunksize - chunk_offset)\n\n    retries = 0\n    while retries < self.NUM_RETRIES:\n      fd = self._GetChunkForReading(chunk)\n      if fd:\n        break\n      # Arriving here means we know about blobs that cannot be found in the db.\n      # The most likely reason is that they have not been synced yet so we\n      # retry a couple of times just in case they come in eventually.\n      logging.warning(\"Chunk not found.\")\n      time.sleep(1)\n      retries += 1\n\n    if retries >= self.NUM_RETRIES:\n      raise IOError(\"Chunk not found for reading.\")\n\n    fd.seek(chunk_offset)\n\n    result = fd.read(available_to_read)\n    self.offset += len(result)\n\n    return result"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef Read(self, length):\n    result = b\"\"\n\n    # The total available size in the file\n    length = int(length)\n    length = min(length, self.size - self.offset)\n\n    while length > 0:\n      data = self._ReadPartial(length)\n      if not data:\n        break\n\n      length -= len(data)\n      result += data\n    return result", "response": "Reads a block of data from the file."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _WritePartial(self, data):\n\n    chunk = self.offset // self.chunksize\n    chunk_offset = self.offset % self.chunksize\n    data = utils.SmartStr(data)\n\n    available_to_write = min(len(data), self.chunksize - chunk_offset)\n\n    fd = self._GetChunkForWriting(chunk)\n    fd.seek(chunk_offset)\n\n    fd.write(data[:available_to_write])\n    self.offset += available_to_write\n\n    return data[available_to_write:]", "response": "Writes at most one chunk of data."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef Encode(self, attribute, value):\n    required_type = self._attribute_types.get(attribute, \"bytes\")\n\n    if required_type == \"integer\":\n      return rdf_structs.SignedVarintEncode(int(value))\n    elif required_type == \"unsigned_integer\":\n      return rdf_structs.VarintEncode(int(value))\n    elif hasattr(value, \"SerializeToString\"):\n      return value.SerializeToString()\n    elif isinstance(value, bytes):\n      return value\n    elif isinstance(value, Text):\n      return value.encode(\"utf-8\")\n    else:\n      return str(value).encode(\"utf-8\")", "response": "Encodes the value for the attribute."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef Decode(self, attribute, value):\n    required_type = self._attribute_types.get(attribute, \"bytes\")\n    if required_type == \"integer\":\n      return rdf_structs.SignedVarintReader(value, 0)[0]\n    elif required_type == \"unsigned_integer\":\n      return rdf_structs.VarintReader(value, 0)[0]\n    elif required_type == \"string\":\n      if isinstance(value, bytes):\n        return value.decode(\"utf-8\")\n      else:\n        return utils.SmartUnicode(value)\n    else:\n      return value", "response": "Decode the value to the required type."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef CleanAff4Hunts(self):\n\n    hunts_ttl = config.CONFIG[\"DataRetention.hunts_ttl\"]\n    if not hunts_ttl:\n      self.Log(\"TTL not set - nothing to do...\")\n      return\n\n    exception_label = config.CONFIG[\"DataRetention.hunts_ttl_exception_label\"]\n\n    hunts_root = aff4.FACTORY.Open(\"aff4:/hunts\", token=self.token)\n    hunts_urns = list(hunts_root.ListChildren())\n\n    deadline = rdfvalue.RDFDatetime.Now() - hunts_ttl\n\n    hunts_deleted = 0\n\n    hunts = aff4.FACTORY.MultiOpen(\n        hunts_urns, aff4_type=implementation.GRRHunt, token=self.token)\n    for hunt in hunts:\n      if exception_label in hunt.GetLabelsNames():\n        continue\n\n      runner = hunt.GetRunner()\n      if runner.context.create_time + runner.context.duration < deadline:\n        aff4.FACTORY.Delete(hunt.urn, token=self.token)\n        hunts_deleted += 1\n        self.HeartBeat()\n    self.Log(\"Deleted %d hunts.\" % hunts_deleted)", "response": "Cleans up old hunt data from aff4."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef Start(self):\n    cron_jobs_ttl = config.CONFIG[\"DataRetention.cron_jobs_flows_ttl\"]\n    if not cron_jobs_ttl:\n      self.Log(\"TTL not set - nothing to do...\")\n      return\n\n    manager = aff4_cronjobs.GetCronManager()\n    cutoff_timestamp = rdfvalue.RDFDatetime.Now() - cron_jobs_ttl\n    if data_store.RelationalDBEnabled():\n      deletion_count = manager.DeleteOldRuns(cutoff_timestamp=cutoff_timestamp)\n\n    else:\n      deletion_count = 0\n      for job in manager.ReadJobs(token=self.token):\n        deletion_count += manager.DeleteOldRuns(\n            job, cutoff_timestamp=cutoff_timestamp, token=self.token)\n        self.HeartBeat()\n\n    self.Log(\"Deleted %d cron job runs.\" % deletion_count)", "response": "Cleans up old cron job data."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef CleanAff4Clients(self):\n\n    inactive_client_ttl = config.CONFIG[\"DataRetention.inactive_client_ttl\"]\n    if not inactive_client_ttl:\n      self.Log(\"TTL not set - nothing to do...\")\n      return\n\n    exception_label = config.CONFIG[\n        \"DataRetention.inactive_client_ttl_exception_label\"]\n\n    index = client_index.CreateClientIndex(token=self.token)\n\n    client_urns = index.LookupClients([\".\"])\n\n    deadline = rdfvalue.RDFDatetime.Now() - inactive_client_ttl\n    deletion_count = 0\n\n    for client_group in collection.Batch(client_urns, 1000):\n      inactive_client_urns = []\n      for client in aff4.FACTORY.MultiOpen(\n          client_group,\n          mode=\"r\",\n          aff4_type=aff4_grr.VFSGRRClient,\n          token=self.token):\n        if exception_label in client.GetLabelsNames():\n          continue\n\n        if client.Get(client.Schema.LAST) < deadline:\n          inactive_client_urns.append(client.urn)\n\n      aff4.FACTORY.MultiDelete(inactive_client_urns, token=self.token)\n      deletion_count += len(inactive_client_urns)\n      self.HeartBeat()\n\n    self.Log(\"Deleted %d inactive clients.\" % deletion_count)", "response": "Cleans up old client data from aff4."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ndoes basic token validation. Args: token: User's credentials as access_control.ACLToken. targets: List of targets that were meant to be accessed by the token. This is used for logging purposes only. Returns: True if token is valid. Raises: access_control.UnauthorizedAccess: if token is not valid. ValueError: if targets list is empty.", "response": "def ValidateToken(token, targets):\n  \"\"\"Does basic token validation.\n\n  Args:\n    token: User's credentials as access_control.ACLToken.\n    targets: List of targets that were meant to be accessed by the token. This\n      is used for logging purposes only.\n\n  Returns:\n    True if token is valid.\n\n  Raises:\n    access_control.UnauthorizedAccess: if token is not valid.\n    ValueError: if targets list is empty.\n  \"\"\"\n\n  def GetSubjectForError():\n    if len(targets) == 1:\n      return list(targets)[0]\n    else:\n      return None\n\n  # All accesses need a token.\n  if not token:\n    raise access_control.UnauthorizedAccess(\n        \"Must give an authorization token for %s\" % targets,\n        subject=GetSubjectForError())\n\n  # Token must not be expired here.\n  token.CheckExpiry()\n\n  # Token must have identity\n  if not token.username:\n    raise access_control.UnauthorizedAccess(\n        \"Must specify a username for access to %s.\" % targets,\n        subject=GetSubjectForError())\n\n  return True"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nvalidates that the requested access and subjects are valid.", "response": "def ValidateAccessAndSubjects(requested_access, subjects):\n  \"\"\"Does basic requested access validation.\n\n  Args:\n    requested_access: String consisting or 'r', 'w' and 'q' characters.\n    subjects: A list of subjects that are about to be accessed with a given\n      requested_access. Used for logging purposes only.\n\n  Returns:\n    True if requested_access is valid.\n\n  Raises:\n    access_control.UnauthorizedAccess: if requested_access is not valid.\n    ValueError: if subjects list is empty.\n  \"\"\"\n\n  if not requested_access:\n    raise access_control.UnauthorizedAccess(\n        \"Must specify requested access type for %s\" % subjects)\n\n  for s in requested_access:\n    if s not in \"rwq\":\n      raise ValueError(\n          \"Invalid access requested for %s: %s\" % (subjects, requested_access))\n\n  if \"q\" in requested_access and \"r\" not in requested_access:\n    raise access_control.UnauthorizedAccess(\n        \"Invalid access request: query permissions require read permissions \"\n        \"for %s\" % subjects,\n        requested_access=requested_access)\n\n  return True"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef CheckUserForLabels(username, authorized_labels, token=None):\n  authorized_labels = set(authorized_labels)\n\n  try:\n    user = aff4.FACTORY.Open(\n        \"aff4:/users/%s\" % username, aff4_type=aff4_users.GRRUser, token=token)\n\n    # Only return if all the authorized_labels are found in the user's\n    # label list, otherwise raise UnauthorizedAccess.\n    if (authorized_labels.intersection(\n        user.GetLabelsNames()) == authorized_labels):\n      return True\n    else:\n      raise access_control.UnauthorizedAccess(\n          \"User %s is missing labels (required: %s).\" % (username,\n                                                         authorized_labels))\n  except IOError:\n    raise access_control.UnauthorizedAccess(\"User %s not found.\" % username)", "response": "Verify that the username has all the authorized_labels set."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef CheckFlowCanBeStartedOnClient(flow_name):\n  flow_cls = flow.GRRFlow.GetPlugin(flow_name)\n\n  if flow_cls.category:\n    return True\n  else:\n    raise access_control.UnauthorizedAccess(\n        \"Flow %s can't be started on a client by non-suid users.\" % flow_name)", "response": "Checks if a flow can be started on a particular client."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef Allow(self, path, require=None, *args, **kwargs):\n    regex_text = fnmatch.translate(path)\n    regex = re.compile(regex_text)\n    self.checks.append((regex_text, regex, require, args, kwargs))", "response": "This method checks if given path pattern fits the subject passed in constructor."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef CheckAccess(self, subject, token):\n    subject = rdfvalue.RDFURN(subject)\n    subject_str = subject.SerializeToString()\n\n    for check_tuple in self.checks:\n      regex_text, regex, require, require_args, require_kwargs = check_tuple\n\n      match = regex.match(subject_str)\n      if not match:\n        continue\n\n      if require:\n        # If require() fails, it raises access_control.UnauthorizedAccess.\n        require(subject, token, *require_args, **require_kwargs)\n\n      logging.debug(\n          u\"Datastore access granted to %s on %s by pattern: %s \"\n          u\"with reason: %s (require=%s, require_args=%s, \"\n          u\"require_kwargs=%s, helper_name=%s)\",\n          utils.SmartUnicode(token.username), utils.SmartUnicode(subject_str),\n          utils.SmartUnicode(regex_text), utils.SmartUnicode(token.reason),\n          require, require_args, require_kwargs, self.helper_name)\n      return True\n\n    logging.warning(\"Datastore access denied to %s (no matched rules)\",\n                    subject_str)\n    raise access_control.UnauthorizedAccess(\n        \"Access to %s rejected: (no matched rules).\" % subject, subject=subject)", "response": "Checks for access to given subject with given token."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncheck if user has access to a client under given URN.", "response": "def _HasAccessToClient(self, subject, token):\n    \"\"\"Checks if user has access to a client under given URN.\"\"\"\n    client_id, _ = rdfvalue.RDFURN(subject).Split(2)\n    client_urn = rdf_client.ClientURN(client_id)\n\n    return self.CheckClientAccess(token, client_urn)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _IsHomeDir(self, subject, token):\n    h = CheckAccessHelper(\"IsHomeDir\")\n    h.Allow(\"aff4:/users/%s\" % token.username)\n    h.Allow(\"aff4:/users/%s/*\" % token.username)\n    try:\n      return h.CheckAccess(subject, token)\n    except access_control.UnauthorizedAccess:\n      raise access_control.UnauthorizedAccess(\n          \"User can only access their \"\n          \"home directory.\", subject=subject)", "response": "Checks if the user can access the subject s home directory."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _CreateReadAccessHelper(self):\n    h = CheckAccessHelper(\"read\")\n\n    h.Allow(\"aff4:/\")\n\n    # In order to open directories below aff4:/users, we have to have access to\n    # aff4:/users directory itself.\n    h.Allow(\"aff4:/users\")\n\n    # User is allowed to access anything in their home dir.\n    h.Allow(\"aff4:/users/*\", self._IsHomeDir)\n\n    # Administrators are allowed to see current set of foreman rules.\n    h.Allow(\"aff4:/foreman\", self._UserHasAdminLabel)\n\n    # Querying is not allowed for the blob namespace. Blobs are stored by hashes\n    # as filename. If the user already knows the hash, they can access the blob,\n    # however they must not be allowed to query for hashes.\n    h.Allow(\"aff4:/blobs\")\n    h.Allow(\"aff4:/blobs/*\")\n\n    # The fingerprint namespace typically points to blobs. As such, it follows\n    # the same rules.\n    h.Allow(\"aff4:/FP\")\n    h.Allow(\"aff4:/FP/*\")\n\n    # The files namespace contains hash references to all files downloaded with\n    # GRR, and is extensible via Filestore objects. Users can access files for\n    # which they know the hash.\n    # See lib/aff4_objects/filestore.py\n    h.Allow(\"aff4:/files\")\n    h.Allow(\"aff4:/files/*\")\n\n    # Namespace for indexes. Client index is stored there.\n    h.Allow(\"aff4:/index\")\n    h.Allow(\"aff4:/index/*\")\n\n    # Keyword-based Client index.\n    h.Allow(\"aff4:/client_index\")\n    h.Allow(\"aff4:/client_index/*\")\n\n    # ACL namespace contains approval objects for accessing clients and hunts.\n    h.Allow(\"aff4:/ACL\")\n    h.Allow(\"aff4:/ACL/*\")\n\n    # stats namespace is for different statistics. For example, ClientFleetStats\n    # object is stored there.\n    h.Allow(\"aff4:/stats\")\n    h.Allow(\"aff4:/stats/*\")\n\n    # Configuration namespace used for reading drivers, python hacks etc.\n    h.Allow(\"aff4:/config\")\n    h.Allow(\"aff4:/config/*\")\n\n    # Namespace for flows that run without a client. A lot of internal utilitiy\n    # flows and cron jobs' flows will end up here.\n    h.Allow(\"aff4:/flows\")\n    h.Allow(\"aff4:/flows/*\")\n\n    # Namespace for hunts.\n    h.Allow(\"aff4:/hunts\")\n    h.Allow(\"aff4:/hunts/*\")\n\n    # Namespace for cron jobs.\n    h.Allow(\"aff4:/cron\")\n    h.Allow(\"aff4:/cron/*\")\n\n    # Namespace for audit data.\n    h.Allow(\"aff4:/audit\")\n    h.Allow(\"aff4:/audit/*\")\n    h.Allow(\"aff4:/audit/logs\")\n    h.Allow(\"aff4:/audit/logs/*\")\n\n    # Namespace for clients.\n    h.Allow(self.CLIENT_URN_PATTERN)\n    h.Allow(self.CLIENT_URN_PATTERN + \"/*\", self._HasAccessToClient)\n\n    # Allow everyone to read the artifact store.\n    h.Allow(\"aff4:/artifact_store\")\n    h.Allow(\"aff4:/artifact_store/*\")\n\n    return h", "response": "Creates a CheckAccessHelper for controlling read access."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncreate a CheckAccessHelper for controlling query access.", "response": "def _CreateQueryAccessHelper(self):\n    \"\"\"Creates a CheckAccessHelper for controlling query access.\n\n    This function and _CreateReadAccessHelper essentially define GRR's ACL\n    policy. Please refer to these 2 functions to either review or modify\n    GRR's ACLs.\n\n    Query access gives you the ability to find objects in the tree without\n    knowing their URN, using ListChildren.  If you grant query access,\n    you will also need read access.\n\n    Returns:\n      CheckAccessHelper for controlling query access.\n    \"\"\"\n    h = CheckAccessHelper(\"query\")\n\n    # User is allowed to do anything in their home dir.\n    h.Allow(\"aff4:/users/*\", self._IsHomeDir)\n\n    # Querying is allowed for aff4:/cron/*, as GUI renders list of current\n    # cron jobs. Also, links to flows executed by every cron job are stored\n    # below cron job object,\n    h.Allow(\"aff4:/cron\")\n    h.Allow(\"aff4:/cron/*\")\n\n    # Hunts and data collected by hunts are publicly available and thus can\n    # be queried.\n    h.Allow(\"aff4:/hunts\")\n    h.Allow(\"aff4:/hunts/*\")\n\n    # ACLs have to be queried to search for proper approvals. As a consequence,\n    # any user can query for all approval objects and get a list of all\n    # possible reasons. At the moment we assume that it's ok.\n    h.Allow(\"aff4:/ACL\")\n    h.Allow(\"aff4:/ACL/*\")\n\n    # Querying contents of the client is allowed, as we have to browse VFS\n    # filesystem.\n    h.Allow(self.CLIENT_URN_PATTERN)\n    h.Allow(self.CLIENT_URN_PATTERN + \"/*\")\n\n    # Namespace for indexes. Client index is stored there and users need to\n    # query the index for searching clients.\n    h.Allow(\"aff4:/index\")\n    h.Allow(\"aff4:/index/*\")\n\n    # Configuration namespace used for reading drivers, python hacks etc. The\n    # GUI needs to be able to list these paths.\n    h.Allow(\"aff4:/config\")\n    h.Allow(\"aff4:/config/*\")\n\n    # We do not allow querying for all flows with empty client id, but we do\n    # allow querying for particular flow's children. This is needed, in\n    # particular, for cron jobs, whose flows end up in aff4:/flows and we\n    # have to query them for children flows.\n    # NOTE: Allow(aff4:/flows/*) does not allow querying of aff4:/flows\n    #       children. But, on the other hand, it will allow querying for\n    #       anything under aff4:/flows/W:SOMEFLOW, for example.\n    h.Allow(\"aff4:/flows/*\")\n\n    # Querying the index of filestore objects is allowed since it reveals the\n    # clients which have this file.\n    h.Allow(\"aff4:/files/hash/generic/sha256/\" + \"[a-z0-9]\" * 64)\n\n    # Allow everyone to query the artifact store.\n    h.Allow(\"aff4:/artifact_store\")\n    h.Allow(\"aff4:/artifact_store/*\")\n\n    # Users are allowed to query the artifact store.\n    h.Allow(\"aff4:/artifact_store\")\n    h.Allow(\"aff4:/artifact_store/*\")\n\n    # Namespace for audit data. Required to display usage statistics.\n    h.Allow(\"aff4:/audit/logs\")\n    h.Allow(\"aff4:/audit/logs/*\")\n\n    return h"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef CheckDataStoreAccess(self, token, subjects, requested_access=\"r\"):\n    if any(not x for x in subjects):\n      raise ValueError(\"Subjects list can't contain empty URNs.\")\n    subjects = list(map(rdfvalue.RDFURN, subjects))\n\n    return (ValidateAccessAndSubjects(requested_access, subjects) and\n            ValidateToken(token, subjects) and\n            (token.supervisor or\n             self._CheckAccessWithHelpers(token, subjects, requested_access)))", "response": "Checks if token and requested access are valid."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nstreaming chunks of a given file.", "response": "def StreamFile(self, filedesc, offset=0, amount=None):\n    \"\"\"Streams chunks of a given file starting at given offset.\n\n    Args:\n      filedesc: A `file` object to stream.\n      offset: An integer offset at which the file stream should start on.\n      amount: An upper bound on number of bytes to read.\n\n    Returns:\n      Generator over `Chunk` instances.\n    \"\"\"\n    reader = FileReader(filedesc, offset=offset)\n    return self.Stream(reader, amount=amount)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef StreamFilePath(self, filepath, offset=0, amount=None):\n    with open(filepath, \"rb\") as filedesc:\n      for chunk in self.StreamFile(filedesc, offset=offset, amount=amount):\n        yield chunk", "response": "Streams chunks of a file located at given path starting at given offset."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nstreaming chunks of memory of a given process starting at given offset.", "response": "def StreamMemory(self, process, offset=0, amount=None):\n    \"\"\"Streams chunks of memory of a given process starting at given offset.\n\n    Args:\n      process: A platform-specific `Process` instance.\n      offset: An integer offset at which the memory stream should start on.\n      amount: An upper bound on number of bytes to read.\n\n    Returns:\n      Generator over `Chunk` instances.\n    \"\"\"\n    reader = MemoryReader(process, offset=offset)\n    return self.Stream(reader, amount=amount)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef Stream(self, reader, amount=None):\n    if amount is None:\n      amount = float(\"inf\")\n\n    data = reader.Read(min(self.chunk_size, amount))\n    if not data:\n      return\n\n    amount -= len(data)\n    offset = reader.offset - len(data)\n    yield Chunk(offset=offset, data=data)\n\n    while amount > 0:\n      # We need `len(data)` here because overlap size can be 0.\n      overlap = data[len(data) - self.overlap_size:]\n\n      new = reader.Read(min(self.chunk_size - self.overlap_size, amount))\n      if not new:\n        return\n\n      data = overlap + new\n\n      amount -= len(new)\n      offset = reader.offset - len(data)\n      yield Chunk(offset=offset, data=data, overlap=len(overlap))", "response": "Streams chunks of a given file."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef Scan(self, matcher):\n\n    position = 0\n    while True:\n      span = matcher.Match(self.data, position)\n      if span is None:\n        return\n\n      # We are not interested in hits within overlap-only zone. We continue the\n      # search just after the previous match starts because of situations where\n      # there is a match beginning before the end of the overlap-only zone match\n      # and ending after the overlap zone.\n      if span.end <= self.overlap:\n        position = span.begin + 1\n        continue\n\n      # Since we do not care about overlapping matches we resume our search\n      # at the end of the previous match.\n      position = span.end\n      yield span", "response": "Yields spans occurrences of a given pattern within a chunk."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nlist all the filesystems mounted on the system.", "response": "def GetMountpoints(data=None):\n  \"\"\"List all the filesystems mounted on the system.\"\"\"\n  expiry = 60  # 1 min\n\n  insert_time = MOUNTPOINT_CACHE[0]\n  if insert_time + expiry > time.time():\n    return MOUNTPOINT_CACHE[1]\n\n  devices = {}\n\n  # Check all the mounted filesystems.\n  if data is None:\n    data = \"\\n\".join(\n        [open(x, \"rb\").read() for x in [\"/proc/mounts\", \"/etc/mtab\"]])\n\n  for line in data.splitlines():\n    try:\n      device, mnt_point, fs_type, _ = line.split(\" \", 3)\n      mnt_point = os.path.normpath(mnt_point)\n\n      # What if several devices are mounted on the same mount point?\n      devices[mnt_point] = (device, fs_type)\n    except ValueError:\n      pass\n\n  MOUNTPOINT_CACHE[0] = time.time()\n  MOUNTPOINT_CACHE[1] = devices\n\n  return devices"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nexecutes a command on the GRR client.", "response": "def Execute(cmd,\n            args,\n            time_limit=-1,\n            bypass_whitelist=False,\n            daemon=False,\n            use_client_context=False,\n            cwd=None):\n  \"\"\"Executes commands on the client.\n\n  This function is the only place where commands will be executed\n  by the GRR client. This makes sure that all issued commands are compared to a\n  white list and no malicious commands are issued on the client machine.\n\n  Args:\n    cmd: The command to be executed.\n    args: List of arguments.\n    time_limit: Time in seconds the process is allowed to run.\n    bypass_whitelist: Allow execution of things that are not in the whitelist.\n      Note that this should only ever be called on a binary that passes the\n      VerifySignedBlob check.\n    daemon: Start the new process in the background.\n    use_client_context: Run this script in the client's context. Defaults to\n      system context.\n    cwd: Current working directory for the command.\n\n  Returns:\n    A tuple of stdout, stderr, return value and time taken.\n  \"\"\"\n  if not bypass_whitelist and not IsExecutionWhitelisted(cmd, args):\n    # Whitelist doesn't contain this cmd/arg pair\n    logging.info(\"Execution disallowed by whitelist: %s %s.\", cmd,\n                 \" \".join(args))\n    return (b\"\", b\"Execution disallowed by whitelist.\", -1, -1)\n\n  if daemon:\n    pid = os.fork()\n    if pid == 0:\n      # This is the child, it will run the daemon process. We call os.setsid\n      # here to become the session leader of this new session and the process\n      # group leader of the new process group so we don't get killed when the\n      # main process exits.\n      try:\n        os.setsid()\n      except OSError:\n        # This only works if the process is running as root.\n        pass\n      _Execute(\n          cmd, args, time_limit, use_client_context=use_client_context, cwd=cwd)\n      os._exit(0)  # pylint: disable=protected-access\n  else:\n    return _Execute(\n        cmd, args, time_limit, use_client_context=use_client_context, cwd=cwd)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _Execute(cmd, args, time_limit=-1, use_client_context=False, cwd=None):\n  run = [cmd]\n  run.extend(args)\n  env = os.environ.copy()\n  if use_client_context:\n    env.pop(\"LD_LIBRARY_PATH\", None)\n    env.pop(\"PYTHON_PATH\", None)\n    context = \"client\"\n  else:\n    context = \"system\"\n  logging.info(\"Executing %s in %s context.\", \" \".join(run), context)\n  p = subprocess.Popen(\n      run,\n      stdin=subprocess.PIPE,\n      stdout=subprocess.PIPE,\n      stderr=subprocess.PIPE,\n      env=env,\n      cwd=cwd)\n\n  alarm = None\n  if time_limit > 0:\n    alarm = threading.Timer(time_limit, HandleAlarm, (p,))\n    alarm.setDaemon(True)\n    alarm.start()\n\n  stdout, stderr, exit_status = b\"\", b\"\", -1\n  start_time = time.time()\n  try:\n    stdout, stderr = p.communicate()\n    exit_status = p.returncode\n  except IOError:\n    # If we end up here, the time limit was exceeded\n    pass\n  finally:\n    if alarm:\n      alarm.cancel()\n      alarm.join()\n\n  return (stdout, stderr, exit_status, time.time() - start_time)", "response": "Executes cmd with args."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncheck if a binary and args are whitelisted.", "response": "def IsExecutionWhitelisted(cmd, args):\n  \"\"\"Check if a binary and args is whitelisted.\n\n  Args:\n    cmd: Canonical path to the binary.\n    args: List of arguments to be passed to the binary.\n\n  Returns:\n    Bool, True if it is whitelisted.\n\n  These whitelists could also go in the platform specific client files\n  client_utils_<platform>.py. We chose to leave them here instead of putting\n  them in global arrays to discourage people coding other modules from adding\n  new commands to the whitelist before running them.\n  The idea is to have a single place that lists every command we can run during\n  normal operation (obviously doesn't catch the special cases where we bypass\n  the list).\n  A deployment-specific list is also checked (see local/binary_whitelist.py).\n  \"\"\"\n  if platform.system() == \"Windows\":\n    whitelist = [\n        (\"arp.exe\", [\"-a\"]),\n        (\"driverquery.exe\", [\"/v\"]),\n        (\"ipconfig.exe\", [\"/all\"]),\n        (\"netsh.exe\", [\"advfirewall\", \"firewall\", \"show\", \"rule\", \"name=all\"]),\n        (\"netsh.exe\",\n         [\"advfirewall\", \"monitor\", \"show\", \"firewall\", \"rule\", \"name=all\"]),\n        (\"tasklist.exe\", [\"/SVC\"]),\n        (\"tasklist.exe\", [\"/v\"]),\n    ]\n  elif platform.system() == \"Linux\":\n    whitelist = [\n        (\"/bin/df\", []),\n        (\"/bin/echo\", [\"1\"]),\n        (\"/bin/rpm\", [\"-qa\"]),\n        (\"/bin/sleep\", [\"10\"]),\n        (\"/sbin/auditctl\", [\"-l\"]),\n        (\"/sbin/ifconfig\", [\"-a\"]),\n        (\"/sbin/iptables\", [\"-L\", \"-n\", \"-v\"]),\n        (\"/sbin/lsmod\", []),\n        (\"/usr/bin/dpkg\", [\"--list\"]),\n        (\"/usr/bin/last\", []),\n        (\"/usr/bin/yum\", [\"list\", \"installed\", \"-q\"]),\n        (\"/usr/bin/yum\", [\"repolist\", \"-v\", \"-q\"]),\n        (\"/usr/bin/who\", []),\n        (\"/usr/sbin/arp\", [\"-a\"]),\n        (\"/usr/sbin/dmidecode\", [\"-q\"]),\n        (\"/usr/sbin/sshd\", [\"-T\"]),\n    ]\n  elif platform.system() == \"Darwin\":\n    whitelist = [\n        (\"/bin/echo\", [\"1\"]),\n        (\"/bin/launchctl\", [\"unload\", config.CONFIG[\"Client.plist_path\"]]),\n        (\"/usr/bin/hdiutil\", [\"info\"]),\n        (\"/usr/bin/last\", []),\n        (\"/usr/bin/who\", []),\n        (\"/usr/sbin/arp\", [\"-a\"]),\n        (\"/usr/sbin/kextstat\", []),\n        (\"/usr/sbin/system_profiler\", [\"-xml\", \"SPHardwareDataType\"]),\n        (\"/usr/libexec/firmwarecheckers/ethcheck/ethcheck\", [\"--show-hashes\"]),\n    ]\n  else:\n    whitelist = []\n\n  for (allowed_cmd, allowed_args) in whitelist:\n    if cmd == allowed_cmd and args == allowed_args:\n      return True\n\n  # Check if this is whitelisted locally.\n  if binary_whitelist.IsExecutionWhitelisted(cmd, args):\n    return True\n\n  return False"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef HashFilePath(self, path, byte_count):\n    with open(path, \"rb\") as fd:\n      self.HashFile(fd, byte_count)", "response": "Updates underlying hashers with file on a given path."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef HashFile(self, fd, byte_count):\n    while byte_count > 0:\n      buf_size = min(byte_count, constants.CLIENT_MAX_BUFFER_SIZE)\n      buf = fd.read(buf_size)\n      if not buf:\n        break\n\n      self.HashBuffer(buf)\n      byte_count -= buf_size", "response": "Updates underlying hashers with a given file object."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef HashBuffer(self, buf):\n    for hasher in itervalues(self._hashers):\n      hasher.update(buf)\n      if self._progress:\n        self._progress()\n\n    self._bytes_read += len(buf)", "response": "Updates the underlying hashers with a given byte buffer."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns a Hash object with appropriate fields filled - in.", "response": "def GetHashObject(self):\n    \"\"\"Returns a `Hash` object with appropriate fields filled-in.\"\"\"\n    hash_object = rdf_crypto.Hash()\n    hash_object.num_bytes = self._bytes_read\n    for algorithm in self._hashers:\n      setattr(hash_object, algorithm, self._hashers[algorithm].digest())\n    return hash_object"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef GetAllClientLabels(token, include_catchall=False):\n  labels_index = aff4.FACTORY.Create(\n      standard.LabelSet.CLIENT_LABELS_URN,\n      standard.LabelSet,\n      mode=\"r\",\n      token=token)\n  labels = set(labels_index.ListLabels())\n  if include_catchall:\n    labels.add(ALL_CLIENTS_LABEL)\n  return labels", "response": "Returns the set of all label names applied to all clients."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef age(self):\n    # TODO(user) move up to AFF4Object after some analysis of how .age is\n    # used in the codebase.\n    aff4_type = self.Get(self.Schema.TYPE)\n\n    if aff4_type:\n      return aff4_type.age\n    else:\n      # If there is no type attribute yet, we have only just been created and\n      # not flushed yet, so just set timestamp to now.\n      return rdfvalue.RDFDatetime.Now()", "response": "Returns the age of the object."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef GetSummary(self):\n    self.max_age = 0\n    summary = rdf_client.ClientSummary(client_id=self.urn)\n    summary.system_info.system = self.Get(self.Schema.SYSTEM)\n    summary.system_info.release = self.Get(self.Schema.OS_RELEASE)\n    summary.system_info.version = str(self.Get(self.Schema.OS_VERSION, \"\"))\n    summary.system_info.kernel = self.Get(self.Schema.KERNEL)\n    summary.system_info.fqdn = self.Get(self.Schema.FQDN)\n    summary.system_info.machine = self.Get(self.Schema.ARCH)\n    summary.system_info.install_date = self.Get(self.Schema.INSTALL_DATE)\n    kb = self.Get(self.Schema.KNOWLEDGE_BASE)\n    if kb:\n      summary.users = kb.users\n    summary.interfaces = self.Get(self.Schema.INTERFACES)\n    summary.client_info = self.Get(self.Schema.CLIENT_INFO)\n    hwi = self.Get(self.Schema.HARDWARE_INFO)\n    if hwi:\n      summary.serial_number = hwi.serial_number\n      summary.system_manufacturer = hwi.system_manufacturer\n      summary.system_uuid = hwi.system_uuid\n    summary.timestamp = self.age\n    cloud_instance = self.Get(self.Schema.CLOUD_INSTANCE)\n    if cloud_instance:\n      summary.cloud_type = cloud_instance.cloud_type\n      if cloud_instance.cloud_type == \"GOOGLE\":\n        summary.cloud_instance_id = cloud_instance.google.unique_id\n      elif cloud_instance.cloud_type == \"AMAZON\":\n        summary.cloud_instance_id = cloud_instance.amazon.instance_id\n      else:\n        raise ValueError(\"Bad cloud type: %s\" % cloud_instance.cloud_type)\n\n    return summary", "response": "Retrieves a client summary object."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef Start(self):\n    self.Init()\n    fd = aff4.FACTORY.Open(self.args.vfs_file_urn, mode=\"rw\", token=self.token)\n\n    # Account for implicit directories.\n    if fd.Get(fd.Schema.TYPE) is None:\n      fd = fd.Upgrade(standard.VFSDirectory)\n\n    self.state.get_file_flow_urn = fd.Update(attribute=self.args.attribute)", "response": "Calls the Update method of a given VFSFile or VFSDirectory object."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nupdates an attribute from the client.", "response": "def Update(self, attribute=None):\n    \"\"\"Update an attribute from the client.\"\"\"\n    # List the directory on the client\n    currently_running = self.Get(self.Schema.CONTENT_LOCK)\n\n    # Is this flow still active?\n    if currently_running:\n      flow_obj = aff4.FACTORY.Open(currently_running, token=self.token)\n      if flow_obj and flow_obj.GetRunner().IsRunning():\n        return\n\n    # The client_id is the first element of the URN\n    client_id = self.urn.Path().split(\"/\", 2)[1]\n\n    # Get the pathspec for this object\n    pathspec = self.Get(self.Schema.STAT).pathspec\n    flow_urn = flow.StartAFF4Flow(\n        client_id=client_id,\n        # TODO(user): dependency loop between aff4_grr.py and transfer.py\n        # flow_name=transfer.MultiGetFile.__name__,\n        flow_name=\"MultiGetFile\",\n        token=self.token,\n        pathspecs=[pathspec])\n    self.Set(self.Schema.CONTENT_LOCK(flow_urn))\n    self.Close()\n\n    return flow_urn"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef ExpireRules(self):\n    rules = self.Get(self.Schema.RULES)\n    new_rules = self.Schema.RULES()\n    now = time.time() * 1e6\n    expired_session_ids = set()\n\n    for rule in rules:\n      if rule.expires > now:\n        new_rules.Append(rule)\n      else:\n        for action in rule.actions:\n          if action.hunt_id:\n            expired_session_ids.add(action.hunt_id)\n\n    if expired_session_ids:\n      with data_store.DB.GetMutationPool() as pool:\n        # Notify the worker to mark this hunt as terminated.\n        manager = queue_manager.QueueManager(token=self.token)\n        manager.MultiNotifyQueue([\n            rdf_flows.GrrNotification(session_id=session_id)\n            for session_id in expired_session_ids\n        ],\n                                 mutation_pool=pool)\n\n    if len(new_rules) < len(rules):\n      self.Set(self.Schema.RULES, new_rules)\n      self.Flush()", "response": "Removes any rules with an expiration date in the past."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn True if hunt s task was assigned to this client before.", "response": "def _CheckIfHuntTaskWasAssigned(self, client_id, hunt_id):\n    \"\"\"Will return True if hunt's task was assigned to this client before.\"\"\"\n    client_urn = rdfvalue.RDFURN(client_id)\n    for _ in aff4.FACTORY.Stat([\n        client_urn.Add(\"flows/%s:hunt\" % rdfvalue.RDFURN(hunt_id).Basename())\n    ]):\n      return True\n\n    return False"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _RunActions(self, rule, client_id):\n    actions_count = 0\n\n    for action in rule.actions:\n      try:\n        # Say this flow came from the foreman.\n        token = self.token.Copy()\n        token.username = \"Foreman\"\n\n        if action.HasField(\"hunt_id\"):\n          if self._CheckIfHuntTaskWasAssigned(client_id, action.hunt_id):\n            logging.info(\n                \"Foreman: ignoring hunt %s on client %s: was started \"\n                \"here before\", client_id, action.hunt_id)\n          else:\n            logging.info(\"Foreman: Starting hunt %s on client %s.\",\n                         action.hunt_id, client_id)\n\n            flow_cls = registry.AFF4FlowRegistry.FlowClassByName(\n                action.hunt_name)\n            flow_cls.StartClients(action.hunt_id, [client_id])\n            actions_count += 1\n        else:\n          flow.StartAFF4Flow(\n              client_id=client_id,\n              flow_name=action.flow_name,\n              token=token,\n              **action.argv.ToDict())\n          actions_count += 1\n      # There could be all kinds of errors we don't know about when starting the\n      # flow/hunt so we catch everything here.\n      except Exception as e:  # pylint: disable=broad-except\n        logging.exception(\"Failure running foreman action on client %s: %s\",\n                          action.hunt_id, e)\n\n    return actions_count", "response": "Runs all the actions specified in the rule."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nassign tasks to a client.", "response": "def AssignTasksToClient(self, client_id):\n    \"\"\"Examines our rules and starts up flows based on the client.\n\n    Args:\n      client_id: Client id of the client for tasks to be assigned.\n\n    Returns:\n      Number of assigned tasks.\n    \"\"\"\n    rules = self.Get(self.Schema.RULES)\n    if not rules:\n      return 0\n\n    if data_store.RelationalDBEnabled():\n      last_foreman_run = self._GetLastForemanRunTimeRelational(client_id)\n    else:\n      last_foreman_run = self._GetLastForemanRunTime(client_id)\n\n    latest_rule = max(rule.created for rule in rules)\n\n    if latest_rule <= last_foreman_run:\n      return 0\n\n    # Update the latest checked rule on the client.\n    if data_store.RelationalDBEnabled():\n      try:\n        self._SetLastForemanRunTimeRelational(client_id, latest_rule)\n      except db.UnknownClientError:\n        pass\n\n    # If the relational db is used for reads, we don't have to update the\n    # aff4 object.\n    if not data_store.RelationalDBEnabled():\n      self._SetLastForemanRunTime(client_id, latest_rule)\n\n    relevant_rules = []\n    expired_rules = False\n\n    now = time.time() * 1e6\n\n    for rule in rules:\n      if rule.expires < now:\n        expired_rules = True\n        continue\n      if rule.created <= int(last_foreman_run):\n        continue\n\n      relevant_rules.append(rule)\n\n    if data_store.RelationalDBEnabled():\n      client_data = data_store.REL_DB.ReadClientFullInfo(client_id)\n      if client_data is None:\n        return\n    else:\n      client_data = aff4.FACTORY.Open(client_id, mode=\"rw\", token=self.token)\n\n    actions_count = 0\n    for rule in relevant_rules:\n      if self._EvaluateRules(rule, client_data):\n        actions_count += self._RunActions(rule, client_id)\n\n    if expired_rules:\n      self.ExpireRules()\n\n    return actions_count"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nopens the delegate object.", "response": "def Initialize(self):\n    \"\"\"Open the delegate object.\"\"\"\n    if \"r\" in self.mode:\n      delegate = self.Get(self.Schema.DELEGATE)\n      if delegate:\n        self.delegate = aff4.FACTORY.Open(\n            delegate, mode=self.mode, token=self.token, age=self.age_policy)"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nretrieves the relevant blob from the AFF4 data store or cache.", "response": "def _GetChunkForReading(self, chunk):\n    \"\"\"Retrieve the relevant blob from the AFF4 data store or cache.\"\"\"\n    offset = chunk * self._HASH_SIZE\n    self.index.seek(offset)\n\n    chunk_name = self.index.read(self._HASH_SIZE)\n\n    try:\n      return self.chunk_cache.Get(chunk_name)\n    except KeyError:\n      pass\n\n    # We don't have this chunk already cached. The most common read\n    # access pattern is contiguous reading so since we have to go to\n    # the data store already, we read ahead to reduce round trips.\n    self.index.seek(offset)\n    readahead = []\n\n    for _ in range(self._READAHEAD):\n      name = self.index.read(self._HASH_SIZE)\n      if name and name not in self.chunk_cache:\n        readahead.append(rdf_objects.BlobID.FromBytes(name))\n\n    self._ReadChunks(readahead)\n    try:\n      return self.chunk_cache.Get(chunk_name)\n    except KeyError:\n      raise aff4.ChunkNotFoundError(\"Cannot open chunk %s\" % chunk)"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nappend content to the blob image.", "response": "def AppendContent(self, src_fd):\n    \"\"\"Create new blob hashes and append to BlobImage.\n\n    We don't support writing at arbitrary file offsets, but this method provides\n    a convenient way to add blobs for a new file, or append content to an\n    existing one.\n\n    Args:\n      src_fd: source file handle open for read\n\n    Raises:\n      IOError: if blob has already been finalized.\n    \"\"\"\n    while 1:\n      blob = src_fd.read(self.chunksize)\n      if not blob:\n        break\n\n      blob_id = data_store.BLOBS.WriteBlobWithUnknownHash(blob)\n      self.AddBlob(blob_id, len(blob))\n\n    self.Flush()"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nadding a blob to this blob image using its hash.", "response": "def AddBlob(self, blob_id, length):\n    \"\"\"Add another blob to this image using its hash.\n\n    Once a blob is added that is smaller than the chunksize we finalize the\n    file, since handling adding more blobs makes the code much more complex.\n\n    Args:\n      blob_id: rdf_objects.BlobID object.\n      length: int length of blob\n\n    Raises:\n      IOError: if blob has been finalized.\n    \"\"\"\n    if self.finalized and length > 0:\n      raise IOError(\"Can't add blobs to finalized BlobImage\")\n\n    self.content_dirty = True\n    self.index.seek(0, 2)\n    self.index.write(blob_id.AsBytes())\n    self.size += length\n\n    if length < self.chunksize:\n      self.finalized = True"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef NewFromContent(cls,\n                     content,\n                     urn,\n                     chunk_size=1024,\n                     token=None,\n                     private_key=None,\n                     public_key=None):\n    \"\"\"Alternate constructor for GRRSignedBlob.\n\n    Creates a GRRSignedBlob from a content string by chunking it and signing\n    each chunk.\n\n    Args:\n      content: The data to stored in the GRRSignedBlob.\n      urn: The AFF4 URN to create.\n\n      chunk_size: Data will be chunked into this size (each chunk is\n        individually signed.\n      token: The ACL Token.\n      private_key: An rdf_crypto.RSAPrivateKey() instance.\n      public_key: An rdf_crypto.RSAPublicKey() instance.\n\n    Returns:\n      the URN of the new object written.\n    \"\"\"\n    aff4.FACTORY.Delete(urn, token=token)\n    with data_store.DB.GetMutationPool() as pool:\n      with aff4.FACTORY.Create(\n          urn, cls, mode=\"w\", mutation_pool=pool, token=token) as fd:\n        for start_of_chunk in range(0, len(content), chunk_size):\n          chunk = content[start_of_chunk:start_of_chunk + chunk_size]\n          blob_rdf = rdf_crypto.SignedBlob()\n          blob_rdf.Sign(chunk, private_key, public_key)\n          fd.Add(blob_rdf, mutation_pool=pool)\n\n    return urn", "response": "Alternate constructor for GRRSignedBlob."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef Substitute(self, pattern):\n    if isinstance(pattern, bytes):\n      substs = [re.escape(subst.encode(\"ascii\")) for subst in self._substs]\n      regex = re.compile(b\"|\".join(substs))\n\n      def Replacement(match):\n        key = match.group(0).decode(\"ascii\")\n        return self._substs[key].encode(\"utf-8\")\n\n    elif isinstance(pattern, Text):\n      substs = [re.escape(subst) for subst in self._substs]\n      regex = re.compile(\"|\".join(substs))\n\n      def Replacement(match):\n        key = match.group(0)\n        return self._substs[key]\n\n    else:\n      raise TypeError(\"Unexpected pattern type '{}'\".format(type(pattern)))\n\n    if not substs:\n      return pattern\n    else:\n      return regex.sub(Replacement, pattern)", "response": "Formats given pattern with this substitution environment."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nassociates a value with a given variable.", "response": "def BindVar(self, var_id, value):\n    \"\"\"Associates a value with given variable.\n\n    This can be called multiple times to associate multiple values.\n\n    Args:\n      var_id: A variable id to bind the values to.\n      value: A value to bind to the specified variable.\n\n    Raises:\n      KeyError: If given variable is not specified in the pattern.\n    \"\"\"\n    if var_id not in self._vars:\n      raise KeyError(var_id)\n\n    self._var_bindings[var_id].append(value)"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nassociate given values with given scope.", "response": "def BindScope(self, scope_id, values):\n    \"\"\"Associates given values with given scope.\n\n    This can be called multiple times to associate multiple values.\n\n    Args:\n      scope_id: A scope id to bind the values to.\n      values: A mapping from scope variable ids to values to bind in scope.\n\n    Raises:\n      KeyError: If given scope or scope variable is not specified in the\n        pattern.\n    \"\"\"\n    if scope_id not in self._scopes:\n      raise KeyError(scope_id)\n\n    keys = set(iterkeys(values))\n    if keys != self._scopes[scope_id]:\n      raise KeyError(keys ^ self._scopes[scope_id])\n\n    self._scope_bindings[scope_id].append(values)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef Interpolate(self):\n    for var_config in collection.DictProduct(self._var_bindings):\n      for scope_config in collection.DictProduct(self._scope_bindings):\n        subst = Substitution(var_config=var_config, scope_config=scope_config)\n        yield subst.Substitute(self._pattern)", "response": "Interpolates the pattern.\n\n    Yields:\n      All possible interpolation results."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nregister a report plugin for use in the GRR UI.", "response": "def RegisterPlugin(self, report_plugin_cls):\n    \"\"\"Registers a report plugin for use in the GRR UI.\"\"\"\n\n    name = report_plugin_cls.__name__\n    if name in self.plugins:\n      raise RuntimeError(\"Can't register two report plugins with the same \"\n                         \"name. In particular, can't register the same \"\n                         \"report plugin twice: %r\" % name)\n\n    self.plugins[name] = report_plugin_cls"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef MakeExecutableTemplate(self, output_file=None):\n    super(DarwinClientBuilder,\n          self).MakeExecutableTemplate(output_file=output_file)\n    self.SetBuildVars()\n    self.MakeBuildDirectory()\n    self.BuildWithPyInstaller()\n    self.CopyMissingModules()\n    self.BuildInstallerPkg(output_file)\n    self.MakeZip(output_file, self.template_file)", "response": "Create the executable template."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nadding a zip to the end of the .xar containing build.yaml. The build.yaml is already inside the .xar file, but we can't easily open this on linux. To make repacking easier we add a zip to the end of the .xar and add in the build.yaml. The repack step will then look at the build.yaml and insert the config.yaml. We end up storing the build.yaml twice but it is tiny, so this doesn't matter. Args: xar_file: the name of the xar file. output_file: the name of the output ZIP archive.", "response": "def MakeZip(self, xar_file, output_file):\n    \"\"\"Add a zip to the end of the .xar containing build.yaml.\n\n    The build.yaml is already inside the .xar file, but we can't easily open\n    this on linux. To make repacking easier we add a zip to the end of the .xar\n    and add in the build.yaml. The repack step will then look at the build.yaml\n    and insert the config.yaml. We end up storing the build.yaml twice but it is\n    tiny, so this doesn't matter.\n\n    Args:\n      xar_file: the name of the xar file.\n      output_file: the name of the output ZIP archive.\n    \"\"\"\n    logging.info(\"Generating zip template file at %s\", output_file)\n    with zipfile.ZipFile(output_file, mode=\"a\") as zf:\n      # Get the build yaml\n      # TODO(hanuszczak): YAML, consider using `StringIO` instead.\n      build_yaml = io.BytesIO()\n      self.WriteBuildYaml(build_yaml)\n      build_yaml.seek(0)\n      zf.writestr(\"build.yaml\", build_yaml.read())"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nbuild a package using PackageMaker.", "response": "def BuildInstallerPkg(self, output_file):\n    \"\"\"Builds a package (.pkg) using PackageMaker.\"\"\"\n    self.CreateInstallDirs()\n    self.InterpolateFiles()\n    self.RenameGRRPyinstallerBinaries()\n    self.SignGRRPyinstallerBinaries()\n    self.WriteClientConfig()\n    self.Set755Permissions()\n    self.RunPkgBuild()\n    self.RunProductBuild()\n    self.RenamePkgToTemplate(output_file)"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn a path - like String of client_path with optional prefix.", "response": "def _ClientPathToString(client_path, prefix=\"\"):\n  \"\"\"Returns a path-like String of client_path with optional prefix.\"\"\"\n  return os.path.join(prefix, client_path.client_id, client_path.vfs_path)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ngenerate a MANIFEST file in the archive.", "response": "def _GenerateDescription(self):\n    \"\"\"Generates description into a MANIFEST file in the archive.\"\"\"\n\n    manifest = {\n        \"description\": self.description,\n        \"processed_files\": len(self.processed_files),\n        \"archived_files\": len(self.archived_files),\n        \"ignored_files\": len(self.ignored_files),\n        \"failed_files\": len(self.failed_files)\n    }\n    if self.ignored_files:\n      manifest[\"ignored_files_list\"] = [\n          _ClientPathToString(cp, prefix=\"aff4:\") for cp in self.ignored_files\n      ]\n    if self.failed_files:\n      manifest[\"failed_files_list\"] = [\n          _ClientPathToString(cp, prefix=\"aff4:\") for cp in self.failed_files\n      ]\n\n    manifest_fd = io.BytesIO()\n    if self.total_files != len(self.archived_files):\n      manifest_fd.write(self.FILES_SKIPPED_WARNING)\n    manifest_fd.write(yaml.Dump(manifest).encode(\"utf-8\"))\n\n    manifest_fd.seek(0)\n    st = os.stat_result(\n        (0o644, 0, 0, 0, 0, 0, len(manifest_fd.getvalue()), 0, 0, 0))\n\n    for chunk in self.archive_generator.WriteFromFD(\n        manifest_fd, os.path.join(self.prefix, \"MANIFEST\"), st=st):\n      yield chunk"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _GenerateClientInfo(self, client_id, client_fd):\n    summary_dict = client_fd.ToPrimitiveDict(stringify_leaf_fields=True)\n    summary = yaml.Dump(summary_dict).encode(\"utf-8\")\n\n    client_info_path = os.path.join(self.prefix, client_id, \"client_info.yaml\")\n    st = os.stat_result((0o644, 0, 0, 0, 0, 0, len(summary), 0, 0, 0))\n    yield self.archive_generator.WriteFileHeader(client_info_path, st=st)\n    yield self.archive_generator.WriteFileChunk(summary)\n    yield self.archive_generator.WriteFileFooter()", "response": "Generates the client info file."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ngenerate an AFF4 archive from a given collection.", "response": "def Generate(self, items, token=None):\n    \"\"\"Generates archive from a given collection.\n\n    Iterates the collection and generates an archive by yielding contents\n    of every referenced AFF4Stream.\n\n    Args:\n      items: Iterable of rdf_client_fs.StatEntry objects\n      token: User's ACLToken.\n\n    Yields:\n      Binary chunks comprising the generated archive.\n    \"\"\"\n\n    del token  # unused, to be removed with AFF4 code\n\n    client_ids = set()\n    for item_batch in collection.Batch(items, self.BATCH_SIZE):\n\n      client_paths = set()\n      for item in item_batch:\n        try:\n          client_path = flow_export.CollectionItemToClientPath(\n              item, self.client_id)\n        except flow_export.ItemNotExportableError:\n          continue\n\n        if not self.predicate(client_path):\n          self.ignored_files.add(client_path)\n          self.processed_files.add(client_path)\n          continue\n\n        client_ids.add(client_path.client_id)\n        client_paths.add(client_path)\n\n      for chunk in file_store.StreamFilesChunks(client_paths):\n        self.processed_files.add(chunk.client_path)\n        for output in self._WriteFileChunk(chunk=chunk):\n          yield output\n\n      self.processed_files |= client_paths - (\n          self.ignored_files | self.archived_files)\n\n    if client_ids:\n      for client_id, client_info in iteritems(\n          data_store.REL_DB.MultiReadClientFullInfo(client_ids)):\n        client = api_client.ApiClient().InitFromClientInfo(client_info)\n        for chunk in self._GenerateClientInfo(client_id, client):\n          yield chunk\n\n    for chunk in self._GenerateDescription():\n      yield chunk\n\n    yield self.archive_generator.Close()"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nyielding binary chunks respecting archive file headers and footers.", "response": "def _WriteFileChunk(self, chunk):\n    \"\"\"Yields binary chunks, respecting archive file headers and footers.\n\n    Args:\n      chunk: the StreamedFileChunk to be written\n    \"\"\"\n    if chunk.chunk_index == 0:\n      # Make sure size of the original file is passed. It's required\n      # when output_writer is StreamingTarWriter.\n      st = os.stat_result((0o644, 0, 0, 0, 0, 0, chunk.total_size, 0, 0, 0))\n      target_path = _ClientPathToString(chunk.client_path, prefix=self.prefix)\n      yield self.archive_generator.WriteFileHeader(target_path, st=st)\n\n    yield self.archive_generator.WriteFileChunk(chunk.data)\n\n    if chunk.chunk_index == chunk.total_chunks - 1:\n      yield self.archive_generator.WriteFileFooter()\n      self.archived_files.add(chunk.client_path)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef DefineFromProtobuf(cls, protobuf):\n  mro_chain = [c.__name__ for c in inspect.getmro(cls)]\n  if (protobuf.__name__ not in mro_chain and\n      not getattr(cls, \"allow_custom_class_name\", False)):\n    raise ProtobufNameMustMatchClassOrParentClassError(\n        \"Can't define RDFProtoStruct class %s from proto %s \"\n        \"(proto name must match one of the classes in the MRO chain: %s)\" %\n        (cls.__name__, protobuf.__name__, \", \".join(mro_chain)))\n\n  cls.recorded_rdf_deps = set()\n\n  # Parse message level options.\n  message_options = protobuf.DESCRIPTOR.GetOptions()\n  semantic_options = message_options.Extensions[semantic_pb2.semantic]\n\n  # Hack to avoid dependency loop.\n  # TODO(amoser): remove this hack\n  classes_dict = type_info.TypeInfoObject.classes\n\n  # Support message descriptions\n  if semantic_options.description and not cls.__doc__:\n    cls.__doc__ = semantic_options.description\n\n  cls.union_field = semantic_options.union_field or None\n\n  # We search through all the field descriptors and build type info\n  # descriptors from them.\n  for field in protobuf.DESCRIPTOR.fields:\n    type_descriptor = None\n\n    # Does this field have semantic options?\n    options = field.GetOptions().Extensions[semantic_pb2.sem_type]\n    kwargs = dict(\n        description=options.description,\n        name=field.name,\n        friendly_name=options.friendly_name,\n        field_number=field.number,\n        labels=list(options.label))\n\n    if field.has_default_value:\n      kwargs[\"default\"] = field.default_value\n\n    # This field is a non-protobuf semantic value.\n    if options.type and field.type != TYPE_MESSAGE:\n      cls.recorded_rdf_deps.add(options.type)\n      rdf_type = rdfvalue.RDFValue.classes.get(options.type)\n      if rdf_type:\n        if (CHECK_PROTOBUF_DEPENDENCIES and rdf_type not in cls.rdf_deps and\n            options.type not in cls.rdf_deps):\n          raise rdfvalue.InitializeError(\n              \"%s.%s: field %s is of type %s, \"\n              \"but type is missing from its dependencies list\" %\n              (cls.__module__, cls.__name__, field.name, options.type))\n\n        # Make sure that the field type is the same as what is required by the\n        # semantic type.\n        required_field_type = _SEMANTIC_PRIMITIVE_TO_FIELD_TYPE[\n            rdf_type.data_store_type]\n\n        if required_field_type != field.type:\n          raise rdfvalue.InitializeError(\n              (\"%s: .proto file uses incorrect field to store Semantic Value \"\n               \"%s: Should be %s\") %\n              (cls.__name__, field.name, rdf_type.data_store_type))\n\n      type_descriptor = classes_dict[\"ProtoRDFValue\"](\n          rdf_type=options.type, **kwargs)\n\n    # A semantic protobuf is already a semantic value so it is an error to\n    # specify it in two places.\n    elif options.type and field.type == TYPE_MESSAGE:\n      raise rdfvalue.InitializeError(\n          (\"%s: .proto file specified both Semantic Value type %s and \"\n           \"Semantic protobuf %s\") %\n          (cls.__name__, options.type, field.message_type.name))\n\n    # Try to figure out what this field actually is from the descriptor.\n    elif field.type == TYPE_DOUBLE:\n      type_descriptor = classes_dict[\"ProtoDouble\"](**kwargs)\n\n    elif field.type == TYPE_FLOAT:\n      type_descriptor = classes_dict[\"ProtoFloat\"](**kwargs)\n\n    elif field.type == TYPE_BOOL:\n      type_descriptor = classes_dict[\"ProtoBoolean\"](**kwargs)\n\n    elif field.type == TYPE_STRING:\n      type_descriptor = classes_dict[\"ProtoString\"](**kwargs)\n\n    elif field.type == TYPE_BYTES:\n      type_descriptor = classes_dict[\"ProtoBinary\"](**kwargs)\n      if options.dynamic_type:\n        # This may be a dynamic type. In this case the dynamic_type option\n        # names a method (which must exist) which should return the class of\n        # the embedded semantic value.\n        dynamic_cb = getattr(cls, options.dynamic_type, None)\n        if dynamic_cb is not None:\n          type_descriptor = classes_dict[\"ProtoDynamicEmbedded\"](\n              dynamic_cb=dynamic_cb, **kwargs)\n        else:\n          logging.warning(\"Dynamic type specifies a non existant callback %s\",\n                          options.dynamic_type)\n\n    elif (field.type == TYPE_MESSAGE and field.message_type.name == \"Any\"):\n      dynamic_cb = getattr(cls, options.dynamic_type, None)\n      type_descriptor = classes_dict[\"ProtoDynamicAnyValueEmbedded\"](\n          dynamic_cb=dynamic_cb, **kwargs)\n\n    elif field.type == TYPE_INT64 or field.type == TYPE_INT32:\n      type_descriptor = classes_dict[\"ProtoSignedInteger\"](**kwargs)\n\n    elif field.type == TYPE_UINT32 or field.type == TYPE_UINT64:\n      type_descriptor = classes_dict[\"ProtoUnsignedInteger\"](**kwargs)\n\n    # An embedded protocol buffer.\n    elif field.type == TYPE_MESSAGE and field.message_type:\n      # Refer to another protobuf. Note that the target does not need to be\n      # known at this time. It will be resolved using the late binding algorithm\n      # when it is known. Therefore this can actually also refer to this current\n      # protobuf (i.e. nested proto).\n      type_descriptor = classes_dict[\"ProtoEmbedded\"](\n          nested=field.message_type.name, **kwargs)\n\n      cls.recorded_rdf_deps.add(field.message_type.name)\n      if CHECK_PROTOBUF_DEPENDENCIES:\n        found = False\n        for d in cls.rdf_deps:\n          if (hasattr(d, \"__name__\") and d.__name__ == field.message_type.name\n              or d == field.message_type.name):\n            found = True\n\n        if not found:\n          raise rdfvalue.InitializeError(\n              \"%s.%s: TYPE_MESSAGE field %s is %s, \"\n              \"but type is missing from its dependencies list\" %\n              (cls.__module__, cls.__name__, field.name,\n               field.message_type.name))\n\n      # TODO(user): support late binding here.\n      if type_descriptor.type:\n        # This traps the following problem:\n        # class Certificate(rdf_protodict.RDFValueArray):\n        #    protobuf = jobs_pb2.BlobArray\n        #\n\n        # A primitive Protobuf definition like:\n        # message Certificate {\n        #   ....\n        # };\n\n        # And a field like:\n        # optional Certificate csr = 1 [(sem_type) = {\n        #   description: \"A Certificate RDFValue with the CSR in it.\",\n        # }];\n\n        # If we blindly allowed the Certificate RDFValue to be used, the\n        # semantic library will end up embedding a BlobArray protobuf, but the\n        # primitive library will still use Certificate.\n\n        # The name of the primitive protobuf the semantic type implements.\n        semantic_protobuf_primitive = type_descriptor.type.protobuf.__name__\n\n        # This is an error because the primitive library will use the protobuf\n        # named in the field, but the semantic library will implement a\n        # different protobuf.\n        if semantic_protobuf_primitive != field.message_type.name:\n          raise rdfvalue.InitializeError(\n              (\"%s.%s: Conflicting primitive (%s) and semantic protobuf %s \"\n               \"which implements primitive protobuf (%s)\") %\n              (cls.__name__, field.name, field.message_type.name,\n               type_descriptor.type.__name__, semantic_protobuf_primitive))\n\n    elif field.enum_type:  # It is an enum.\n      # TODO(hanuszczak): Protobuf descriptors use `bytes` objects to represent\n      # string values. Hence, we add additional `unicode` calls to convert them.\n      # It should be investigated whether this behaviour is needed in Python 3\n      # as well.\n\n      enum_desc = field.enum_type\n      enum_desc_name = str(enum_desc.name)\n      enum_dict = {}\n      enum_descriptions = {}\n      enum_labels = {}\n\n      for enum_value in enum_desc.values:\n        enum_value_name = str(enum_value.name)\n\n        enum_dict[enum_value_name] = enum_value.number\n        description = enum_value.GetOptions().Extensions[\n            semantic_pb2.description]\n        enum_descriptions[enum_value_name] = description\n        labels = [\n            label\n            for label in enum_value.GetOptions().Extensions[semantic_pb2.label]\n        ]\n        enum_labels[enum_value_name] = labels\n\n      type_descriptor = classes_dict[\"ProtoEnum\"](\n          enum_name=enum_desc_name,\n          enum=enum_dict,\n          enum_descriptions=enum_descriptions,\n          enum_labels=enum_labels,\n          **kwargs)\n\n      # Attach the enum container to the class for easy reference:\n      setattr(cls, enum_desc_name, type_descriptor.enum_container)\n\n    # If we do not recognize the type descriptor we ignore this field.\n    if type_descriptor is not None:\n      # If the field is repeated, wrap it in a ProtoList.\n      if field.label == LABEL_REPEATED:\n        options = field.GetOptions().Extensions[semantic_pb2.sem_type]\n        type_descriptor = classes_dict[\"ProtoList\"](\n            type_descriptor, labels=list(options.label))\n\n      try:\n        cls.AddDescriptor(type_descriptor)\n      except Exception:\n        logging.error(\"Failed to parse protobuf %s\", cls)\n        raise\n\n    else:\n      logging.error(\"Unknown field type for %s - Ignoring.\", field.name)\n\n  if hasattr(cls, \"rdf_deps\"):\n    leftover_deps = set()\n    for d in cls.rdf_deps:\n      try:\n        leftover_deps.add(d.__name__)\n      except AttributeError:\n        leftover_deps.add(d)\n    for d in cls.recorded_rdf_deps:\n      leftover_deps.remove(d)\n    if leftover_deps:\n      raise rdfvalue.InitializeError(\n          \"Found superfluous dependencies for %s: %s\" %\n          (cls.__name__, \",\".join(leftover_deps)))", "response": "Define from an existing protobuf file."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nstart all the component in their own threads.", "response": "def main(argv):\n  \"\"\"Sets up all the component in their own threads.\"\"\"\n\n  if flags.FLAGS.version:\n    print(\"GRR server {}\".format(config_server.VERSION[\"packageversion\"]))\n    return\n\n  # We use .startswith so that multiple copies of services can easily be\n  # created using systemd as worker1 worker2 ... worker25 etc.\n\n  if not flags.FLAGS.component:\n    raise ValueError(\"Need to specify which component to start.\")\n\n  # Start as a worker.\n  if flags.FLAGS.component.startswith(\"worker\"):\n    worker.main([argv])\n\n  # Start as a frontend that clients communicate with.\n  elif flags.FLAGS.component.startswith(\"frontend\"):\n    frontend.main([argv])\n\n  # Start as an AdminUI.\n  elif flags.FLAGS.component.startswith(\"admin_ui\"):\n    admin_ui.main([argv])\n\n  # Raise on invalid component.\n  else:\n    raise ValueError(\"No valid component specified. Got: \"\n                     \"%s.\" % flags.FLAGS.component)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef Operation(self, x, y):\n    if x in y:\n      return True\n\n    # x might be an iterable\n    # first we need to skip strings or we'll do silly things\n    if isinstance(x, string_types) or isinstance(x, bytes):\n      return False\n\n    try:\n      for value in x:\n        if value not in y:\n          return False\n      return True\n    except TypeError:\n      return False", "response": "Whether x is fully contained in y."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncall when at a leaf value. Should yield a value.", "response": "def _AtLeaf(self, attr_value):\n    \"\"\"Called when at a leaf value. Should yield a value.\"\"\"\n    if isinstance(attr_value, collections.Mapping):\n      # If the result is a dict, return each key/value pair as a new dict.\n      for k, v in iteritems(attr_value):\n        yield {k: v}\n    else:\n      yield attr_value"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _AtNonLeaf(self, attr_value, path):\n    try:\n      if isinstance(attr_value, collections.Mapping):\n        # If it's dictionary-like, treat the dict key as the attribute..\n        sub_obj = attr_value.get(path[1])\n        if len(path) > 2:\n          # Expand any additional elements underneath the key.\n          sub_obj = self.Expand(sub_obj, path[2:])\n        if isinstance(sub_obj, string_types):\n          # If it is a string, stop here\n          yield sub_obj\n        elif isinstance(sub_obj, collections.Mapping):\n          # If the result is a dict, return each key/value pair as a new dict.\n          for k, v in iteritems(sub_obj):\n            yield {k: v}\n        else:\n          for value in sub_obj:\n            yield value\n      else:\n        # If it's an iterable, we recurse on each value.\n        for sub_obj in attr_value:\n          for value in self.Expand(sub_obj, path[1:]):\n            yield value\n    except TypeError:  # This is then not iterable, we recurse with the value\n      for value in self.Expand(attr_value, path[1:]):\n        yield value", "response": "Recursively yield the values at the given path."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef Compile(self, filter_implemention):\n    operator = self.operator.lower()\n    if operator == \"and\" or operator == \"&&\":\n      method = \"AndFilter\"\n    elif operator == \"or\" or operator == \"||\":\n      method = \"OrFilter\"\n    else:\n      raise ParseError(\"Invalid binary operator %s\" % operator)\n\n    args = [x.Compile(filter_implemention) for x in self.args]\n    return filter_implemention.FILTERS[method](arguments=args)", "response": "Compile the binary expression into a filter object."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ninsert an arg to the current expression.", "response": "def InsertArg(self, string=\"\", **_):\n    \"\"\"Insert an arg to the current expression.\"\"\"\n    if self.state == \"LIST_ARG\":\n      self.list_args.append(string)\n    elif self.current_expression.AddArg(string):\n      # This expression is complete\n      self.stack.append(self.current_expression)\n      self.current_expression = self.expression_cls()\n      # We go to the BINARY state, to find if there's an AND or OR operator\n      return \"BINARY\""}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef InsertFloatArg(self, string=\"\", **_):\n    try:\n      float_value = float(string)\n      return self.InsertArg(float_value)\n    except (TypeError, ValueError):\n      raise ParseError(\"%s is not a valid float.\" % string)", "response": "Inserts a Float argument."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ninserts an Integer in base16 argument.", "response": "def InsertInt16Arg(self, string=\"\", **_):\n    \"\"\"Inserts an Integer in base16 argument.\"\"\"\n    try:\n      int_value = int(string, 16)\n      return self.InsertArg(int_value)\n    except (TypeError, ValueError):\n      raise ParseError(\"%s is not a valid base16 integer.\" % string)"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nescapes backslashes found inside a string quote.", "response": "def StringEscape(self, string, match, **_):\n    \"\"\"Escape backslashes found inside a string quote.\n\n    Backslashes followed by anything other than [\\'\"rnbt] will raise an Error.\n\n    Args:\n      string: The string that matched.\n      match: The match object (m.group(1) is the escaped code)\n\n    Raises:\n      ParseError: For strings other than those used to define a regexp, raise an\n        error if the escaped string is not one of [\\'\"rnbt].\n    \"\"\"\n    precondition.AssertType(string, Text)\n\n    # Allow unfiltered strings for regexp operations so that escaped special\n    # characters (e.g. \\*) or special sequences (e.g. \\w) can be used in\n    # objectfilter.\n    if self.current_expression.operator == \"regexp\":\n      self.string += compatibility.UnescapeString(string)\n    elif match.group(1) in \"\\\\'\\\"rnbt\":\n      self.string += compatibility.UnescapeString(string)\n    else:\n      raise ParseError(\"Invalid escape character %s.\" % string)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef HexEscape(self, string, match, **_):\n    hex_string = match.group(1)\n    try:\n      self.string += binascii.unhexlify(hex_string).decode(\"utf-8\")\n    # TODO: In Python 2 `binascii` throws `TypeError` for invalid\n    # input values (for whathever reason). This behaviour is fixed in Python 3\n    # where `binascii.Error` (a subclass of `ValueError`) is raised. Once we do\n    # not have to support Python 2 anymore, this `TypeError` catch should be\n    # removed.\n    except (binascii.Error, TypeError) as error:\n      raise ParseError(\"Invalid hex escape '{}': {}\".format(hex_string, error))", "response": "Converts a hex escaped string."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ngenerates a CSRF token based on a secret key id and time.", "response": "def GenerateCSRFToken(user_id, time):\n  \"\"\"Generates a CSRF token based on a secret key, id and time.\"\"\"\n  precondition.AssertType(user_id, Text)\n  precondition.AssertOptionalType(time, int)\n\n  time = time or rdfvalue.RDFDatetime.Now().AsMicrosecondsSinceEpoch()\n\n  secret = config.CONFIG.Get(\"AdminUI.csrf_secret_key\", None)\n  # TODO(amoser): Django is deprecated. Remove this at some point.\n  if not secret:\n    secret = config.CONFIG[\"AdminUI.django_secret_key\"]\n  digester = hmac.new(secret.encode(\"ascii\"), digestmod=hashlib.sha256)\n  digester.update(user_id.encode(\"ascii\"))\n  digester.update(CSRF_DELIMITER)\n  digester.update(str(time).encode(\"ascii\"))\n  digest = digester.digest()\n\n  token = base64.urlsafe_b64encode(b\"%s%s%d\" % (digest, CSRF_DELIMITER, time))\n  return token.rstrip(b\"=\")"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef StoreCSRFCookie(user, response):\n\n  csrf_token = GenerateCSRFToken(user, None)\n  response.set_cookie(\n      \"csrftoken\", csrf_token, max_age=CSRF_TOKEN_DURATION.seconds)", "response": "Decorator for WSGI handler that inserts CSRF cookie into response."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef ValidateCSRFTokenOrRaise(request):\n\n  # CSRF check doesn't make sense for GET/HEAD methods, because they can\n  # (and are) used when downloading files through <a href> links - and\n  # there's no way to set X-CSRFToken header in this case.\n  if request.method in (\"GET\", \"HEAD\"):\n    return\n\n  # In the ideal world only JavaScript can be used to add a custom header, and\n  # only within its origin. By default, browsers don't allow JavaScript to\n  # make cross origin requests.\n  #\n  # Unfortunately, in the real world due to bugs in browsers plugins, it can't\n  # be guaranteed that a page won't set an HTTP request with a custom header\n  # set. That's why we also check the contents of a header via an HMAC check\n  # with a server-stored secret.\n  #\n  # See for more details:\n  # https://www.owasp.org/index.php/Cross-Site_Request_Forgery_(CSRF)_Prevention_Cheat_Sheet\n  # (Protecting REST Services: Use of Custom Request Headers).\n  csrf_token = request.headers.get(\"X-CSRFToken\", \"\").encode(\"ascii\")\n  if not csrf_token:\n    logging.info(\"Did not find headers CSRF token for: %s\", request.path)\n    raise werkzeug_exceptions.Forbidden(\"CSRF token is missing\")\n\n  try:\n    decoded = base64.urlsafe_b64decode(csrf_token + b\"==\")\n    digest, token_time = decoded.rsplit(CSRF_DELIMITER, 1)\n    token_time = int(token_time)\n  except (TypeError, ValueError):\n    logging.info(\"Malformed CSRF token for: %s\", request.path)\n    raise werkzeug_exceptions.Forbidden(\"Malformed CSRF token\")\n\n  if len(digest) != hashlib.sha256().digest_size:\n    logging.info(\"Invalid digest size for: %s\", request.path)\n    raise werkzeug_exceptions.Forbidden(\"Malformed CSRF token digest\")\n\n  expected = GenerateCSRFToken(request.user, token_time)\n  if not constant_time.bytes_eq(csrf_token, expected):\n    logging.info(\"Non-matching CSRF token for: %s\", request.path)\n    raise werkzeug_exceptions.Forbidden(\"Non-matching CSRF token\")\n\n  current_time = rdfvalue.RDFDatetime.Now().AsMicrosecondsSinceEpoch()\n  if current_time - token_time > CSRF_TOKEN_DURATION.microseconds:\n    logging.info(\"Expired CSRF token for: %s\", request.path)\n    raise werkzeug_exceptions.Forbidden(\"Expired CSRF token\")", "response": "Decorator for WSGI handler that checks CSRF token against the request."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _BuildToken(self, request, execution_time):\n    token = access_control.ACLToken(\n        username=request.user,\n        reason=request.args.get(\"reason\", \"\"),\n        process=\"GRRAdminUI\",\n        expiry=rdfvalue.RDFDatetime.Now() + execution_time)\n\n    for field in [\"Remote_Addr\", \"X-Forwarded-For\"]:\n      remote_addr = request.headers.get(field, \"\")\n      if remote_addr:\n        token.source_ips.append(remote_addr)\n    return token", "response": "Builds an ACLToken from the request."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _HandleHomepage(self, request):\n\n    _ = request\n\n    env = jinja2.Environment(\n        loader=jinja2.FileSystemLoader(config.CONFIG[\"AdminUI.template_root\"]),\n        autoescape=True)\n\n    create_time = psutil.Process(os.getpid()).create_time()\n    context = {\n        \"heading\":\n            config.CONFIG[\"AdminUI.heading\"],\n        \"report_url\":\n            config.CONFIG[\"AdminUI.report_url\"],\n        \"help_url\":\n            config.CONFIG[\"AdminUI.help_url\"],\n        \"timestamp\":\n            utils.SmartStr(create_time),\n        \"use_precompiled_js\":\n            config.CONFIG[\"AdminUI.use_precompiled_js\"],\n        # Used in conjunction with FirebaseWebAuthManager.\n        \"firebase_api_key\":\n            config.CONFIG[\"AdminUI.firebase_api_key\"],\n        \"firebase_auth_domain\":\n            config.CONFIG[\"AdminUI.firebase_auth_domain\"],\n        \"firebase_auth_provider\":\n            config.CONFIG[\"AdminUI.firebase_auth_provider\"],\n        \"grr_version\":\n            config.CONFIG[\"Source.version_string\"]\n    }\n    template = env.get_template(\"base.html\")\n    response = werkzeug_wrappers.Response(\n        template.render(context), mimetype=\"text/html\")\n\n    # For a redirect-based Firebase authentication scheme we won't have any\n    # user information at this point - therefore checking if the user is\n    # present.\n    try:\n      StoreCSRFCookie(request.user, response)\n    except RequestHasNoUser:\n      pass\n\n    return response", "response": "Renders GRR home page by rendering base. html Jinja template."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nredirect to GitHub - hosted documentation.", "response": "def _RedirectToRemoteHelp(self, path):\n    \"\"\"Redirect to GitHub-hosted documentation.\"\"\"\n    allowed_chars = set(string.ascii_letters + string.digits + \"._-/\")\n    if not set(path) <= allowed_chars:\n      raise RuntimeError(\"Unusual chars in path %r - \"\n                         \"possible exploit attempt.\" % path)\n\n    target_path = os.path.join(config.CONFIG[\"AdminUI.docs_location\"], path)\n\n    # We have to redirect via JavaScript to have access to and to preserve the\n    # URL hash. We don't know the hash part of the url on the server.\n    return werkzeug_wrappers.Response(\n        \"\"\"\n<script>\nvar friendly_hash = window.location.hash;\nwindow.location = '%s' + friendly_hash;\n</script>\n\"\"\" % target_path,\n        mimetype=\"text/html\")"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef WSGIHandler(self):\n    sdm = werkzeug_wsgi.SharedDataMiddleware(self, {\n        \"/\": config.CONFIG[\"AdminUI.document_root\"],\n    })\n    # Use DispatcherMiddleware to make sure that SharedDataMiddleware is not\n    # used at all if the URL path doesn't start with \"/static\". This is a\n    # workaround for cases when unicode URLs are used on systems with\n    # non-unicode filesystems (as detected by Werkzeug). In this case\n    # SharedDataMiddleware may fail early while trying to convert the\n    # URL into the file path and not dispatch the call further to our own\n    # WSGI handler.\n    return werkzeug_wsgi.DispatcherMiddleware(self, {\n        \"/static\": sdm,\n    })", "response": "Returns GRR s WSGI handler."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nimports the plugins once only.", "response": "def RunOnce(self):\n    \"\"\"Import the plugins once only.\"\"\"\n    # pylint: disable=unused-variable,g-import-not-at-top\n    from grr_response_server.gui import gui_plugins\n    # pylint: enable=unused-variable,g-import-not-at-top\n\n    if config.CONFIG.Get(\"AdminUI.django_secret_key\", None):\n      logging.warning(\n          \"The AdminUI.django_secret_key option has been deprecated, \"\n          \"please use AdminUI.csrf_secret_key instead.\")"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nlists all registered Grr binaries.", "response": "def ListGrrBinaries(context=None):\n  \"\"\"Lists all registered Grr binaries.\"\"\"\n\n  items = context.SendIteratorRequest(\"ListGrrBinaries\", None)\n  return utils.MapItemsIterator(\n      lambda data: GrrBinary(data=data, context=context), items)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef Execute(self, message):\n    self.message = message\n    if message:\n      self.require_fastpoll = message.require_fastpoll\n\n    args = None\n    try:\n      if self.message.args_rdf_name:\n        if not self.in_rdfvalue:\n          raise RuntimeError(\"Did not expect arguments, got %s.\" %\n                             self.message.args_rdf_name)\n\n        if self.in_rdfvalue.__name__ != self.message.args_rdf_name:\n          raise RuntimeError(\n              \"Unexpected arg type %s != %s.\" %\n              (self.message.args_rdf_name, self.in_rdfvalue.__name__))\n\n        args = self.message.payload\n\n      # Only allow authenticated messages in the client\n      if self._authentication_required and (\n          self.message.auth_state !=\n          rdf_flows.GrrMessage.AuthorizationState.AUTHENTICATED):\n        raise RuntimeError(\"Message for %s was not Authenticated.\" %\n                           self.message.name)\n\n      self.cpu_start = self.proc.cpu_times()\n      self.cpu_limit = self.message.cpu_limit\n\n      if getattr(flags.FLAGS, \"debug_client_actions\", False):\n        pdb.set_trace()\n\n      try:\n        self.Run(args)\n\n      # Ensure we always add CPU usage even if an exception occurred.\n      finally:\n        used = self.proc.cpu_times()\n        self.cpu_used = (used.user - self.cpu_start.user,\n                         used.system - self.cpu_start.system)\n\n    except NetworkBytesExceededError as e:\n      self.SetStatus(rdf_flows.GrrStatus.ReturnedStatus.NETWORK_LIMIT_EXCEEDED,\n                     \"%r: %s\" % (e, e), traceback.format_exc())\n\n    # We want to report back all errors and map Python exceptions to\n    # Grr Errors.\n    except Exception as e:  # pylint: disable=broad-except\n      self.SetStatus(rdf_flows.GrrStatus.ReturnedStatus.GENERIC_ERROR,\n                     \"%r: %s\" % (e, e), traceback.format_exc())\n\n      if flags.FLAGS.pdb_post_mortem:\n        self.DisableNanny()\n        pdb.post_mortem()\n\n    if self.status.status != rdf_flows.GrrStatus.ReturnedStatus.OK:\n      logging.info(\"Job Error (%s): %s\", self.__class__.__name__,\n                   self.status.error_message)\n\n      if self.status.backtrace:\n        logging.debug(self.status.backtrace)\n\n    if self.cpu_used:\n      self.status.cpu_time_used.user_cpu_time = self.cpu_used[0]\n      self.status.cpu_time_used.system_cpu_time = self.cpu_used[1]\n\n    # This returns the error status of the Actions to the flow.\n    self.SendReply(self.status, message_type=rdf_flows.GrrMessage.Type.STATUS)\n\n    self._RunGC()", "response": "This function parses the RDFValue from the server and runs the appropriate method."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef SetStatus(self, status, message=\"\", backtrace=None):\n    self.status.status = status\n    self.status.error_message = utils.SmartUnicode(message)\n    if backtrace:\n      self.status.backtrace = utils.SmartUnicode(backtrace)", "response": "Sets a status to report back to the server."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nsend a reply to the server.", "response": "def SendReply(self,\n                rdf_value=None,\n                session_id=None,\n                message_type=rdf_flows.GrrMessage.Type.MESSAGE):\n    \"\"\"Send response back to the server.\"\"\"\n    # TODO(hanuszczak): This is pretty bad. Here we assume that if the session\n    # id is not none we are \"replying\" to a well-known flow. If we are replying\n    # to a well-known flow we cannot increment the response id (since these are\n    # not really responses) because otherwise the parent flow will think it is\n    # missing some responses.\n    #\n    # Moreover, the message queue (at least in the flow test helper workflow)\n    # expects flow messages to have request id equal to 0. This is rather\n    # convoluted and should be untangled in the future.\n    #\n    # Once we have a separate method for communicating with well-known flows\n    # this if-statement should no longer be relevant (since setting custom\n    # session ids would become illegal).\n    if session_id is None:\n      response_id = self.response_id\n      request_id = self.message.request_id\n      session_id = self.message.session_id\n      self.response_id += 1\n    else:\n      response_id = 0\n      request_id = 0\n\n    self.grr_worker.SendReply(\n        rdf_value,\n        # This is not strictly necessary but adds context\n        # to this response.\n        name=self.__class__.__name__,\n        session_id=session_id,\n        response_id=response_id,\n        request_id=request_id,\n        message_type=message_type,\n        task_id=self.message.Get(\"task_id\") or None,\n        require_fastpoll=self.require_fastpoll)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef Progress(self):\n    now = time.time()\n    if now - self.last_progress_time <= 2:\n      return\n\n    self.last_progress_time = now\n\n    # Prevent the machine from sleeping while the action is running.\n    client_utils.KeepAlive()\n\n    self.grr_worker.Heartbeat()\n\n    user_start = self.cpu_start.user\n    system_start = self.cpu_start.system\n    cpu_times = self.proc.cpu_times()\n    user_end = cpu_times.user\n    system_end = cpu_times.system\n\n    used_cpu = user_end - user_start + system_end - system_start\n\n    if used_cpu > self.cpu_limit:\n      self.grr_worker.SendClientAlert(\"Cpu limit exceeded.\")\n      raise CPUExceededError(\"Action exceeded cpu limit.\")", "response": "Indicate progress of the client action."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nwriting the provided graphs to the DB with the given client label and timestamp.", "response": "def WriteClientGraphSeries(\n      self,\n      graph_series,\n      client_label,\n      timestamp,\n      cursor=None,\n  ):\n    \"\"\"Writes the provided graphs to the DB with the given client label.\"\"\"\n    args = {\n        \"client_label\": client_label,\n        \"report_type\": graph_series.report_type.SerializeToDataStore(),\n        \"timestamp\": mysql_utils.RDFDatetimeToTimestamp(timestamp),\n        \"graph_series\": graph_series.SerializeToString(),\n    }\n\n    query = \"\"\"\n      INSERT INTO client_report_graphs (client_label, report_type, timestamp,\n                                        graph_series)\n      VALUES (%(client_label)s, %(report_type)s, FROM_UNIXTIME(%(timestamp)s),\n              %(graph_series)s)\n      ON DUPLICATE KEY UPDATE graph_series = VALUES(graph_series)\n    \"\"\"\n\n    cursor.execute(query, args)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef ReadAllClientGraphSeries(\n      self,\n      client_label,\n      report_type,\n      time_range = None,\n      cursor=None):\n    \"\"\"Reads graph series for the given label and report-type from the DB.\"\"\"\n    query = \"\"\"\n      SELECT UNIX_TIMESTAMP(timestamp), graph_series\n      FROM client_report_graphs\n      WHERE client_label = %s AND report_type = %s\n    \"\"\"\n    args = [client_label, report_type.SerializeToDataStore()]\n\n    if time_range is not None:\n      query += \"AND `timestamp` BETWEEN FROM_UNIXTIME(%s) AND FROM_UNIXTIME(%s)\"\n      args += [\n          mysql_utils.RDFDatetimeToTimestamp(time_range.start),\n          mysql_utils.RDFDatetimeToTimestamp(time_range.end)\n      ]\n\n    cursor.execute(query, args)\n    results = {}\n    for timestamp, raw_series in cursor.fetchall():\n      # TODO(hanuszczak): pytype does not seem to understand overloads, so it is\n      # not possible to correctly annotate `TimestampToRDFDatetime`.\n      timestamp = cast(rdfvalue.RDFDatetime,\n                       mysql_utils.TimestampToRDFDatetime(timestamp))\n\n      series = rdf_stats.ClientGraphSeries.FromSerializedString(raw_series)\n      results[timestamp] = series\n    return results", "response": "Reads graph series for the given label and report - type from the DB."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreading the latest graph series for a given client - label.", "response": "def ReadMostRecentClientGraphSeries(\n      self,\n      client_label,\n      report_type,\n      cursor=None):\n    \"\"\"Fetches the latest graph series for a client-label from the DB.\"\"\"\n    query = \"\"\"\n      SELECT graph_series\n      FROM client_report_graphs\n      WHERE client_label = %s AND report_type = %s\n      ORDER BY timestamp DESC\n      LIMIT 1\n    \"\"\"\n    args = [client_label, report_type.SerializeToDataStore()]\n    cursor.execute(query, args)\n    result = cursor.fetchone()\n    if result is None:\n      return None\n    else:\n      return rdf_stats.ClientGraphSeries.FromSerializedString(result[0])"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nconvert given RDFValue to an RDFURN of a file to be downloaded.", "response": "def CollectionItemToAff4Path(item, client_id=None):\n  \"\"\"Converts given RDFValue to an RDFURN of a file to be downloaded.\"\"\"\n  if isinstance(item, rdf_flows.GrrMessage):\n    client_id = item.source\n    item = item.payload\n  elif isinstance(item, rdf_flow_objects.FlowResult):\n    client_id = item.client_id\n    item = item.payload\n\n  if not client_id:\n    raise ValueError(\"Could not determine client_id.\")\n  elif isinstance(client_id, string_types):\n    client_id = rdf_client.ClientURN(client_id)\n\n  if isinstance(item, rdf_client_fs.StatEntry):\n    return item.AFF4Path(client_id)\n  elif isinstance(item, rdf_file_finder.FileFinderResult):\n    return item.stat_entry.AFF4Path(client_id)\n  elif isinstance(item, collectors.ArtifactFilesDownloaderResult):\n    if item.HasField(\"downloaded_file\"):\n      return item.downloaded_file.AFF4Path(client_id)\n\n  raise ItemNotExportableError(item)"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nconverts given RDFValue to a ClientPath of a file to be downloaded.", "response": "def CollectionItemToClientPath(item, client_id=None):\n  \"\"\"Converts given RDFValue to a ClientPath of a file to be downloaded.\"\"\"\n  if isinstance(item, rdf_flows.GrrMessage):\n    client_id = item.source\n    item = item.payload\n  elif isinstance(item, rdf_flow_objects.FlowResult):\n    client_id = item.client_id\n    item = item.payload\n\n  if client_id is None:\n    raise ValueError(\"Could not determine client_id.\")\n  elif isinstance(client_id, rdfvalue.RDFURN):\n    client_id = client_id.Basename()\n\n  if isinstance(item, rdf_client_fs.StatEntry):\n    return db.ClientPath.FromPathSpec(client_id, item.pathspec)\n  elif isinstance(item, rdf_file_finder.FileFinderResult):\n    return db.ClientPath.FromPathSpec(client_id, item.stat_entry.pathspec)\n  elif isinstance(item, collectors.ArtifactFilesDownloaderResult):\n    if item.HasField(\"downloaded_file\"):\n      return db.ClientPath.FromPathSpec(client_id,\n                                        item.downloaded_file.pathspec)\n\n  raise ItemNotExportableError(item)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn a knowledgebase from an rdf client object.", "response": "def GetKnowledgeBase(rdf_client_obj, allow_uninitialized=False):\n  \"\"\"Returns a knowledgebase from an rdf client object.\"\"\"\n  if not allow_uninitialized:\n    if rdf_client_obj is None:\n      raise artifact_utils.KnowledgeBaseUninitializedError(\n          \"No client snapshot given.\")\n    if rdf_client_obj.knowledge_base is None:\n      raise artifact_utils.KnowledgeBaseUninitializedError(\n          \"KnowledgeBase empty for %s.\" % rdf_client_obj.client_id)\n    kb = rdf_client_obj.knowledge_base\n    if not kb.os:\n      raise artifact_utils.KnowledgeBaseAttributesMissingError(\n          \"KnowledgeBase missing OS for %s. Knowledgebase content: %s\" %\n          (rdf_client_obj.client_id, kb))\n  if rdf_client_obj is None or rdf_client_obj.knowledge_base is None:\n    return rdf_client.KnowledgeBase()\n\n  version = rdf_client_obj.os_version.split(\".\")\n  kb = rdf_client_obj.knowledge_base\n  try:\n    kb.os_major_version = int(version[0])\n    if len(version) >= 1:\n      kb.os_minor_version = int(version[1])\n  except ValueError:\n    pass\n\n  return kb"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nset core values from GRR into the knowledgebase.", "response": "def SetCoreGRRKnowledgeBaseValues(kb, client_obj):\n  \"\"\"Set core values from GRR into the knowledgebase.\"\"\"\n  client_schema = client_obj.Schema\n  kb.fqdn = utils.SmartUnicode(client_obj.Get(client_schema.FQDN, \"\"))\n  if not kb.fqdn:\n    kb.fqdn = utils.SmartUnicode(client_obj.Get(client_schema.HOSTNAME, \"\"))\n  versions = client_obj.Get(client_schema.OS_VERSION)\n  if versions and versions.versions:\n    try:\n      kb.os_major_version = versions.versions[0]\n      kb.os_minor_version = versions.versions[1]\n    except IndexError:\n      # Some OSs don't have a minor version.\n      pass\n  client_os = client_obj.Get(client_schema.SYSTEM)\n  if client_os:\n    kb.os = utils.SmartUnicode(client_obj.Get(client_schema.SYSTEM))"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef ApplyParsersToResponses(parser_factory, responses, flow_obj):\n  # We have some processors to run.\n  knowledge_base = flow_obj.state.knowledge_base\n\n  parsed_responses = []\n\n  if parser_factory.HasSingleResponseParsers():\n    for response in responses:\n      for parser in parser_factory.SingleResponseParsers():\n        parsed_responses.extend(\n            parser.ParseResponse(knowledge_base, response,\n                                 flow_obj.args.path_type))\n\n  for parser in parser_factory.MultiResponseParsers():\n    parsed_responses.extend(parser.ParseResponses(knowledge_base, responses))\n\n  has_single_file_parsers = parser_factory.HasSingleFileParsers()\n  has_multi_file_parsers = parser_factory.HasMultiFileParsers()\n\n  if has_single_file_parsers or has_multi_file_parsers:\n    precondition.AssertIterableType(responses, rdf_client_fs.StatEntry)\n    pathspecs = [response.pathspec for response in responses]\n    if data_store.RelationalDBEnabled():\n      # TODO(amoser): This is not super efficient, AFF4 provided an api to open\n      # all pathspecs at the same time, investigate if optimizing this is worth\n      # it.\n      filedescs = []\n      for pathspec in pathspecs:\n        client_path = db.ClientPath.FromPathSpec(flow_obj.client_id, pathspec)\n        filedescs.append(file_store.OpenFile(client_path))\n    else:\n      filedescs = MultiOpenAff4File(flow_obj, pathspecs)\n\n  if has_single_file_parsers:\n    for response, filedesc in zip(responses, filedescs):\n      for parser in parser_factory.SingleFileParsers():\n        parsed_responses.extend(\n            parser.ParseFile(knowledge_base, response.pathspec, filedesc))\n\n  if has_multi_file_parsers:\n    for parser in parser_factory.MultiFileParsers():\n      parsed_responses.extend(\n          parser.ParseFiles(knowledge_base, pathspecs, filedescs))\n\n  return parsed_responses or responses", "response": "Parse the given list of responses with applicable parsers."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef UploadArtifactYamlFile(file_content,\n                           overwrite=True,\n                           overwrite_system_artifacts=False):\n  \"\"\"Upload a yaml or json file as an artifact to the datastore.\"\"\"\n  loaded_artifacts = []\n  registry_obj = artifact_registry.REGISTRY\n  # Make sure all artifacts are loaded so we don't accidentally overwrite one.\n  registry_obj.GetArtifacts(reload_datastore_artifacts=True)\n\n  new_artifacts = registry_obj.ArtifactsFromYaml(file_content)\n  new_artifact_names = set()\n  # A quick syntax check before we upload anything.\n  for artifact_value in new_artifacts:\n    artifact_registry.ValidateSyntax(artifact_value)\n    new_artifact_names.add(artifact_value.name)\n\n  # Iterate through each artifact adding it to the collection.\n  artifact_coll = artifact_registry.ArtifactCollection(ARTIFACT_STORE_ROOT_URN)\n  current_artifacts = list(artifact_coll)\n\n  # We need to remove artifacts we are overwriting.\n  filtered_artifacts = [\n      art for art in current_artifacts if art.name not in new_artifact_names\n  ]\n\n  artifact_coll.Delete()\n  with data_store.DB.GetMutationPool() as pool:\n    for artifact_value in filtered_artifacts:\n      artifact_coll.Add(artifact_value, mutation_pool=pool)\n\n    for artifact_value in new_artifacts:\n      registry_obj.RegisterArtifact(\n          artifact_value,\n          source=\"datastore:%s\" % ARTIFACT_STORE_ROOT_URN,\n          overwrite_if_exists=overwrite,\n          overwrite_system_artifacts=overwrite_system_artifacts)\n\n      artifact_coll.Add(artifact_value, mutation_pool=pool)\n      if data_store.RelationalDBEnabled():\n        data_store.REL_DB.WriteArtifact(artifact_value)\n\n      loaded_artifacts.append(artifact_value)\n\n      name = artifact_value.name\n      logging.info(\"Uploaded artifact %s to %s\", name, ARTIFACT_STORE_ROOT_URN)\n\n  # Once all artifacts are loaded we can validate dependencies. Note that we do\n  # not have to perform a syntax validation because it is already done after\n  # YAML is parsed.\n  for artifact_value in loaded_artifacts:\n    artifact_registry.ValidateDependencies(artifact_value)", "response": "Uploads a yaml file as an artifact to the datastore."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef CallLoggedAndAccounted(f):\n\n  @functools.wraps(f)\n  def Decorator(*args, **kwargs):\n    try:\n      start_time = time.time()\n      result = f(*args, **kwargs)\n      latency = time.time() - start_time\n\n      stats_collector_instance.Get().RecordEvent(\n          \"db_request_latency\", latency, fields=[f.__name__])\n      logging.debug(\"DB request %s SUCCESS (%.3fs)\", f.__name__, latency)\n\n      return result\n    except db.Error as e:\n      stats_collector_instance.Get().IncrementCounter(\n          \"db_request_errors\", fields=[f.__name__, \"grr\"])\n      logging.debug(\"DB request %s GRR ERROR: %s\", f.__name__,\n                    utils.SmartUnicode(e))\n      raise\n    except Exception as e:\n      stats_collector_instance.Get().IncrementCounter(\n          \"db_request_errors\", fields=[f.__name__, \"db\"])\n      logging.debug(\"DB request %s INTERNAL DB ERROR : %s\", f.__name__,\n                    utils.SmartUnicode(e))\n      raise\n\n  return Decorator", "response": "Decorator to log and account for a DB call."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nescaping wildcard characters for strings intended to be used with LIKE.", "response": "def EscapeWildcards(string):\n  \"\"\"Escapes wildcard characters for strings intended to be used with `LIKE`.\n\n  Databases don't automatically escape wildcard characters ('%', '_'), so any\n  non-literal string that is passed to `LIKE` and is expected to match literally\n  has to be manually escaped.\n\n  Args:\n    string: A string to escape.\n\n  Returns:\n    An escaped string.\n  \"\"\"\n  precondition.AssertType(string, Text)\n  return string.replace(\"%\", r\"\\%\").replace(\"_\", r\"\\_\")"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nconverting hunt id string to an integer.", "response": "def HuntIDToInt(hunt_id):\n  \"\"\"Convert hunt id string to an integer.\"\"\"\n  # TODO(user): This code is only needed for a brief period of time when we\n  # allow running new rel-db flows with old aff4-based hunts. In this scenario\n  # parent_hunt_id is effectively not used, but it has to be an\n  # integer. Stripping \"H:\" from hunt ids then makes the rel-db happy. Remove\n  # this code when hunts are rel-db only.\n  if hunt_id.startswith(\"H:\"):\n    hunt_id = hunt_id[2:]\n\n  try:\n    return int(hunt_id or \"0\", 16)\n  except ValueError as e:\n    raise HuntIDIsNotAnIntegerError(e)"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nwrites metadata about the client.", "response": "def WriteClientMetadata(self,\n                          client_id,\n                          certificate=None,\n                          fleetspeak_enabled=None,\n                          first_seen=None,\n                          last_ping=None,\n                          last_clock=None,\n                          last_ip=None,\n                          last_foreman=None):\n    \"\"\"Write metadata about the client.\"\"\"\n    md = {}\n    if certificate is not None:\n      md[\"certificate\"] = certificate\n\n    if fleetspeak_enabled is not None:\n      md[\"fleetspeak_enabled\"] = fleetspeak_enabled\n\n    if first_seen is not None:\n      md[\"first_seen\"] = first_seen\n\n    if last_ping is not None:\n      md[\"ping\"] = last_ping\n\n    if last_clock is not None:\n      md[\"clock\"] = last_clock\n\n    if last_ip is not None:\n      md[\"ip\"] = last_ip\n\n    if last_foreman is not None:\n      md[\"last_foreman_time\"] = last_foreman\n\n    if not md:\n      raise ValueError(\"NOOP write.\")\n\n    self.metadatas.setdefault(client_id, {}).update(md)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef MultiReadClientMetadata(self, client_ids):\n    res = {}\n    for client_id in client_ids:\n      md = self.metadatas.get(client_id, None)\n      if md is None:\n        continue\n\n      res[client_id] = rdf_objects.ClientMetadata(\n          certificate=md.get(\"certificate\"),\n          fleetspeak_enabled=md.get(\"fleetspeak_enabled\"),\n          first_seen=md.get(\"first_seen\"),\n          ping=md.get(\"ping\"),\n          clock=md.get(\"clock\"),\n          ip=md.get(\"ip\"),\n          last_foreman_time=md.get(\"last_foreman_time\"),\n          last_crash_timestamp=md.get(\"last_crash_timestamp\"),\n          startup_info_timestamp=md.get(\"startup_info_timestamp\"))\n\n    return res", "response": "Reads ClientMetadata records for a list of clients."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef WriteClientSnapshot(self, snapshot):\n    client_id = snapshot.client_id\n\n    if client_id not in self.metadatas:\n      raise db.UnknownClientError(client_id)\n\n    startup_info = snapshot.startup_info\n    snapshot.startup_info = None\n\n    ts = rdfvalue.RDFDatetime.Now()\n    history = self.clients.setdefault(client_id, {})\n    history[ts] = snapshot.SerializeToString()\n\n    history = self.startup_history.setdefault(client_id, {})\n    history[ts] = startup_info.SerializeToString()\n\n    snapshot.startup_info = startup_info", "response": "Writes a new client snapshot."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef MultiReadClientSnapshot(self, client_ids):\n    res = {}\n    for client_id in client_ids:\n      history = self.clients.get(client_id, None)\n      if not history:\n        res[client_id] = None\n        continue\n      last_timestamp = max(history)\n      last_serialized = history[last_timestamp]\n      client_obj = rdf_objects.ClientSnapshot.FromSerializedString(\n          last_serialized)\n      client_obj.timestamp = last_timestamp\n      client_obj.startup_info = rdf_client.StartupInfo.FromSerializedString(\n          self.startup_history[client_id][last_timestamp])\n      res[client_id] = client_obj\n    return res", "response": "Reads the latest client snapshots for a list of clients."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nread full client information for a list of clients.", "response": "def MultiReadClientFullInfo(self, client_ids, min_last_ping=None):\n    \"\"\"Reads full client information for a list of clients.\"\"\"\n    res = {}\n    for client_id in client_ids:\n      try:\n        md = self.ReadClientMetadata(client_id)\n      except db.UnknownClientError:\n        continue\n\n      if md and min_last_ping and md.ping < min_last_ping:\n        continue\n      last_snapshot = self.ReadClientSnapshot(client_id)\n      full_info = rdf_objects.ClientFullInfo(\n          metadata=md,\n          labels=self.ReadClientLabels(client_id),\n          last_startup_info=self.ReadClientStartupInfo(client_id))\n      if last_snapshot is None:\n        full_info.last_snapshot = rdf_objects.ClientSnapshot(\n            client_id=client_id)\n      else:\n        full_info.last_snapshot = last_snapshot\n      res[client_id] = full_info\n    return res"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreading the last - ping timestamps for clients in the DB.", "response": "def ReadClientLastPings(self,\n                          min_last_ping=None,\n                          max_last_ping=None,\n                          fleetspeak_enabled=None):\n    \"\"\"Reads last-ping timestamps for clients in the DB.\"\"\"\n    last_pings = {}\n    for client_id, metadata in iteritems(self.metadatas):\n      last_ping = metadata.get(\"ping\", rdfvalue.RDFDatetime(0))\n      is_fleetspeak_client = metadata.get(\"fleetspeak_enabled\", False)\n      if min_last_ping is not None and last_ping < min_last_ping:\n        continue\n      elif max_last_ping is not None and last_ping > max_last_ping:\n        continue\n      elif (fleetspeak_enabled is not None and\n            is_fleetspeak_client != fleetspeak_enabled):\n        continue\n      else:\n        last_pings[client_id] = metadata.get(\"ping\", None)\n    return last_pings"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nwriting the full history for a particular client.", "response": "def WriteClientSnapshotHistory(self, clients):\n    \"\"\"Writes the full history for a particular client.\"\"\"\n    if clients[0].client_id not in self.metadatas:\n      raise db.UnknownClientError(clients[0].client_id)\n\n    for client in clients:\n      startup_info = client.startup_info\n      client.startup_info = None\n\n      snapshots = self.clients.setdefault(client.client_id, {})\n      snapshots[client.timestamp] = client.SerializeToString()\n\n      startup_infos = self.startup_history.setdefault(client.client_id, {})\n      startup_infos[client.timestamp] = startup_info.SerializeToString()\n\n      client.startup_info = startup_info"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreading the full history for a particular client.", "response": "def ReadClientSnapshotHistory(self, client_id, timerange=None):\n    \"\"\"Reads the full history for a particular client.\"\"\"\n    from_time, to_time = self._ParseTimeRange(timerange)\n\n    history = self.clients.get(client_id)\n    if not history:\n      return []\n    res = []\n    for ts in sorted(history, reverse=True):\n      if ts < from_time or ts > to_time:\n        continue\n\n      client_obj = rdf_objects.ClientSnapshot.FromSerializedString(history[ts])\n      client_obj.timestamp = ts\n      client_obj.startup_info = rdf_client.StartupInfo.FromSerializedString(\n          self.startup_history[client_id][ts])\n      res.append(client_obj)\n    return res"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nassociate the provided keywords with the client.", "response": "def AddClientKeywords(self, client_id, keywords):\n    \"\"\"Associates the provided keywords with the client.\"\"\"\n    if client_id not in self.metadatas:\n      raise db.UnknownClientError(client_id)\n\n    for kw in keywords:\n      self.keywords.setdefault(kw, {})\n      self.keywords[kw][client_id] = rdfvalue.RDFDatetime.Now()"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef ListClientsForKeywords(self, keywords, start_time=None):\n    res = {kw: [] for kw in keywords}\n    for kw in keywords:\n      for client_id, timestamp in iteritems(self.keywords.get(kw, {})):\n        if start_time is not None and timestamp < start_time:\n          continue\n        res[kw].append(client_id)\n    return res", "response": "Lists the clients associated with keywords."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef RemoveClientKeyword(self, client_id, keyword):\n    if keyword in self.keywords and client_id in self.keywords[keyword]:\n      del self.keywords[keyword][client_id]", "response": "Removes the association of a particular client to a keyword."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nadds a user label to a client.", "response": "def AddClientLabels(self, client_id, owner, labels):\n    \"\"\"Attaches a user label to a client.\"\"\"\n    if client_id not in self.metadatas:\n      raise db.UnknownClientError(client_id)\n\n    labelset = self.labels.setdefault(client_id, {}).setdefault(owner, set())\n    for l in labels:\n      labelset.add(utils.SmartUnicode(l))"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreads the user labels for a list of clients.", "response": "def MultiReadClientLabels(self, client_ids):\n    \"\"\"Reads the user labels for a list of clients.\"\"\"\n    res = {}\n    for client_id in client_ids:\n      res[client_id] = []\n      owner_dict = self.labels.get(client_id, {})\n      for owner, labels in iteritems(owner_dict):\n        for l in labels:\n          res[client_id].append(rdf_objects.ClientLabel(owner=owner, name=l))\n      res[client_id].sort(key=lambda label: (label.owner, label.name))\n    return res"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef RemoveClientLabels(self, client_id, owner, labels):\n    labelset = self.labels.setdefault(client_id, {}).setdefault(owner, set())\n    for l in labels:\n      labelset.discard(utils.SmartUnicode(l))", "response": "Removes a list of user labels from a given client."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nread all client labels known to the system.", "response": "def ReadAllClientLabels(self):\n    \"\"\"Lists all client labels known to the system.\"\"\"\n    result = set()\n    for labels_dict in itervalues(self.labels):\n      for owner, names in iteritems(labels_dict):\n        for name in names:\n          result.add(rdf_objects.ClientLabel(owner=owner, name=name))\n\n    return list(result)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nwriting a new client startup record.", "response": "def WriteClientStartupInfo(self, client_id, startup_info):\n    \"\"\"Writes a new client startup record.\"\"\"\n    if client_id not in self.metadatas:\n      raise db.UnknownClientError(client_id)\n\n    ts = rdfvalue.RDFDatetime.Now()\n    self.metadatas[client_id][\"startup_info_timestamp\"] = ts\n    history = self.startup_history.setdefault(client_id, {})\n    history[ts] = startup_info.SerializeToString()"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef ReadClientStartupInfo(self, client_id):\n    history = self.startup_history.get(client_id, None)\n    if not history:\n      return None\n\n    ts = max(history)\n    res = rdf_client.StartupInfo.FromSerializedString(history[ts])\n    res.timestamp = ts\n    return res", "response": "Reads the latest client startup record for a single client."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef ReadClientStartupInfoHistory(self, client_id, timerange=None):\n    from_time, to_time = self._ParseTimeRange(timerange)\n\n    history = self.startup_history.get(client_id)\n    if not history:\n      return []\n    res = []\n    for ts in sorted(history, reverse=True):\n      if ts < from_time or ts > to_time:\n        continue\n\n      client_data = rdf_client.StartupInfo.FromSerializedString(history[ts])\n      client_data.timestamp = ts\n      res.append(client_data)\n    return res", "response": "Reads the full startup history for a particular client."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef WriteClientCrashInfo(self, client_id, crash_info):\n    if client_id not in self.metadatas:\n      raise db.UnknownClientError(client_id)\n\n    ts = rdfvalue.RDFDatetime.Now()\n    self.metadatas[client_id][\"last_crash_timestamp\"] = ts\n    history = self.crash_history.setdefault(client_id, {})\n    history[ts] = crash_info.SerializeToString()", "response": "Writes a new client crash record."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreads the latest client crash record for a single client.", "response": "def ReadClientCrashInfo(self, client_id):\n    \"\"\"Reads the latest client crash record for a single client.\"\"\"\n    history = self.crash_history.get(client_id, None)\n    if not history:\n      return None\n\n    ts = max(history)\n    res = rdf_client.ClientCrash.FromSerializedString(history[ts])\n    res.timestamp = ts\n    return res"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreading the full crash history for a particular client.", "response": "def ReadClientCrashInfoHistory(self, client_id):\n    \"\"\"Reads the full crash history for a particular client.\"\"\"\n    history = self.crash_history.get(client_id)\n    if not history:\n      return []\n    res = []\n    for ts in sorted(history, reverse=True):\n      client_data = rdf_client.ClientCrash.FromSerializedString(history[ts])\n      client_data.timestamp = ts\n      res.append(client_data)\n    return res"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef WriteClientStats(self, client_id,\n                       stats):\n    \"\"\"Stores a ClientStats instance.\"\"\"\n    if client_id not in self.ReadAllClientIDs():\n      raise db.UnknownClientError(client_id)\n\n    self.client_stats[client_id][rdfvalue.RDFDatetime.Now()] = stats", "response": "Stores a ClientStats instance."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef ReadClientStats(self, client_id,\n                      min_timestamp,\n                      max_timestamp\n                     ):\n    \"\"\"Reads ClientStats for a given client and time range.\"\"\"\n    results = []\n    for timestamp, stats in iteritems(self.client_stats[client_id]):\n      if min_timestamp <= timestamp <= max_timestamp:\n        results.append(stats)\n    return results", "response": "Reads ClientStats for a given client and time range."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ndeletes ClientStats older than a given timestamp.", "response": "def DeleteOldClientStats(self, yield_after_count,\n                           retention_time\n                          ):\n    \"\"\"Deletes ClientStats older than a given timestamp.\"\"\"\n    deleted_count = 0\n    yielded = False\n\n    for stats_dict in itervalues(self.client_stats):\n      for timestamp in list(stats_dict.keys()):\n        if timestamp < retention_time:\n          del stats_dict[timestamp]\n          deleted_count += 1\n\n          if deleted_count >= yield_after_count:\n            yield deleted_count\n            yielded = True\n            deleted_count = 0\n\n    if deleted_count > 0 or not yielded:\n      yield deleted_count"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncomputing client - activity stats for all GRR versions in the DB.", "response": "def CountClientVersionStringsByLabel(self, day_buckets):\n    \"\"\"Computes client-activity stats for all GRR versions in the DB.\"\"\"\n\n    def ExtractVersion(client_info):\n      return client_info.last_snapshot.GetGRRVersionString()\n\n    return self._CountClientStatisticByLabel(day_buckets, ExtractVersion)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef CountClientPlatformsByLabel(self, day_buckets):\n\n    def ExtractPlatform(client_info):\n      return client_info.last_snapshot.knowledge_base.os\n\n    return self._CountClientStatisticByLabel(day_buckets, ExtractPlatform)", "response": "Computes client - activity stats for all client platforms in the DB."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ncompute client - activity stats for all OS - releases in the DB.", "response": "def CountClientPlatformReleasesByLabel(self, day_buckets):\n    \"\"\"Computes client-activity stats for OS-release strings in the DB.\"\"\"\n    return self._CountClientStatisticByLabel(\n        day_buckets, lambda client_info: client_info.last_snapshot.Uname())"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _CountClientStatisticByLabel(self, day_buckets, extract_statistic_fn):\n    counts = collections.defaultdict(int)\n    now = rdfvalue.RDFDatetime.Now()\n    for info in self.IterateAllClientsFullInfo(batch_size=db.MAX_COUNT):\n      if not info.metadata.ping:\n        continue\n      statistic_value = extract_statistic_fn(info)\n      for client_label in info.GetLabelsNames(owner=\"GRR\"):\n        for day_bucket in day_buckets:\n          time_boundary = now - rdfvalue.Duration.FromDays(day_bucket)\n          if info.metadata.ping > time_boundary:\n            # Count the client if it has been active in the last 'day_bucket'\n            # days.\n            counts[(statistic_value, client_label, day_bucket)] += 1\n    return dict(counts)", "response": "Counts the client - activity metrics for a particular statistic."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef CreateServer(frontend=None):\n  max_port = config.CONFIG.Get(\"Frontend.port_max\",\n                               config.CONFIG[\"Frontend.bind_port\"])\n\n  for port in range(config.CONFIG[\"Frontend.bind_port\"], max_port + 1):\n\n    server_address = (config.CONFIG[\"Frontend.bind_address\"], port)\n    try:\n      httpd = GRRHTTPServer(\n          server_address, GRRHTTPServerHandler, frontend=frontend)\n      break\n    except socket.error as e:\n      if e.errno == socket.errno.EADDRINUSE and port < max_port:\n        logging.info(\"Port %s in use, trying %s\", port, port + 1)\n      else:\n        raise\n\n  sa = httpd.socket.getsockname()\n  logging.info(\"Serving HTTP on %s port %d ...\", sa[0], sa[1])\n  return httpd", "response": "Start frontend http server."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nsending a response to the client.", "response": "def Send(self,\n           data,\n           status=200,\n           ctype=\"application/octet-stream\",\n           additional_headers=None,\n           last_modified=0):\n    \"\"\"Sends a response to the client.\"\"\"\n    if additional_headers:\n      additional_header_strings = [\n          \"%s: %s\\r\\n\" % (name, val)\n          for name, val in iteritems(additional_headers)\n      ]\n    else:\n      additional_header_strings = []\n\n    header = \"\"\n    header += \"HTTP/1.0 %s\\r\\n\" % self.statustext[status]\n    header += \"Server: GRR Server\\r\\n\"\n    header += \"Content-type: %s\\r\\n\" % ctype\n    header += \"Content-Length: %d\\r\\n\" % len(data)\n    header += \"Last-Modified: %s\\r\\n\" % self.date_time_string(last_modified)\n    header += \"\".join(additional_header_strings)\n    header += \"\\r\\n\"\n\n    self.wfile.write(header.encode(\"utf-8\"))\n    self.wfile.write(data)"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nserves the server. pem with GET requests.", "response": "def do_GET(self):  # pylint: disable=g-bad-name\n    \"\"\"Serve the server pem with GET requests.\"\"\"\n    self._IncrementActiveCount()\n    try:\n      if self.path.startswith(\"/server.pem\"):\n        stats_collector_instance.Get().IncrementCounter(\n            \"frontend_http_requests\", fields=[\"cert\", \"http\"])\n        self.ServerPem()\n      elif self.path.startswith(self.static_content_path):\n        stats_collector_instance.Get().IncrementCounter(\n            \"frontend_http_requests\", fields=[\"static\", \"http\"])\n        self.ServeStatic(self.path[len(self.static_content_path):])\n    finally:\n      self._DecrementActiveCount()"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _GenerateChunk(self, length):\n\n    while 1:\n      to_read = min(length, self.RECV_BLOCK_SIZE)\n      if to_read == 0:\n        return\n\n      data = self.rfile.read(to_read)\n      if not data:\n        return\n\n      yield data\n      length -= len(data)", "response": "Generates data for a single chunk of data."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ngenerates the file data for a chunk encoded file.", "response": "def GenerateFileData(self):\n    \"\"\"Generates the file data for a chunk encoded file.\"\"\"\n    # Handle chunked encoding:\n    # https://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.6.1\n    while 1:\n      line = self.rfile.readline()\n      # We do not support chunked extensions, just ignore them.\n      chunk_size = int(line.split(\";\")[0], 16)\n      if chunk_size == 0:\n        break\n\n      for chunk in self._GenerateChunk(chunk_size):\n        yield chunk\n\n      # Chunk is followed by \\r\\n.\n      lf = self.rfile.read(2)\n      if lf != \"\\r\\n\":\n        raise IOError(\"Unable to parse chunk.\")\n\n    # Skip entity headers.\n    for header in self.rfile.readline():\n      if not header:\n        break"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef do_POST(self):  # pylint: disable=g-bad-name\n    self._IncrementActiveCount()\n    try:\n      if self.path.startswith(\"/upload\"):\n        stats_collector_instance.Get().IncrementCounter(\n            \"frontend_http_requests\", fields=[\"upload\", \"http\"])\n\n        logging.error(\"Requested no longer supported file upload through HTTP.\")\n        self.Send(\"File upload though HTTP is no longer supported\", status=404)\n      else:\n        stats_collector_instance.Get().IncrementCounter(\n            \"frontend_http_requests\", fields=[\"control\", \"http\"])\n        self.Control()\n\n    except Exception as e:  # pylint: disable=broad-except\n      if flags.FLAGS.pdb_post_mortem:\n        pdb.post_mortem()\n\n      logging.exception(\"Had to respond with status 500.\")\n      self.Send(\"Error: %s\" % e, status=500)\n    finally:\n      self._DecrementActiveCount()", "response": "Process encrypted message bundles."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ninitializes the Fleetspeak connector.", "response": "def Init(service_client=None):\n  \"\"\"Initializes the Fleetspeak connector.\"\"\"\n  global CONN\n  global label_map\n  global unknown_label\n\n  if service_client is None:\n    service_client_cls = fs_client.InsecureGRPCServiceClient\n\n    fleetspeak_message_listen_address = (\n        config.CONFIG[\"Server.fleetspeak_message_listen_address\"] or None)\n    fleetspeak_server = config.CONFIG[\"Server.fleetspeak_server\"] or None\n\n    if fleetspeak_message_listen_address is None and fleetspeak_server is None:\n      logging.warning(\n          \"Missing config options `Server.fleetspeak_message_listen_address', \"\n          \"`Server.fleetspeak_server', at least one of which is required to \"\n          \"initialize a connection to Fleetspeak; Not using Fleetspeak.\")\n      return\n\n    service_client = service_client_cls(\n        \"GRR\",\n        fleetspeak_message_listen_address=fleetspeak_message_listen_address,\n        fleetspeak_server=fleetspeak_server,\n        threadpool_size=50)\n\n  unknown_label = config.CONFIG[\"Server.fleetspeak_unknown_label\"]\n  label_map = {}\n  for entry in config.CONFIG[\"Server.fleetspeak_label_map\"]:\n    key, value = entry.split(\":\")\n    label_map[key] = value\n\n  CONN = service_client\n  logging.info(\"Fleetspeak connector initialized.\")"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nwrites blob references for a signed binary to the DB.", "response": "def WriteSignedBinaryReferences(self,\n                                  binary_id,\n                                  references,\n                                  cursor=None):\n    \"\"\"Writes blob references for a signed binary to the DB.\"\"\"\n    args = {\n        \"binary_type\":\n            binary_id.binary_type.SerializeToDataStore(),\n        \"binary_path\":\n            binary_id.path,\n        \"binary_path_hash\":\n            mysql_utils.Hash(binary_id.path),\n        \"blob_references\":\n            references.SerializeToString()\n    }\n    query = \"\"\"\n      INSERT INTO signed_binary_references {cols}\n      VALUES {vals}\n      ON DUPLICATE KEY UPDATE\n        blob_references = VALUES(blob_references)\n    \"\"\".format(\n        cols=mysql_utils.Columns(args),\n        vals=mysql_utils.NamedPlaceholders(args))\n    cursor.execute(query, args)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreading blob references for the signed binary with the given id.", "response": "def ReadSignedBinaryReferences(\n      self, binary_id,\n      cursor=None):\n    \"\"\"Reads blob references for the signed binary with the given id.\"\"\"\n    cursor.execute(\n        \"\"\"\n      SELECT blob_references, UNIX_TIMESTAMP(timestamp)\n      FROM signed_binary_references\n      WHERE binary_type = %s AND binary_path_hash = %s\n    \"\"\", [\n        binary_id.binary_type.SerializeToDataStore(),\n        mysql_utils.Hash(binary_id.path)\n    ])\n    row = cursor.fetchone()\n\n    if not row:\n      raise db.UnknownSignedBinaryError(binary_id)\n\n    raw_references, timestamp = row\n    # TODO(hanuszczak): pytype does not understand overloads, so we have to cast\n    # to a non-optional object.\n    datetime = cast(rdfvalue.RDFDatetime,\n                    mysql_utils.TimestampToRDFDatetime(timestamp))\n\n    references = rdf_objects.BlobReferences.FromSerializedString(raw_references)\n    return references, datetime"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nread all signed binaries in the DB.", "response": "def ReadIDsForAllSignedBinaries(self, cursor=None\n                                 ):\n    \"\"\"Returns ids for all signed binaries in the DB.\"\"\"\n    cursor.execute(\n        \"SELECT binary_type, binary_path FROM signed_binary_references\")\n    return [\n        rdf_objects.SignedBinaryID(binary_type=binary_type, path=binary_path)\n        for binary_type, binary_path in cursor.fetchall()\n    ]"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef DeleteSignedBinaryReferences(self,\n                                   binary_id,\n                                   cursor=None):\n    \"\"\"Deletes blob references for the given signed binary from the DB.\"\"\"\n    cursor.execute(\n        \"\"\"\n      DELETE FROM signed_binary_references\n      WHERE binary_type = %s AND binary_path_hash = %s\n    \"\"\", [\n        binary_id.binary_type.SerializeToDataStore(),\n        mysql_utils.Hash(binary_id.path)\n    ])", "response": "Deletes blob references for the given signed binary from the DB."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nquery the client for available Volume Shadow Copies using a WMI query.", "response": "def Start(self):\n    \"\"\"Query the client for available Volume Shadow Copies using a WMI query.\"\"\"\n    self.state.shadows = []\n    self.state.raw_device = None\n\n    self.CallClient(\n        server_stubs.WmiQuery,\n        query=\"SELECT * FROM Win32_ShadowCopy\",\n        next_state=\"ListDeviceDirectories\")"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef ProcessListDirectory(self, responses):\n    if not responses.success:\n      raise flow.FlowError(\"Unable to list directory.\")\n\n    with data_store.DB.GetMutationPool() as pool:\n      for response in responses:\n        stat_entry = rdf_client_fs.StatEntry(response)\n        filesystem.CreateAFF4Object(\n            stat_entry, self.client_urn, pool, token=self.token)\n        self.SendReply(stat_entry)", "response": "Processes the results of the ListDirectory client action."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef ParseMultiple(self, stats, file_objects, knowledge_base):\n    _ = knowledge_base\n\n    # Collate files into path: contents dictionary.\n    found_files = self._Combine(stats, file_objects)\n\n    # Determine collected files and apply weighting.\n    weights = [w for w in self.WEIGHTS if w.path in found_files]\n    weights = sorted(weights, key=lambda x: x.weight)\n\n    for _, path, handler in weights:\n      contents = found_files[path]\n      obj = handler(contents)\n\n      complete, result = obj.Parse()\n      if result is None:\n        continue\n      elif complete:\n        yield rdf_protodict.Dict({\n            'os_release': result.release,\n            'os_major_version': result.major,\n            'os_minor_version': result.minor\n        })\n        return\n\n    # Amazon AMIs place release info in /etc/system-release.\n    # https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/amazon-linux-ami-basics.html\n    system_release = found_files.get('/etc/system-release', None)\n    if system_release and 'Amazon Linux' in system_release:\n      match_object = ReleaseFileParseHandler.RH_RE.search(system_release)\n      if match_object and match_object.lastindex > 1:\n        yield rdf_protodict.Dict({\n            'os_release': 'AmazonLinuxAMI',\n            'os_major_version': int(match_object.group(1)),\n            'os_minor_version': int(match_object.group(2))\n        })\n        return\n\n    # Fall back to /etc/os-release.\n    results_dict = self._ParseOSReleaseFile(found_files)\n    if results_dict is not None:\n      yield results_dict\n      return\n\n    # No successful parse.\n    yield rdf_anomaly.Anomaly(\n        type='PARSER_ANOMALY', symptom='Unable to determine distribution.')", "response": "Parse the found release files."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef Verify(self, public_key):\n    # TODO(amoser): We have to do this manually for now since cryptography does\n    # not yet support cert verification. There is PR 2460:\n    # https://github.com/pyca/cryptography/pull/2460/files\n    # that will add it, once it's in we should switch to using this.\n\n    # Note that all times here are in UTC.\n    now = rdfvalue.RDFDatetime.Now().AsDatetime()\n    if now > self._value.not_valid_after:\n      raise VerificationError(\"Certificate expired!\")\n    if now < self._value.not_valid_before:\n      raise VerificationError(\"Certificate not yet valid!\")\n\n    public_key.Verify(\n        self._value.tbs_certificate_bytes,\n        self._value.signature,\n        hash_algorithm=self._value.signature_hash_algorithm)\n    return True", "response": "Verifies the certificate using the given public key."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef ClientCertFromCSR(cls, csr):\n    builder = x509.CertificateBuilder()\n    # Use the client CN for a cert serial_id. This will ensure we do\n    # not have clashing cert id.\n    common_name = csr.GetCN()\n    serial = int(common_name.split(\".\")[1], 16)\n    builder = builder.serial_number(serial)\n    builder = builder.subject_name(\n        x509.Name(\n            [x509.NameAttribute(oid.NameOID.COMMON_NAME, str(common_name))]))\n\n    now = rdfvalue.RDFDatetime.Now()\n    now_plus_year = now + rdfvalue.Duration(\"52w\")\n    builder = builder.not_valid_after(now_plus_year.AsDatetime())\n    now_minus_ten = now - rdfvalue.Duration(\"10s\")\n    builder = builder.not_valid_before(now_minus_ten.AsDatetime())\n    # TODO(user): dependency loop with\n    # grr/core/grr_response_core/config/client.py.\n    # pylint: disable=protected-access\n    ca_cert = config_lib._CONFIG[\"CA.certificate\"]\n    # pylint: enable=protected-access\n    builder = builder.issuer_name(ca_cert.GetIssuer())\n    builder = builder.public_key(csr.GetPublicKey().GetRawPublicKey())\n\n    # TODO(user): dependency loop with\n    # grr/core/grr_response_core/config/client.py.\n    # pylint: disable=protected-access\n    ca_key = config_lib._CONFIG[\"PrivateKeys.ca_key\"]\n    # pylint: enable=protected-access\n\n    return RDFX509Cert(\n        builder.sign(\n            private_key=ca_key.GetRawPrivateKey(),\n            algorithm=hashes.SHA256(),\n            backend=openssl.backend))", "response": "Creates a new client certificate from a certificate signing request."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef Verify(self, message, signature, hash_algorithm=None):\n    # This method accepts both PSS and PKCS1v15 padding. PSS is preferred but\n    # old clients only support PKCS1v15.\n\n    if hash_algorithm is None:\n      hash_algorithm = hashes.SHA256()\n\n    last_e = None\n    for padding_algorithm in [\n        padding.PSS(\n            mgf=padding.MGF1(hash_algorithm),\n            salt_length=padding.PSS.MAX_LENGTH),\n        padding.PKCS1v15()\n    ]:\n      try:\n        self._value.verify(signature, message, padding_algorithm,\n                           hash_algorithm)\n        return True\n\n      except exceptions.InvalidSignature as e:\n        last_e = e\n\n    raise VerificationError(last_e)", "response": "Verifies a given message."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef Sign(self, message, use_pss=False):\n    precondition.AssertType(message, bytes)\n\n    # TODO(amoser): This should use PSS by default at some point.\n    if not use_pss:\n      padding_algorithm = padding.PKCS1v15()\n    else:\n      padding_algorithm = padding.PSS(\n          mgf=padding.MGF1(hashes.SHA256()), salt_length=padding.PSS.MAX_LENGTH)\n\n    return self._value.sign(message, padding_algorithm, hashes.SHA256())", "response": "Sign a given message."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef Verify(self, public_key):\n    if self.digest_type != self.HashType.SHA256:\n      raise rdfvalue.DecodeError(\"Unsupported digest.\")\n    if self.signature_type not in [\n        self.SignatureType.RSA_PKCS1v15, self.SignatureType.RSA_PSS\n    ]:\n      raise rdfvalue.DecodeError(\"Unsupported signature type.\")\n\n    try:\n      public_key.Verify(self.data, self.signature)\n    except InvalidSignature as e:\n      raise rdfvalue.DecodeError(\"Could not verify blob. Error: %s\" % e)\n\n    return True", "response": "Verifies the data in this blob."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nuses the data to sign this blob.", "response": "def Sign(self, data, signing_key, verify_key=None):\n    \"\"\"Use the data to sign this blob.\n\n    Args:\n      data: String containing the blob data.\n      signing_key: The key to sign with.\n      verify_key: Key to verify with. If None we assume the signing key also\n        contains the public key.\n\n    Returns:\n      self for call chaining.\n    \"\"\"\n\n    if signing_key.KeyLen() < 2048:\n      logging.warning(\"signing key is too short.\")\n\n    self.signature = signing_key.Sign(data)\n    self.signature_type = self.SignatureType.RSA_PKCS1v15\n\n    self.digest = hashlib.sha256(data).digest()\n    self.digest_type = self.HashType.SHA256\n    self.data = data\n\n    # Test we can verify before we send it off.\n    if verify_key is None:\n      verify_key = signing_key.GetPublicKey()\n\n    # Verify our own data.\n    self.Verify(verify_key)\n\n    return self"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef Encrypt(self, data):\n    encryptor = self.GetEncryptor()\n    padded_data = self.Pad(data)\n\n    try:\n      return encryptor.update(padded_data) + encryptor.finalize()\n    except ValueError as e:\n      raise CipherError(e)", "response": "A convenience method which pads and encrypts at once."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef Decrypt(self, data):\n    decryptor = self.GetDecryptor()\n\n    try:\n      padded_data = decryptor.update(data) + decryptor.finalize()\n      return self.UnPad(padded_data)\n    except ValueError as e:\n      raise CipherError(e)", "response": "A convenience method which pads and decrypts at once."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef HMAC(self, message, use_sha256=False):\n    h = self._NewHMAC(use_sha256=use_sha256)\n    h.update(message)\n    return h.finalize()", "response": "Calculates the HMAC for a given message."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nverify the signature for a given message.", "response": "def Verify(self, message, signature):\n    \"\"\"Verifies the signature for a given message.\"\"\"\n    siglen = len(signature)\n    if siglen == 20:\n      hash_algorithm = hashes.SHA1()\n    elif siglen == 32:\n      hash_algorithm = hashes.SHA256()\n    else:\n      raise VerificationError(\"Invalid signature length %d.\" % siglen)\n\n    h = hmac.HMAC(self.key, hash_algorithm, backend=openssl.backend)\n    h.update(message)\n    try:\n      h.verify(signature)\n      return True\n    except exceptions.InvalidSignature as e:\n      raise VerificationError(e)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _ItemsToUrns(self, items):\n    for item in items:\n      try:\n        yield flow_export.CollectionItemToAff4Path(item, self.client_id)\n      except flow_export.ItemNotExportableError:\n        pass", "response": "Converts a list of items to aff4 urns suitable for downloading."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef Generate(self, items, token=None):\n    clients = set()\n    for fd_urn_batch in collection.Batch(\n        self._ItemsToUrns(items), self.BATCH_SIZE):\n      self.total_files += len(fd_urn_batch)\n\n      fds_to_write = {}\n      for fd in aff4.FACTORY.MultiOpen(fd_urn_batch, token=token):\n        # Derive a ClientPath from AFF4 URN to make new and old\n        # archive_generator predicate input consistent.\n        # TODO(user): This code is clearly hacky and intended to be removed.\n        urn_components = fd.urn.Split()\n        if urn_components[1:3] != [\"fs\", \"os\"]:\n          raise AssertionError(\"URN components are expected to start with \"\n                               \"client, 'fs', 'os'. Got %r\" % (urn_components,))\n\n        client_path = db.ClientPath.OS(\n            client_id=urn_components[0], components=urn_components[3:])\n\n        if not self.predicate(client_path):\n          self.ignored_files.add(utils.SmartUnicode(fd.urn))\n          continue\n\n        # Any file-like object with data in AFF4 should inherit AFF4Stream.\n        if isinstance(fd, aff4.AFF4Stream):\n          urn_components = fd.urn.Split()\n          clients.add(rdf_client.ClientURN(urn_components[0]))\n\n          content_path = os.path.join(self.prefix, *urn_components)\n\n          # Make sure size of the original file is passed. It's required\n          # when output_writer is StreamingTarWriter.\n          st = os.stat_result((0o644, 0, 0, 0, 0, 0, fd.size, 0, 0, 0))\n          fds_to_write[fd] = (content_path, st)\n\n      if fds_to_write:\n        prev_fd = None\n        for fd, chunk, exception in aff4.AFF4Stream.MultiStream(fds_to_write):\n          if exception:\n            logging.exception(exception)\n\n            try:\n              self.archived_files.remove(utils.SmartUnicode(fd.urn))\n            except KeyError:\n              pass  # Failing is fine, since removal should be idempotent.\n            self.failed_files.add(utils.SmartUnicode(fd.urn))\n            continue\n\n          if prev_fd != fd:\n            if prev_fd:\n              yield self.archive_generator.WriteFileFooter()\n            prev_fd = fd\n\n            content_path, st = fds_to_write[fd]\n            yield self.archive_generator.WriteFileHeader(content_path, st=st)\n\n          yield self.archive_generator.WriteFileChunk(chunk)\n          self.archived_files.add(utils.SmartUnicode(fd.urn))\n\n        if self.archive_generator.is_file_write_in_progress:\n          yield self.archive_generator.WriteFileFooter()\n\n    if clients:\n      for client_urn_batch in collection.Batch(clients, self.BATCH_SIZE):\n        for fd in aff4.FACTORY.MultiOpen(\n            client_urn_batch, aff4_type=aff4_grr.VFSGRRClient, token=token):\n          for chunk in self._GenerateClientInfo(fd):\n            yield chunk\n\n    for chunk in self._GenerateDescription():\n      yield chunk\n\n    yield self.archive_generator.Close()", "response": "Generates a new archive from a given collection."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nhandle the GetArtifacts command.", "response": "def Handle(self, args, token=None):\n    \"\"\"Get available artifact information for rendering.\"\"\"\n\n    # Get all artifacts that aren't Bootstrap and aren't the base class.\n    artifacts_list = sorted(\n        artifact_registry.REGISTRY.GetArtifacts(\n            reload_datastore_artifacts=True),\n        key=lambda art: art.name)\n\n    total_count = len(artifacts_list)\n\n    if args.count:\n      artifacts_list = artifacts_list[args.offset:args.offset + args.count]\n    else:\n      artifacts_list = artifacts_list[args.offset:]\n\n    descriptors = self.BuildArtifactDescriptors(artifacts_list)\n    return ApiListArtifactsResult(items=descriptors, total_count=total_count)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef Insert(self, index, rdfpathspec=None, **kwarg):\n    if rdfpathspec is None:\n      rdfpathspec = self.__class__(**kwarg)\n\n    if index == 0:\n      # Copy ourselves to a temp copy.\n      nested_proto = self.__class__()\n      nested_proto.SetRawData(self.GetRawData())\n\n      # Replace ourselves with the new object.\n      self.SetRawData(rdfpathspec.GetRawData())\n\n      # Append the temp copy to the end.\n      self.last.nested_path = nested_proto\n    else:\n      previous = self[index - 1]\n      rdfpathspec.last.nested_path = previous.nested_path\n      previous.nested_path = rdfpathspec", "response": "Insert a single component at index."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nappend a new pathspec component to this pathspec.", "response": "def Append(self, component=None, **kwarg):\n    \"\"\"Append a new pathspec component to this pathspec.\"\"\"\n    if component is None:\n      component = self.__class__(**kwarg)\n\n    if self.HasField(\"pathtype\"):\n      self.last.nested_path = component\n    else:\n      for k, v in iteritems(kwarg):\n        setattr(self, k, v)\n\n      self.SetRawData(component.GetRawData())\n\n    return self"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nremoving and returns the pathspec at the specified index.", "response": "def Pop(self, index=0):\n    \"\"\"Removes and returns the pathspec at the specified index.\"\"\"\n    if index < 0:\n      index += len(self)\n\n    if index == 0:\n      result = self.__class__()\n      result.SetRawData(self.GetRawData())\n\n      self.SetRawData(self.nested_path.GetRawData())\n\n    else:\n      # Get the raw protobufs for the previous member.\n      previous = self[index - 1]\n\n      result = previous.nested_path\n\n      # Manipulate the previous members protobuf to patch the next component in.\n      previous.nested_path = result.nested_path\n\n    result.nested_path = None\n\n    return result"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef Dirname(self):\n    result = self.Copy()\n\n    while 1:\n      last_directory = posixpath.dirname(result.last.path)\n      if last_directory != \"/\" or len(result) <= 1:\n        result.last.path = last_directory\n        # Make sure to clear the inode information.\n        result.last.inode = None\n\n        break\n\n      result.Pop(-1)\n\n    return result", "response": "Get a new object with only the directory path."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn the AFF4 URN that corresponds to this pathspec.", "response": "def AFF4Path(self, client_urn):\n    \"\"\"Returns the AFF4 URN this pathspec will be stored under.\n\n    Args:\n      client_urn: A ClientURN.\n\n    Returns:\n      A urn that corresponds to this pathspec.\n\n    Raises:\n      ValueError: If pathspec is not of the correct type.\n    \"\"\"\n    # If the first level is OS and the second level is TSK its probably a mount\n    # point resolution. We map it into the tsk branch. For example if we get:\n    # path: \\\\\\\\.\\\\Volume{1234}\\\\\n    # pathtype: OS\n    # mount_point: /c:/\n    # nested_path {\n    #    path: /windows/\n    #    pathtype: TSK\n    # }\n    # We map this to aff4://client_id/fs/tsk/\\\\\\\\.\\\\Volume{1234}\\\\/windows/\n\n    if not self.HasField(\"pathtype\"):\n      raise ValueError(\"Can't determine AFF4 path without a valid pathtype.\")\n\n    first_component = self[0]\n    dev = first_component.path\n    if first_component.HasField(\"offset\"):\n      # We divide here just to get prettier numbers in the GUI\n      dev += \":{}\".format(first_component.offset // 512)\n\n    if (len(self) > 1 and first_component.pathtype == PathSpec.PathType.OS and\n        self[1].pathtype == PathSpec.PathType.TSK):\n      result = [self.AFF4_PREFIXES[PathSpec.PathType.TSK], dev]\n\n      # Skip the top level pathspec.\n      start = 1\n    else:\n      # For now just map the top level prefix based on the first pathtype\n      result = [self.AFF4_PREFIXES[first_component.pathtype]]\n      start = 0\n\n    for p in self[start]:\n      component = p.path\n\n      # The following encode different pathspec properties into the AFF4 path in\n      # such a way that unique files on the client are mapped to unique URNs in\n      # the AFF4 space. Note that this transformation does not need to be\n      # reversible since we always use the PathSpec when accessing files on the\n      # client.\n      if p.HasField(\"offset\"):\n        component += \":{}\".format(p.offset // 512)\n\n      # Support ADS names.\n      if p.HasField(\"stream_name\"):\n        component += \":\" + p.stream_name\n\n      result.append(component)\n\n    return client_urn.Add(\"/\".join(result))"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nvalidate that the glob expression is valid.", "response": "def Validate(self):\n    \"\"\"GlobExpression is valid.\"\"\"\n    if len(self.RECURSION_REGEX.findall(self._value)) > 1:\n      raise ValueError(\"Only one ** is permitted per path: %s.\" % self._value)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ninterpolating inline globbing groups.", "response": "def InterpolateGrouping(self, pattern):\n    \"\"\"Interpolate inline globbing groups.\"\"\"\n    components = []\n    offset = 0\n    for match in GROUPING_PATTERN.finditer(pattern):\n      components.append([pattern[offset:match.start()]])\n\n      # Expand the attribute into the set of possibilities:\n      alternatives = match.group(1).split(\",\")\n      components.append(set(alternatives))\n      offset = match.end()\n\n    components.append([pattern[offset:]])\n    # Now calculate the cartesian products of all these sets to form all\n    # strings.\n    for vector in itertools.product(*components):\n      yield u\"\".join(vector)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef AsRegEx(self):\n    parts = self.__class__.REGEX_SPLIT_PATTERN.split(self._value)\n    result = u\"\".join(self._ReplaceRegExPart(p) for p in parts)\n\n    return rdf_standard.RegularExpression(u\"(?i)\\\\A%s\\\\Z\" % result)", "response": "Return the current glob as a regular expression."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef dateadd(value: fields.DateTime(),\n            addend: fields.Int(validate=Range(min=1)),\n            unit: fields.Str(validate=OneOf(['minutes', 'days']))='days'):\n    \"\"\"Add a value to a date.\"\"\"\n    value = value or dt.datetime.utcnow()\n    if unit == 'minutes':\n        delta = dt.timedelta(minutes=addend)\n    else:\n        delta = dt.timedelta(days=addend)\n    result = value + delta\n    return {'result': result}", "response": "Adds a value to a date."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef arguments(function, extra_arguments=0):\n    if not hasattr(function, '__code__'):\n        return ()\n\n    return function.__code__.co_varnames[:function.__code__.co_argcount + extra_arguments]", "response": "Returns the name of all arguments a function takes"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef authenticator(function, challenges=()):\n    challenges = challenges or ('{} realm=\"simple\"'.format(function.__name__), )\n\n    def wrapper(verify_user):\n        def authenticate(request, response, **kwargs):\n            result = function(request, response, verify_user, **kwargs)\n\n            def authenticator_name():\n                try:\n                    return function.__doc__.splitlines()[0]\n                except AttributeError:\n                    return function.__name__\n\n            if result is None:\n                raise HTTPUnauthorized('Authentication Required',\n                                       'Please provide valid {0} credentials'.format(authenticator_name()),\n                                       challenges=challenges)\n\n            if result is False:\n                raise HTTPUnauthorized('Invalid Authentication',\n                                       'Provided {0} credentials were invalid'.format(authenticator_name()),\n                                       challenges=challenges)\n\n            request.context['user'] = result\n            return True\n\n        authenticate.__doc__ = function.__doc__\n        return authenticate\n\n    return wrapper", "response": "Decorator that wraps the authentication logic in a function that accepts an API key and returns a user object to be used in the request context."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef token(request, response, verify_user, context=None, **kwargs):\n    token = request.get_header('Authorization')\n    if token:\n        try:\n            verified_token = verify_user(token)\n        except TypeError:\n            verified_token = verify_user(token, context)\n        if verified_token:\n            return verified_token\n        else:\n            return False\n    return None", "response": "Token verification\n    Checks for the Authorization header and verifies using the verify_user function"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning a verification callback that verifies that the users and password match that provided", "response": "def verify(user, password):\n    \"\"\"Returns a simple verification callback that simply verifies that the users and password match that provided\"\"\"\n    def verify_user(user_name, user_password):\n        if user_name == user and user_password == password:\n            return user_name\n        return False\n    return verify_user"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef process_request(self, request, response):\n        sid = request.cookies.get(self.cookie_name, None)\n        data = {}\n        if sid is not None:\n            if self.store.exists(sid):\n                data = self.store.get(sid)\n        request.context.update({self.context_name: data})", "response": "Get session ID from cookie, load corresponding session data from coupled store and inject session data into\n            the request context."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef process_response(self, request, response, resource, req_succeeded):\n        sid = request.cookies.get(self.cookie_name, None)\n        if sid is None or not self.store.exists(sid):\n            sid = self.generate_sid()\n\n        self.store.set(sid, request.context.get(self.context_name, {}))\n        response.set_cookie(self.cookie_name, sid, expires=self.cookie_expires, max_age=self.cookie_max_age,\n                            domain=self.cookie_domain, path=self.cookie_path, secure=self.cookie_secure,\n                            http_only=self.cookie_http_only)", "response": "Save request context in coupled store object. Set cookie containing a session ID."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _generate_combined_log(self, request, response):\n        current_time = datetime.utcnow()\n        data_len = '-' if response.data is None else len(response.data)\n        return '{0} - - [{1}] {2} {3} {4} {5} {6}'.format(request.remote_addr, current_time, request.method,\n                                                        request.relative_uri, response.status,\n                                                        data_len, request.user_agent)", "response": "Generate a combined log format similar to the NGINX combined style."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef process_request(self, request, response):\n        self.logger.info('Requested: {0} {1} {2}'.format(request.method, request.relative_uri, request.content_type))", "response": "Logs the basic endpoint requested"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef process_response(self, request, response, resource, req_succeeded):\n        self.logger.info(self._generate_combined_log(request, response))", "response": "Logs the basic data returned by the API"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef match_route(self, reqpath):\n        route_dicts = [routes for _, routes in self.api.http.routes.items()][0]\n        routes = [route for route, _ in route_dicts.items()]\n        if reqpath not in routes:\n            for route in routes:  # replace params in route with regex\n                reqpath = re.sub('^(/v\\d*/?)', '/', reqpath)\n                base_url = getattr(self.api.http, 'base_url', '')\n                reqpath = reqpath.replace(base_url, '', 1) if base_url else reqpath\n                if re.match(re.sub(r'/{[^{}]+}', r'/[\\\\w-]+', route) + '$', reqpath):\n                    return route\n\n        return reqpath", "response": "Match a request with parameter to it s corresponding route"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nadds CORS headers to the response", "response": "def process_response(self, request, response, resource, req_succeeded):\n        \"\"\"Add CORS headers to the response\"\"\"\n        response.set_header('Access-Control-Allow-Credentials', str(self.allow_credentials).lower())\n\n        origin = request.get_header('ORIGIN')\n        if origin and (origin in self.allow_origins) or ('*' in self.allow_origins):\n            response.set_header('Access-Control-Allow-Origin', origin)\n\n        if request.method == 'OPTIONS': # check if we are handling a preflight request\n            allowed_methods = set(\n                method\n                for _, routes in self.api.http.routes.items()\n                for method, _ in routes[self.match_route(request.path)].items()\n            )\n            allowed_methods.add('OPTIONS')\n\n            # return allowed methods\n            response.set_header('Access-Control-Allow-Methods', ', '.join(allowed_methods))\n            response.set_header('Allow', ', '.join(allowed_methods))\n\n            # get all requested headers and echo them back\n            requested_headers = request.get_header('Access-Control-Request-Headers')\n            response.set_header('Access-Control-Allow-Headers', requested_headers or '')\n\n            # return valid caching time\n            if self.max_age:\n                response.set_header('Access-Control-Max-Age', self.max_age)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncreating a new type handler with the specified type - casting handler.", "response": "def create(doc=None, error_text=None, exception_handlers=empty.dict, extend=Type, chain=True, auto_instance=True,\n           accept_context=False):\n    \"\"\"Creates a new type handler with the specified type-casting handler\"\"\"\n    extend = extend if type(extend) == type else type(extend)\n\n    def new_type_handler(function):\n        class NewType(extend):\n            __slots__ = ()\n            _accept_context = accept_context\n\n            if chain and extend != Type:\n                if error_text or exception_handlers:\n                    if not accept_context:\n                        def __call__(self, value):\n                            try:\n                                value = super(NewType, self).__call__(value)\n                                return function(value)\n                            except Exception as exception:\n                                for take_exception, rewrite in exception_handlers.items():\n                                    if isinstance(exception, take_exception):\n                                        if isinstance(rewrite, str):\n                                            raise ValueError(rewrite)\n                                        else:\n                                            raise rewrite(value)\n                                if error_text:\n                                    raise ValueError(error_text)\n                                raise exception\n                    else:\n                        if extend._accept_context:\n                            def __call__(self, value, context):\n                                try:\n                                    value = super(NewType, self).__call__(value, context)\n                                    return function(value, context)\n                                except Exception as exception:\n                                    for take_exception, rewrite in exception_handlers.items():\n                                        if isinstance(exception, take_exception):\n                                            if isinstance(rewrite, str):\n                                                raise ValueError(rewrite)\n                                            else:\n                                                raise rewrite(value)\n                                    if error_text:\n                                        raise ValueError(error_text)\n                                    raise exception\n                        else:\n                            def __call__(self, value, context):\n                                try:\n                                    value = super(NewType, self).__call__(value)\n                                    return function(value, context)\n                                except Exception as exception:\n                                    for take_exception, rewrite in exception_handlers.items():\n                                        if isinstance(exception, take_exception):\n                                            if isinstance(rewrite, str):\n                                                raise ValueError(rewrite)\n                                            else:\n                                                raise rewrite(value)\n                                    if error_text:\n                                        raise ValueError(error_text)\n                                    raise exception\n                else:\n                    if not accept_context:\n                        def __call__(self, value):\n                            value = super(NewType, self).__call__(value)\n                            return function(value)\n                    else:\n                        if extend._accept_context:\n                            def __call__(self, value, context):\n                                value = super(NewType, self).__call__(value, context)\n                                return function(value, context)\n                        else:\n                            def __call__(self, value, context):\n                                value = super(NewType, self).__call__(value)\n                                return function(value, context)\n            else:\n                if not accept_context:\n                    if error_text or exception_handlers:\n                        def __call__(self, value):\n                            try:\n                                return function(value)\n                            except Exception as exception:\n                                for take_exception, rewrite in exception_handlers.items():\n                                    if isinstance(exception, take_exception):\n                                        if isinstance(rewrite, str):\n                                            raise ValueError(rewrite)\n                                        else:\n                                            raise rewrite(value)\n                                if error_text:\n                                    raise ValueError(error_text)\n                                raise exception\n                    else:\n                        def __call__(self, value):\n                            return function(value)\n                else:\n                    if error_text or exception_handlers:\n                        def __call__(self, value, context):\n                            try:\n                                return function(value, context)\n                            except Exception as exception:\n                                for take_exception, rewrite in exception_handlers.items():\n                                    if isinstance(exception, take_exception):\n                                        if isinstance(rewrite, str):\n                                            raise ValueError(rewrite)\n                                        else:\n                                            raise rewrite(value)\n                                if error_text:\n                                    raise ValueError(error_text)\n                                raise exception\n                    else:\n                        def __call__(self, value, context):\n                            return function(value, context)\n\n        NewType.__doc__ = function.__doc__ if doc is None else doc\n        if auto_instance and not (introspect.arguments(NewType.__init__, -1) or\n                                  introspect.takes_kwargs(NewType.__init__) or\n                                  introspect.takes_args(NewType.__init__)):\n            return NewType()\n        return NewType\n\n    return new_type_handler"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef accept(kind, doc=None, error_text=None, exception_handlers=empty.dict, accept_context=False):\n    return create(\n        doc,\n        error_text,\n        exception_handlers=exception_handlers,\n        chain=False,\n        accept_context=accept_context\n    )(kind)", "response": "Allows quick wrapping of any Python type cast function for use as a hug type annotation"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef to(location, code=falcon.HTTP_302):\n    raise falcon.http_status.HTTPStatus(code, {'location': location})", "response": "Redirects to the specified location"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nconverts text that may be camelcased into an underscored format", "response": "def underscore(text):\n    \"\"\"Converts text that may be camelcased into an underscored format\"\"\"\n    return UNDERSCORE[1].sub(r'\\1_\\2', UNDERSCORE[0].sub(r'\\1_\\2', text)).lower()"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nhugs API Development Server", "response": "def hug(file: 'A Python file that contains a Hug API'=None, module: 'A Python module that contains a Hug API'=None,\n        host: 'Interface to bind to'='', port: number=8000, no_404_documentation: boolean=False,\n        manual_reload: boolean=False, interval: number=1,\n        command: 'Run a command defined in the given module'=None,\n        silent: boolean=False):\n    \"\"\"Hug API Development Server\"\"\"\n    api_module = None\n    if file and module:\n        print(\"Error: can not define both a file and module source for Hug API.\")\n        sys.exit(1)\n    if file:\n        sys.path.append(os.path.dirname(os.path.abspath(file)))\n        sys.path.append(os.getcwd())\n        api_module = importlib.machinery.SourceFileLoader(file.split(\".\")[0], file).load_module()\n    elif module:\n        sys.path.append(os.getcwd())\n        api_module = importlib.import_module(module)\n    if not api_module or not hasattr(api_module, '__hug__'):\n        print(\"Error: must define a file name or module that contains a Hug API.\")\n        sys.exit(1)\n\n    api = API(api_module, display_intro=not silent)\n    if command:\n        if command not in api.cli.commands:\n            print(str(api.cli))\n            sys.exit(1)\n\n        sys.argv[1:] = sys.argv[(sys.argv.index('-c') if '-c' in sys.argv else sys.argv.index('--command')) + 2:]\n        api.cli.commands[command]()\n        return\n\n    ran = False\n    if not manual_reload:\n        thread.start_new_thread(reload_checker, (interval, ))\n        while True:\n            reload_checker.reloading = False\n            time.sleep(1)\n            try:\n                _start_api(api_module, host, port, no_404_documentation, False if silent else not ran)\n            except KeyboardInterrupt:\n                if not reload_checker.reloading:\n                    sys.exit(1)\n                reload_checker.reloading = False\n                ran = True\n                for name in list(sys.modules.keys()):\n                    if name not in INIT_MODULES:\n                        del(sys.modules[name])\n                if file:\n                    api_module = importlib.machinery.SourceFileLoader(file.split(\".\")[0],\n                                                                        file).load_module()\n                elif module:\n                    api_module = importlib.import_module(module)\n    else:\n        _start_api(api_module, host, port, no_404_documentation, not ran)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef token_gen_call(username, password):\n    secret_key = 'super-secret-key-please-change'\n    mockusername = 'User2'\n    mockpassword = 'Mypassword'\n    if mockpassword == password and mockusername == username: # This is an example. Don't do that.\n        return {\"token\" : jwt.encode({'user': username, 'data': 'mydata'}, secret_key, algorithm='HS256')}\n    return 'Invalid username and/or password for user: {0}'.format(username)", "response": "Authenticate and return a token"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns documentation for the current api", "response": "def documentation(default=None, api_version=None, api=None, **kwargs):\n    \"\"\"returns documentation for the current api\"\"\"\n    api_version = default or api_version\n    if api:\n        return api.http.documentation(base_url=\"\", api_version=api_version)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef json(body, charset='utf-8', **kwargs):\n    return json_converter.loads(text(body, charset=charset))", "response": "Takes JSON formatted data converting it into native Python objects"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nconverts a URL - encoded string into a native Python object.", "response": "def urlencoded(body, charset='ascii', **kwargs):\n    \"\"\"Converts query strings into native Python objects\"\"\"\n    return parse_query_string(text(body, charset=charset), False)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef multipart(body, content_length=0, **header_params):\n    header_params.setdefault('CONTENT-LENGTH', content_length)\n    if header_params and 'boundary' in header_params:\n        if type(header_params['boundary']) is str:\n            header_params['boundary'] = header_params['boundary'].encode()\n\n    form = parse_multipart((body.stream if hasattr(body, 'stream') else body), header_params)\n    for key, value in form.items():\n        if type(value) is list and len(value) is 1:\n            form[key] = value[0]\n    return form", "response": "Converts multipart form data into native Python objects"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nsets up and fills test directory for serving.", "response": "def setup(api=None):\n    \"\"\"Sets up and fills test directory for serving.\n\n    Using different filetypes to see how they are dealt with.\n    The tempoary directory will clean itself up.\n    \"\"\"\n    global tmp_dir_object\n\n    tmp_dir_object = tempfile.TemporaryDirectory()\n\n    dir_name = tmp_dir_object.name\n\n    dir_a = os.path.join(dir_name, \"a\")\n    os.mkdir(dir_a)\n    dir_b = os.path.join(dir_name, \"b\")\n    os.mkdir(dir_b)\n\n    # populate directory a with text files\n    file_list = [\n            [\"hi.txt\", \"\"\"Hi World!\"\"\"],\n            [\"hi.html\", \"\"\"<strong>Hi World!</strong>\"\"\"],\n            [\"hello.html\", \"\"\"\n                <img src='/static/b/smile.png'</img>\n                pop-up\n                <script src='/static/a/hi.js'></script>\"\"\"],\n            [\"hi.js\", \"\"\"alert('Hi World')\"\"\" ]\n    ]\n\n    for f in file_list:\n        with open(os.path.join(dir_a, f[0]), mode=\"wt\") as fo:\n            fo.write(f[1])\n\n    # populate directory b with binary file\n    image = b'\\x89PNG\\r\\n\\x1a\\n\\x00\\x00\\x00\\rIHDR\\x00\\x00\\x00\\n\\x00\\x00\\x00\\n\\x08\\x02\\x00\\x00\\x00\\x02PX\\xea\\x00\\x00\\x006IDAT\\x18\\xd3c\\xfc\\xff\\xff?\\x03n\\xc0\\xc4\\x80\\x170100022222\\xc2\\x85\\x90\\xb9\\x04t3\\x92`7\\xb2\\x15D\\xeb\\xc6\\xe34\\xa8n4c\\xe1F\\x120\\x1c\\x00\\xc6z\\x12\\x1c\\x8cT\\xf2\\x1e\\x00\\x00\\x00\\x00IEND\\xaeB`\\x82'\n\n    with open(os.path.join(dir_b, \"smile.png\"), mode=\"wb\") as fo:\n            fo.write(image)"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns a transformer depending on the content type passed in.", "response": "def content_type(transformers, default=None):\n    \"\"\"Returns a different transformer depending on the content type passed in.\n       If none match and no default is given no transformation takes place.\n\n       should pass in a dict with the following format:\n\n            {'[content-type]': transformation_action,\n             ...\n            }\n    \"\"\"\n    transformers = {content_type: auto_kwargs(transformer) if transformer else transformer\n                    for content_type, transformer in transformers.items()}\n    default = default and auto_kwargs(default)\n\n    def transform(data, request):\n        transformer = transformers.get(request.content_type.split(';')[0], default)\n        if not transformer:\n            return data\n\n        return transformer(data)\n    return transform"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns a transformer depending on the suffix at the end of the requested URL.", "response": "def suffix(transformers, default=None):\n    \"\"\"Returns a different transformer depending on the suffix at the end of the requested URL.\n       If none match and no default is given no transformation takes place.\n\n       should pass in a dict with the following format:\n\n            {'[suffix]': transformation_action,\n             ...\n            }\n    \"\"\"\n    transformers = {suffix: auto_kwargs(transformer) if transformer\n                    else transformer for suffix, transformer in transformers.items()}\n    default = default and auto_kwargs(default)\n\n    def transform(data, request):\n        path = request.path\n        transformer = default\n        for suffix_test, suffix_transformer in transformers.items():\n            if path.endswith(suffix_test):\n                transformer = suffix_transformer\n                break\n\n        return transformer(data) if transformer else data\n    return transform"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef prefix(transformers, default=None):\n    transformers = {prefix: auto_kwargs(transformer) if transformer else transformer\n                    for prefix, transformer in transformers.items()}\n    default = default and auto_kwargs(default)\n\n    def transform(data, request=None, response=None):\n        path = request.path\n        transformer = default\n        for prefix_test, prefix_transformer in transformers.items():\n            if path.startswith(prefix_test):\n                transformer = prefix_transformer\n                break\n\n        return transformer(data) if transformer else data\n    return transform", "response": "Returns a different transformer depending on the prefix at the end of the requested URL."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef all(*transformers):\n    transformers = tuple(auto_kwargs(transformer) for transformer in transformers)\n\n    def transform(data, request=None, response=None):\n        for transformer in transformers:\n            data = transformer(data, request=request, response=response)\n\n        return data\n    return transform", "response": "Returns the results of applying all passed in transformers to data\nOrderItem should pass in list of transformers\nAppId"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef validate(self, input_parameters, context):\n        errors = {}\n\n        for key, type_handler in self.input_transformations.items():\n            if self.raise_on_invalid:\n                if key in input_parameters:\n                    input_parameters[key] = self.initialize_handler(\n                        type_handler,\n                        input_parameters[key],\n                        context=context\n                    )\n            else:\n                try:\n                    if key in input_parameters:\n                        input_parameters[key] = self.initialize_handler(\n                            type_handler,\n                            input_parameters[key],\n                            context=context\n                        )\n                except InvalidTypeData as error:\n                    errors[key] = error.reasons or str(error.message)\n                except Exception as error:\n                    if hasattr(error, 'args') and error.args:\n                        errors[key] = error.args[0]\n                    else:\n                        errors[key] = str(error)\n        for require in self.required:\n            if not require in input_parameters:\n                errors[require] = \"Required parameter '{}' not supplied\".format(require)\n        if not errors and getattr(self, 'validate_function', False):\n            errors = self.validate_function(input_parameters)\n        return errors", "response": "Runs all input transformers and validators against the provided input parameters and returns any errors that are found."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef check_requirements(self, request=None, response=None, context=None):\n        for requirement in self.requires:\n            conclusion = requirement(response=response, request=request, context=context, module=self.api.module)\n            if conclusion and conclusion is not True:\n                return conclusion", "response": "Checks to see if all requirements set pass\n\n          "}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nproduces general documentation for the interface", "response": "def documentation(self, add_to=None):\n        \"\"\"Produces general documentation for the interface\"\"\"\n        doc = OrderedDict if add_to is None else add_to\n\n        usage = self.interface.spec.__doc__\n        if usage:\n            doc['usage'] = usage\n        if getattr(self, 'requires', None):\n            doc['requires'] = [getattr(requirement, '__doc__', requirement.__name__) for requirement in self.requires]\n        doc['outputs'] = OrderedDict()\n        doc['outputs']['format'] = self.outputs.__doc__\n        doc['outputs']['content_type'] = self.outputs.content_type\n        parameters = [param for param in self.parameters if not param in ('request', 'response', 'self')\n                                                        and not param in ('api_version', 'body')\n                                                        and not param.startswith('hug_')\n                                                        and not hasattr(param, 'directive')]\n        if parameters:\n            inputs = doc.setdefault('inputs', OrderedDict())\n            types = self.interface.spec.__annotations__\n            for argument in parameters:\n                kind = types.get(argument, text)\n                if getattr(kind, 'directive', None) is True:\n                    continue\n\n                input_definition = inputs.setdefault(argument, OrderedDict())\n                input_definition['type'] = kind if isinstance(kind, str) else kind.__doc__\n                default = self.defaults.get(argument, None)\n                if default is not None:\n                    input_definition['default'] = default\n\n        return doc"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\noutput the provided data using the transformations and output format specified for this CLI endpoint", "response": "def output(self, data, context):\n        \"\"\"Outputs the provided data using the transformations and output format specified for this CLI endpoint\"\"\"\n        if self.transform:\n            if hasattr(self.transform, 'context'):\n                self.transform.context = context\n            data = self.transform(data)\n        if hasattr(data, 'read'):\n            data = data.read().decode('utf8')\n        if data is not None:\n            data = self.outputs(data)\n            if data:\n                sys.stdout.buffer.write(data)\n                if not data.endswith(b'\\n'):\n                    sys.stdout.buffer.write(b'\\n')\n        return data"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef gather_parameters(self, request, response, context, api_version=None, **input_parameters):\n        input_parameters.update(request.params)\n\n        if self.parse_body and request.content_length:\n            body = request.stream\n            content_type, content_params = parse_content_type(request.content_type)\n            body_formatter = body and self.inputs.get(content_type, self.api.http.input_format(content_type))\n            if body_formatter:\n                body = body_formatter(body, content_length=request.content_length, **content_params)\n            if 'body' in self.all_parameters:\n                input_parameters['body'] = body\n            if isinstance(body, dict):\n                input_parameters.update(body)\n        elif 'body' in self.all_parameters:\n            input_parameters['body'] = None\n\n        if 'request' in self.all_parameters:\n            input_parameters['request'] = request\n        if 'response' in self.all_parameters:\n            input_parameters['response'] = response\n        if 'api_version' in self.all_parameters:\n            input_parameters['api_version'] = api_version\n        for parameter, directive in self.directives.items():\n            arguments = (self.defaults[parameter], ) if parameter in self.defaults else ()\n            input_parameters[parameter] = directive(*arguments, response=response, request=request,\n                                                    api=self.api, api_version=api_version, context=context,\n                                                    interface=self)\n        return input_parameters", "response": "Gathers and returns all parameters that will be used for this endpoint"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef transform_data(self, data, request=None, response=None, context=None):\n        transform = self.transform\n        if hasattr(transform, 'context'):\n            self.transform.context = context\n        \"\"\"Runs the transforms specified on this endpoint with the provided data, returning the data modified\"\"\"\n        if transform and not (isinstance(transform, type) and isinstance(data, transform)):\n            if self._params_for_transform:\n                return transform(data, **self._arguments(self._params_for_transform, request, response))\n            else:\n                return transform(data)\n        return data", "response": "Runs the transforms specified on this endpoint with the provided data returning the modified data"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef content_type(self, request=None, response=None):\n        if callable(self.outputs.content_type):\n            return self.outputs.content_type(request=request, response=response)\n        else:\n            return self.outputs.content_type", "response": "Returns the content type that should be used by default for this endpoint"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef invalid_content_type(self, request=None, response=None):\n        if callable(self.invalid_outputs.content_type):\n            return self.invalid_outputs.content_type(request=request, response=response)\n        else:\n            return self.invalid_outputs.content_type", "response": "Returns the content type that should be used by default on validation errors"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef set_response_defaults(self, response, request=None):\n        for header_name, header_value in self.response_headers:\n            response.set_header(header_name, header_value)\n        if self.set_status:\n            response.status = self.set_status\n        response.content_type = self.content_type(request, response)", "response": "Sets up the response defaults that are defined in the URL route"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn the documentation specific to an HTTP interface", "response": "def documentation(self, add_to=None, version=None, prefix=\"\", base_url=\"\", url=\"\"):\n        \"\"\"Returns the documentation specific to an HTTP interface\"\"\"\n        doc = OrderedDict() if add_to is None else add_to\n\n        usage = self.interface.spec.__doc__\n        if usage:\n            doc['usage'] = usage\n\n        for example in self.examples:\n            example_text = \"{0}{1}{2}{3}\".format(prefix, base_url, '/v{0}'.format(version) if version else '', url)\n            if isinstance(example, str):\n                example_text += \"?{0}\".format(example)\n            doc_examples = doc.setdefault('examples', [])\n            if not example_text in doc_examples:\n                doc_examples.append(example_text)\n\n        doc = super().documentation(doc)\n\n        if getattr(self, 'output_doc', ''):\n            doc['outputs']['type'] = self.output_doc\n\n        return doc"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn all URLS that are mapped to this interface", "response": "def urls(self, version=None):\n        \"\"\"Returns all URLS that are mapped to this interface\"\"\"\n        urls = []\n        for base_url, routes in self.api.http.routes.items():\n            for url, methods in routes.items():\n                for method, versions in methods.items():\n                    for interface_version, interface in versions.items():\n                        if interface_version == version and interface == self:\n                            if not url in urls:\n                                urls.append(('/v{0}'.format(version) if version else '') + url)\n        return urls"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef url(self, version=None, **kwargs):\n        for url in self.urls(version):\n            if [key for key in kwargs.keys() if not '{' + key + '}' in url]:\n                continue\n\n            return url.format(**kwargs)\n\n        raise KeyError('URL that takes all provided parameters not found')", "response": "Returns the first matching URL for the specified arguments"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef greet(event: str):\n    greetings = \"Happy\"\n    if event == \"Christmas\":\n        greetings = \"Merry\"\n    if event == \"Kwanzaa\":\n        greetings = \"Joyous\"\n    if event == \"wishes\":\n        greetings = \"Warm\"\n\n    return \"{greetings} {event}!\".format(**locals())", "response": "Greets appropriately (from http://blog.ketchum.com/how-to-write-10-common-holiday-greetings/)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef hash_password(password, salt):\n    password = str(password).encode('utf-8')\n    salt = str(salt).encode('utf-8')\n    return hashlib.sha512(password + salt).hexdigest()", "response": "Hash a password using a provided salt"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ngenerate a random API key for a user", "response": "def gen_api_key(username):\n    \"\"\"\n    Create a random API key for a user\n    :param username:\n    :return: Hex encoded SHA512 random string\n    \"\"\"\n    salt = str(os.urandom(64)).encode('utf-8')\n    return hash_password(username, salt)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nauthenticating a username and password against our database", "response": "def authenticate_user(username, password):\n    \"\"\"\n    Authenticate a username and password against our database\n    :param username:\n    :param password:\n    :return: authenticated username\n    \"\"\"\n    user_model = Query()\n    user = db.get(user_model.username == username)\n\n    if not user:\n        logger.warning(\"User %s not found\", username)\n        return False\n\n    if user['password'] == hash_password(password, user.get('salt')):\n        return user['username']\n\n    return False"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef authenticate_key(api_key):\n    user_model = Query()\n    user = db.search(user_model.api_key == api_key)[0]\n    if user:\n        return user['username']\n    return False", "response": "Authenticate an API key against our database returning the username of the authenticated user"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef add_user(username, password):\n\n    user_model = Query()\n    if db.search(user_model.username == username):\n        return {\n            'error': 'User {0} already exists'.format(username)\n        }\n\n    salt = hashlib.sha512(str(os.urandom(64)).encode('utf-8')).hexdigest()\n    password = hash_password(password, salt)\n    api_key = gen_api_key(username)\n\n    user = {\n        'username': username,\n        'password': password,\n        'salt': salt,\n        'api_key': api_key\n    }\n    user_id = db.insert(user)\n\n    return {\n       'result': 'success',\n       'eid': user_id,\n       'user_created': user\n    }", "response": "This function adds a user to the database"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nget Job details :param authed_user: :return:", "response": "def get_token(authed_user: hug.directives.user):\n    \"\"\"\n    Get Job details\n    :param authed_user:\n    :return:\n    \"\"\"\n    user_model = Query()\n    user = db.search(user_model.username == authed_user)[0]\n\n    if user:\n        out = {\n            'user': user['username'],\n            'api_key': user['api_key']\n        }\n    else:\n        # this should never happen\n        out = {\n            'error': 'User {0} does not exist'.format(authed_user)\n        }\n\n    return out"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nsay happy birthday to a user", "response": "def happy_birthday(name: hug.types.text, age: hug.types.number, hug_timer=3):\n    \"\"\"Says happy birthday to a user\"\"\"\n    return {'message': 'Happy {0} Birthday {1}!'.format(age, name),\n            'took': float(hug_timer)}"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef json_convert(*kinds):\n    def register_json_converter(function):\n        for kind in kinds:\n            json_converters[kind] = function\n        return function\n    return register_json_converter", "response": "Registers a function as a JSON converter for the provided types."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef json(content, request=None, response=None, ensure_ascii=False, **kwargs):\n    if hasattr(content, 'read'):\n        return content\n\n    if isinstance(content, tuple) and getattr(content, '_fields', None):\n        content = {field: getattr(content, field) for field in content._fields}\n    return json_converter.dumps(content, default=_json_converter, ensure_ascii=ensure_ascii, **kwargs).encode('utf8')", "response": "JSON - serializable object Notation"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn a function that renders as the specified content type only if no errors are found in the provided data object.", "response": "def on_valid(valid_content_type, on_invalid=json):\n    \"\"\"Renders as the specified content type only if no errors are found in the provided data object\"\"\"\n    invalid_kwargs = introspect.generate_accepted_kwargs(on_invalid, 'request', 'response')\n    invalid_takes_response = introspect.takes_all_arguments(on_invalid, 'response')\n\n    def wrapper(function):\n        valid_kwargs = introspect.generate_accepted_kwargs(function, 'request', 'response')\n        valid_takes_response = introspect.takes_all_arguments(function, 'response')\n\n        @content_type(valid_content_type)\n        @wraps(function)\n        def output_content(content, response, **kwargs):\n            if type(content) == dict and 'errors' in content:\n                response.content_type = on_invalid.content_type\n                if invalid_takes_response:\n                    kwargs['response'] = response\n                return on_invalid(content, **invalid_kwargs(kwargs))\n\n            if valid_takes_response:\n                kwargs['response'] = response\n            return function(content, **valid_kwargs(kwargs))\n        return output_content\n    return wrapper"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef image(image_format, doc=None):\n    @on_valid('image/{0}'.format(image_format))\n    def image_handler(data, **kwargs):\n        if hasattr(data, 'read'):\n            return data\n        elif hasattr(data, 'save'):\n            output = stream()\n            if introspect.takes_all_arguments(data.save, 'format') or introspect.takes_kwargs(data.save):\n                data.save(output, format=image_format.upper())\n            else:\n                data.save(output)\n            output.seek(0)\n            return output\n        elif hasattr(data, 'render'):\n            return data.render()\n        elif os.path.isfile(data):\n            return open(data, 'rb')\n\n    image_handler.__doc__ = doc or \"{0} formatted image\".format(image_format)\n    return image_handler", "response": "Dynamically creates an image type handler for the specified image type"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef video(video_type, video_mime, doc=None):\n    @on_valid(video_mime)\n    def video_handler(data, **kwargs):\n        if hasattr(data, 'read'):\n            return data\n        elif hasattr(data, 'save'):\n            output = stream()\n            data.save(output, format=video_type.upper())\n            output.seek(0)\n            return output\n        elif hasattr(data, 'render'):\n            return data.render()\n        elif os.path.isfile(data):\n            return open(data, 'rb')\n\n    video_handler.__doc__ = doc or \"{0} formatted video\".format(video_type)\n    return video_handler", "response": "Dynamically creates a video type handler for the specified video type"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef file(data, response, **kwargs):\n    if not data:\n        response.content_type = 'text/plain'\n        return ''\n\n    if hasattr(data, 'read'):\n        name, data = getattr(data, 'name', ''), data\n    elif os.path.isfile(data):\n        name, data = data, open(data, 'rb')\n    else:\n        response.content_type = 'text/plain'\n        response.status = HTTP_NOT_FOUND\n        return 'File not found!'\n\n    response.content_type = mimetypes.guess_type(name, None)[0] or 'application/octet-stream'\n    return data", "response": "A dynamically retrieved file"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns a function that will handle the content type of the current resource based on the clients provided content type.", "response": "def on_content_type(handlers, default=None, error='The requested content type does not match any of those allowed'):\n    \"\"\"Returns a content in a different format based on the clients provided content type,\n       should pass in a dict with the following format:\n\n            {'[content-type]': action,\n             ...\n            }\n    \"\"\"\n    def output_type(data, request, response):\n        handler = handlers.get(request.content_type.split(';')[0], default)\n        if not handler:\n            raise falcon.HTTPNotAcceptable(error)\n\n        response.content_type = handler.content_type\n        return handler(data, request=request, response=response)\n    output_type.__doc__ = 'Supports any of the following formats: {0}'.format(', '.join(\n        function.__doc__ or function.__name__ for function in handlers.values()))\n    output_type.content_type = ', '.join(handlers.keys())\n    return output_type"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nseparating out the quality score from the accepted content_type", "response": "def accept_quality(accept, default=1):\n    \"\"\"Separates out the quality score from the accepted content_type\"\"\"\n    quality = default\n    if accept and \";\" in accept:\n        accept, rest = accept.split(\";\", 1)\n        accept_quality = RE_ACCEPT_QUALITY.search(rest)\n        if accept_quality:\n            quality = float(accept_quality.groupdict().get('quality', quality).strip())\n\n    return (quality, accept.strip())"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef accept(handlers, default=None, error='The requested content type does not match any of those allowed'):\n    def output_type(data, request, response):\n        accept = request.accept\n        if accept in ('', '*', '/'):\n            handler = default or handlers and next(iter(handlers.values()))\n        else:\n            handler = default\n            accepted = [accept_quality(accept_type) for accept_type in accept.split(',')]\n            accepted.sort(key=itemgetter(0))\n            for quality, accepted_content_type in reversed(accepted):\n                if accepted_content_type in handlers:\n                    handler = handlers[accepted_content_type]\n                    break\n\n        if not handler:\n            raise falcon.HTTPNotAcceptable(error)\n\n        response.content_type = handler.content_type\n        return handler(data, request=request, response=response)\n    output_type.__doc__ = 'Supports any of the following formats: {0}'.format(', '.join(function.__doc__ for function in\n                                                                                        handlers.values()))\n    output_type.content_type = ', '.join(handlers.keys())\n    return output_type", "response": "Returns a function that returns a content in a different format based on the clients defined accepted content type."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef suffix(handlers, default=None, error='The requested suffix does not match any of those allowed'):\n    def output_type(data, request, response):\n        path = request.path\n        handler = default\n        for suffix_test, suffix_handler in handlers.items():\n            if path.endswith(suffix_test):\n                handler = suffix_handler\n                break\n\n        if not handler:\n            raise falcon.HTTPNotAcceptable(error)\n\n        response.content_type = handler.content_type\n        return handler(data, request=request, response=response)\n    output_type.__doc__ = 'Supports any of the following formats: {0}'.format(', '.join(function.__doc__ for function in\n                                                                                        handlers.values()))\n    output_type.content_type = ', '.join(handlers.keys())\n    return output_type", "response": "Returns a function that returns a content in a different format based on the requested suffix."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a function that returns a content based on the requested prefix.", "response": "def prefix(handlers, default=None, error='The requested prefix does not match any of those allowed'):\n    \"\"\"Returns a content in a different format based on the prefix placed at the end of the URL route\n       should pass in a dict with the following format:\n\n            {'[prefix]': action,\n             ...\n            }\n    \"\"\"\n    def output_type(data, request, response):\n        path = request.path\n        handler = default\n        for prefix_test, prefix_handler in handlers.items():\n            if path.startswith(prefix_test):\n                handler = prefix_handler\n                break\n\n        if not handler:\n            raise falcon.HTTPNotAcceptable(error)\n\n        response.content_type = handler.content_type\n        return handler(data, request=request, response=response)\n    output_type.__doc__ = 'Supports any of the following formats: {0}'.format(', '.join(function.__doc__ for function in\n                                                                                        handlers.values()))\n    output_type.content_type = ', '.join(handlers.keys())\n    return output_type"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef show():\n    items = list(collection.find())\n    # JSON conversion chokes on the _id objects, so we convert\n    # them to strings here\n    for i in items:\n        i['_id'] = str(i['_id'])\n    return items", "response": "Returns a list of items currently in the database"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef new(name: hug.types.text, description: hug.types.text):\n    item_doc = {'name': name, 'description': description}\n    collection.insert_one(item_doc)\n    return str(item_doc['_id'])", "response": "Inserts the given object as a new item in the database. Returns the ID of the newly created item."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef request(self, method, url, url_params=empty.dict, headers=empty.dict, timeout=None, **params):\n        raise NotImplementedError(\"Concrete services must define the request method\")", "response": "Calls the service at the specified URL using the CALL method."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef options(self, url, url_params=empty.dict, headers=empty.dict, timeout=None, **params):\n        return self.request('OPTIONS', url=url, headers=headers, timeout=timeout, **params)", "response": "Calls the specified URL using the OPTIONS method"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nadds socket options to set", "response": "def setsockopt(self, *sockopts):\n        \"\"\"Add socket options to set\"\"\"\n        if type(sockopts[0]) in (list, tuple):\n            for sock_opt in sockopts[0]:\n                level, option, value = sock_opt\n                self.connection.sockopts.add((level, option, value))\n        else:\n            level, option, value = sockopts\n            self.connection.sockopts.add((level, option, value))"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _register_socket(self):\n        _socket = socket.socket(*Socket.protocols[self.connection.proto])\n        _socket.settimeout(self.timeout)\n\n        # Reconfigure original socket options.\n        if self.connection.sockopts:\n            for sock_opt in self.connection.sockopts:\n                level, option, value = sock_opt\n                _socket.setsockopt(level, option, value)\n\n        _socket.connect(self.connection.connect_to)\n        return _socket", "response": "Create and connect a new socket with the same protocol and options."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _dgram_send_and_receive(self, _socket, message, buffer_size=4096, *args):\n        _socket.send(message.encode('utf-8'))\n        data, address = _socket.recvfrom(buffer_size)\n        return BytesIO(data)", "response": "Send and receive a message to the socket and return a BytesIO."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\npopulates connection pool send message return BytesIO and cleanup", "response": "def request(self, message, timeout=False, *args, **kwargs):\n        \"\"\"Populate connection pool, send message, return BytesIO, and cleanup\"\"\"\n        if not self.connection_pool.full():\n            self.connection_pool.put(self._register_socket())\n\n        _socket = self.connection_pool.get()\n\n        # setting timeout to None enables the socket to block.\n        if timeout or timeout is None:\n            _socket.settimeout(timeout)\n\n        data = self.send_and_receive(_socket, message, *args, **kwargs)\n\n        if self.connection.proto in Socket.streams:\n            _socket.shutdown(socket.SHUT_RDWR)\n\n        return Response(data, None, None)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef top_post(section: hug.types.one_of(('news', 'newest', 'show'))='news'):\n    content = requests.get('https://news.ycombinator.com/{0}'.format(section)).content\n    text = content.decode('utf-8')\n    return text.split('<tr class=\\'athing\\'>')[1].split(\"<a href\")[1].split(\">\")[1].split(\"<\")[0]", "response": "Returns the top post from the provided section"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef directive(apply_globally=False, api=None):\n    def decorator(directive_method):\n        if apply_globally:\n            hug.defaults.directives[underscore(directive_method.__name__)] = directive_method\n        else:\n            apply_to_api = hug.API(api) if api else hug.api.from_object(directive_method)\n            apply_to_api.add_directive(directive_method)\n        directive_method.directive = True\n        return directive_method\n    return decorator", "response": "A decorator that registers a single hug directive"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef delete_context(apply_globally=False, api=None):\n    def decorator(delete_context_):\n        if apply_globally:\n            hug.defaults.delete_context = delete_context_\n        else:\n            apply_to_api = hug.API(api) if api else hug.api.from_object(delete_context_)\n            apply_to_api.delete_context = delete_context_\n        return delete_context_\n    return decorator", "response": "A decorator that registers a single hug delete context function"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef startup(api=None):\n    def startup_wrapper(startup_function):\n        apply_to_api = hug.API(api) if api else hug.api.from_object(startup_function)\n        apply_to_api.add_startup_handler(startup_function)\n        return startup_function\n    return startup_wrapper", "response": "Runs the provided function on startup passing in an instance of the api"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef request_middleware(api=None):\n    def decorator(middleware_method):\n        apply_to_api = hug.API(api) if api else hug.api.from_object(middleware_method)\n\n        class MiddlewareRouter(object):\n            __slots__ = ()\n\n            def process_request(self, request, response):\n                return middleware_method(request, response)\n\n        apply_to_api.http.add_middleware(MiddlewareRouter())\n        return middleware_method\n    return decorator", "response": "Registers a middleware function that will be called on every request"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef response_middleware(api=None):\n    def decorator(middleware_method):\n        apply_to_api = hug.API(api) if api else hug.api.from_object(middleware_method)\n\n        class MiddlewareRouter(object):\n            __slots__ = ()\n\n            def process_response(self, request, response, resource, req_succeeded):\n                return middleware_method(request, response, resource)\n\n        apply_to_api.http.add_middleware(MiddlewareRouter())\n        return middleware_method\n    return decorator", "response": "Registers a middleware function that will be called on every response"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef reqresp_middleware(api=None):\n    def decorator(middleware_generator):\n        apply_to_api = hug.API(api) if api else hug.api.from_object(middleware_generator)\n\n        class MiddlewareRouter(object):\n            __slots__ = ('gen', )\n\n            def process_request(self, request, response):\n                self.gen = middleware_generator(request)\n                return self.gen.__next__()\n\n            def process_response(self, request, response, resource, req_succeeded):\n                return self.gen.send((response, resource))\n\n        apply_to_api.http.add_middleware(MiddlewareRouter())\n        return middleware_generator\n    return decorator", "response": "Registers a middleware function that will be called on every request and response"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef middleware_class(api=None):\n    def decorator(middleware_class):\n        apply_to_api = hug.API(api) if api else hug.api.from_object(middleware_class)\n        apply_to_api.http.add_middleware(middleware_class())\n        return middleware_class\n    return decorator", "response": "Registers a middleware class"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nextends the current api with handlers from an imported api. Optionally provide a route that prefixes access.", "response": "def extend_api(route=\"\", api=None, base_url=\"\", **kwargs):\n    \"\"\"Extends the current api, with handlers from an imported api. Optionally provide a route that prefixes access\"\"\"\n    def decorator(extend_with):\n        apply_to_api = hug.API(api) if api else hug.api.from_object(extend_with)\n        for extended_api in extend_with():\n            apply_to_api.extend(extended_api, route, base_url, **kwargs)\n        return extend_with\n    return decorator"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef wraps(function):\n    def wrap(decorator):\n        decorator = functools.wraps(function)(decorator)\n        if not hasattr(function, 'original'):\n            decorator.original = function\n        else:\n            decorator.original = function.original\n            delattr(function, 'original')\n        return decorator\n    return wrap", "response": "Enables building decorators around functions used for hug routes without changing their function signature"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef auto_kwargs(function):\n    supported = introspect.arguments(function)\n\n    @wraps(function)\n    def call_function(*args, **kwargs):\n        return function(*args, **{key: value for key, value in kwargs.items() if key in supported})\n    return call_function", "response": "Modifies the provided function to support kwargs by only passing along kwargs for parameters it accepts"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_time():\n\n    time_request = '\\x1b' + 47 * '\\0'\n    now = struct.unpack(\"!12I\", ntp_service.request(time_request, timeout=5.0).data.read())[10]\n    return time.ctime(now - EPOCH_START)", "response": "Get time from a locally running NTP server"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef reverse_http_proxy(length: int=100):\n\n    http_request = \"\"\"\nGET / HTTP/1.0\\r\\n\\r\\n\nHost: www.google.com\\r\\n\\r\\n\n\\r\\n\\r\\n\n\"\"\"\n    return http_socket.request(http_request, timeout=5.0).data.read()[0:length]", "response": "Simple http proxy function that returns data from another http server"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nvalidating only succeeds if all passed in validators return no errors", "response": "def all(*validators):\n    \"\"\"Validation only succeeds if all passed in validators return no errors\"\"\"\n    def validate_all(fields):\n        for validator in validators:\n            errors = validator(fields)\n            if errors:\n                return errors\n\n    validate_all.__doc__ = \" and \".join(validator.__doc__ for validator in validators)\n    return validate_all"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns a validation function that checks if any of the specified validators pass the validation succeeds.", "response": "def any(*validators):\n    \"\"\"If any of the specified validators pass the validation succeeds\"\"\"\n    def validate_any(fields):\n        errors = {}\n        for validator in validators:\n            validation_errors = validator(fields)\n            if not validation_errors:\n                return\n            errors.update(validation_errors)\n        return errors\n\n    validate_any.__doc__ = \" or \".join(validator.__doc__ for validator in validators)\n    return validate_any"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef contains_one_of(*fields):\n    message = 'Must contain any one of the following fields: {0}'.format(', '.join(fields))\n\n    def check_contains(endpoint_fields):\n        for field in fields:\n            if field in endpoint_fields:\n                return\n\n        errors = {}\n        for field in fields:\n            errors[field] = 'one of these must have a value'\n        return errors\n    check_contains.__doc__ = message\n    return check_contains", "response": "Enables ensuring that one of multiple optional fields is set"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef requires(self, requirements, **overrides):\n        return self.where(requires=tuple(self.route.get('requires', ())) + tuple(requirements), **overrides)", "response": "Adds additional requirements to the specified route"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nremoving individual requirements while keeping all other defined ones within a route", "response": "def doesnt_require(self, requirements, **overrides):\n        \"\"\"Removes individual requirements while keeping all other defined ones within a route\"\"\"\n        return self.where(requires=tuple(set(self.route.get('requires', ())).difference(requirements if\n                                                            type(requirements) in (list, tuple) else (requirements, ))))"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef where(self, **overrides):\n        route_data = self.route.copy()\n        route_data.update(overrides)\n        return self.__class__(**route_data)", "response": "Creates a new route based on the current route with the specified overrided values"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nsetting the route to raise validation errors instead of catching them", "response": "def raise_on_invalid(self, setting=True, **overrides):\n        \"\"\"Sets the route to raise validation errors instead of catching them\"\"\"\n        return self.where(raise_on_invalid=setting, **overrides)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ntell hug to automatically parse the input body if it matches a registered input format", "response": "def parse_body(self, automatic=True, **overrides):\n        \"\"\"Tells hug to automatically parse the input body if it matches a registered input format\"\"\"\n        return self.where(parse_body=automatic, **overrides)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef add_response_headers(self, headers, **overrides):\n        response_headers = self.route.get('response_headers', {}).copy()\n        response_headers.update(headers)\n        return self.where(response_headers=response_headers, **overrides)", "response": "Adds the specified response headers while keeping existing ones in - tact"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef allow_origins(self, *origins, methods=None, max_age=None, credentials=None, headers=None, **overrides):\n        response_headers = {}\n        if origins:\n            @hug.response_middleware()\n            def process_data(request, response, resource):\n                if 'ORIGIN' in request.headers:\n                    origin = request.headers['ORIGIN']\n                    if origin in origins:\n                        response.set_header('Access-Control-Allow-Origin', origin)\n        else:\n            response_headers['Access-Control-Allow-Origin'] = '*'\n\n        if methods:\n            response_headers['Access-Control-Allow-Methods'] = ', '.join(methods)\n        if max_age:\n            response_headers['Access-Control-Max-Age'] = max_age\n        if credentials:\n            response_headers['Access-Control-Allow-Credentials'] = str(credentials).lower()\n        if headers:\n            response_headers['Access-Control-Allow-Headers'] = headers\n        return self.add_response_headers(response_headers, **overrides)", "response": "Convenience method for quickly allowing other resources to access this one"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nsets the acceptable HTTP method to a GET", "response": "def get(self, urls=None, **overrides):\n        \"\"\"Sets the acceptable HTTP method to a GET\"\"\"\n        if urls is not None:\n            overrides['urls'] = urls\n        return self.where(accept='GET', **overrides)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef delete(self, urls=None, **overrides):\n        if urls is not None:\n            overrides['urls'] = urls\n        return self.where(accept='DELETE', **overrides)", "response": "Sets the acceptable HTTP method to DELETE"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nsetting the acceptable HTTP method to POST", "response": "def post(self, urls=None, **overrides):\n        \"\"\"Sets the acceptable HTTP method to POST\"\"\"\n        if urls is not None:\n            overrides['urls'] = urls\n        return self.where(accept='POST', **overrides)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef put(self, urls=None, **overrides):\n        if urls is not None:\n            overrides['urls'] = urls\n        return self.where(accept='PUT', **overrides)", "response": "Sets the acceptable HTTP method to PUT"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef trace(self, urls=None, **overrides):\n        if urls is not None:\n            overrides['urls'] = urls\n        return self.where(accept='TRACE', **overrides)", "response": "Sets the acceptable HTTP method to TRACE"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nsets the acceptable HTTP method to PATCH", "response": "def patch(self, urls=None, **overrides):\n        \"\"\"Sets the acceptable HTTP method to PATCH\"\"\"\n        if urls is not None:\n            overrides['urls'] = urls\n        return self.where(accept='PATCH', **overrides)"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nsets the acceptable HTTP method to OPTIONS", "response": "def options(self, urls=None, **overrides):\n        \"\"\"Sets the acceptable HTTP method to OPTIONS\"\"\"\n        if urls is not None:\n            overrides['urls'] = urls\n        return self.where(accept='OPTIONS', **overrides)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef head(self, urls=None, **overrides):\n        if urls is not None:\n            overrides['urls'] = urls\n        return self.where(accept='HEAD', **overrides)", "response": "Sets the acceptable HTTP method to HEAD"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef connect(self, urls=None, **overrides):\n        if urls is not None:\n            overrides['urls'] = urls\n        return self.where(accept='CONNECT', **overrides)", "response": "Sets the acceptable HTTP method to CONNECT"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef http_methods(self, urls=None, **route_data):\n        def decorator(class_definition):\n            instance = class_definition\n            if isinstance(class_definition, type):\n                instance = class_definition()\n\n            router = self.urls(urls if urls else \"/{0}\".format(instance.__class__.__name__.lower()), **route_data)\n            for method in HTTP_METHODS:\n                handler = getattr(instance, method.lower(), None)\n                if handler:\n                    http_routes = getattr(handler, '_hug_http_routes', ())\n                    if http_routes:\n                        for route in http_routes:\n                            http(**router.accept(method).where(**route).route)(handler)\n                    else:\n                        http(**router.accept(method).route)(handler)\n\n                    cli_routes = getattr(handler, '_hug_cli_routes', ())\n                    if cli_routes:\n                        for route in cli_routes:\n                            cli(**self.where(**route).route)(handler)\n            return class_definition\n        return decorator", "response": "Creates routes from a class where the class method names should line up to HTTP METHOD types"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nregister a method on an Object as a CLI route", "response": "def cli(self, method):\n        \"\"\"Registers a method on an Object as a CLI route\"\"\"\n        routes = getattr(method, '_hug_cli_routes', [])\n        routes.append(self.route)\n        method._hug_cli_routes = routes\n        return method"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nstarting the process of building a new HTTP route linked to this API instance", "response": "def http(self, *args, **kwargs):\n        \"\"\"Starts the process of building a new HTTP route linked to this API instance\"\"\"\n        kwargs['api'] = self.api\n        return http(*args, **kwargs)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef not_found(self, *args, **kwargs):\n        kwargs['api'] = self.api\n        return not_found(*args, **kwargs)", "response": "Defines the handler that should handle not found requests against this API"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ndefine the routes to static files the API should expose", "response": "def static(self, *args, **kwargs):\n        \"\"\"Define the routes to static files the API should expose\"\"\"\n        kwargs['api'] = self.api\n        return static(*args, **kwargs)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ndefining URL prefixes and handler matches where everything under the URL prefix should be handled", "response": "def sink(self, *args, **kwargs):\n        \"\"\"Define URL prefixes/handler matches where everything under the URL prefix should be handled\"\"\"\n        kwargs['api'] = self.api\n        return sink(*args, **kwargs)"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ndefine how this API should handle the provided exceptions", "response": "def exception(self, *args, **kwargs):\n        \"\"\"Defines how this API should handle the provided exceptions\"\"\"\n        kwargs['api'] = self.api\n        return exception(*args, **kwargs)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef cli(self, *args, **kwargs):\n        kwargs['api'] = self.api\n        return cli(*args, **kwargs)", "response": "Defines a function that should be routed by this API"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nregisters a class based router to this API", "response": "def object(self, *args, **kwargs):\n        \"\"\"Registers a class based router to this API\"\"\"\n        kwargs['api'] = self.api\n        return Object(*args, **kwargs)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning a generator of all URLs attached to this API", "response": "def urls(self):\n        \"\"\"Returns a generator of all URLs attached to this API\"\"\"\n        for base_url, mapping in self.routes.items():\n            for url, _ in mapping.items():\n                yield base_url + url"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn all registered handlers attached to this API", "response": "def handlers(self):\n        \"\"\"Returns all registered handlers attached to this API\"\"\"\n        used = []\n        for base_url, mapping in self.routes.items():\n            for url, methods in mapping.items():\n                for method, versions in methods.items():\n                    for version, handler in versions.items():\n                        if not handler in used:\n                            used.append(handler)\n                            yield handler"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns the set input_format handler for the given content_type", "response": "def input_format(self, content_type):\n        \"\"\"Returns the set input_format handler for the given content_type\"\"\"\n        return getattr(self, '_input_format', {}).get(content_type, hug.defaults.input_format.get(content_type, None))"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nsets an input format handler for this Hug API given the specified content_type", "response": "def set_input_format(self, content_type, handler):\n        \"\"\"Sets an input format handler for this Hug API, given the specified content_type\"\"\"\n        if getattr(self, '_input_format', None) is None:\n            self._input_format = {}\n        self._input_format[content_type] = handler"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nadd a middleware object used to process all incoming requests against the API.", "response": "def add_middleware(self, middleware):\n        \"\"\"Adds a middleware object used to process all incoming requests against the API\"\"\"\n        if self.middleware is None:\n            self._middleware = []\n        self.middleware.append(middleware)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nadding an exception handler to the hug api", "response": "def add_exception_handler(self, exception_type, error_handler, versions=(None, )):\n        \"\"\"Adds a error handler to the hug api\"\"\"\n        versions = (versions, ) if not isinstance(versions, (tuple, list)) else versions\n        if not hasattr(self, '_exception_handlers'):\n            self._exception_handlers = {}\n\n        for version in versions:\n            placement = self._exception_handlers.setdefault(version, OrderedDict())\n            placement[exception_type] = (error_handler, ) + placement.get(exception_type, tuple())"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nextending this instance with the given http_api.", "response": "def extend(self, http_api, route=\"\", base_url=\"\", **kwargs):\n        \"\"\"Adds handlers from a different Hug API to this one - to create a single API\"\"\"\n        self.versions.update(http_api.versions)\n        base_url = base_url or self.base_url\n\n        for router_base_url, routes in http_api.routes.items():\n            self.routes.setdefault(base_url, OrderedDict())\n            for item_route, handler in routes.items():\n                for method, versions in handler.items():\n                    for version, function in versions.items():\n                        function.interface.api = self.api\n                self.routes[base_url].setdefault(route + item_route, {}).update(handler)\n\n        for sink_base_url, sinks in http_api.sinks.items():\n            for url, sink in sinks.items():\n                self.add_sink(sink, route + url, base_url=base_url)\n\n        for middleware in (http_api.middleware or ()):\n            self.add_middleware(middleware)\n\n        for version, handler in getattr(http_api, '_exception_handlers', {}).items():\n            for exception_type, exception_handlers in handler.items():\n                target_exception_handlers = self.exception_handlers(version) or {}\n                for exception_handler in exception_handlers:\n                    if exception_type not in target_exception_handlers:\n                        self.add_exception_handler(exception_type, exception_handler, version)\n\n        for input_format, input_format_handler in getattr(http_api, '_input_format', {}).items():\n            if not input_format in getattr(self, '_input_format', {}):\n                self.set_input_format(input_format, input_format_handler)\n\n        for version, handler in http_api.not_found_handlers.items():\n            if version not in self.not_found_handlers:\n                self.set_not_found_handler(handler, version)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef set_not_found_handler(self, handler, version=None):\n        if not self.not_found_handlers:\n            self._not_found_handlers = {}\n\n        self.not_found_handlers[version] = handler", "response": "Sets the not_found handler for the specified version of the api"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef documentation(self, base_url=None, api_version=None, prefix=\"\"):\n        documentation = OrderedDict()\n        base_url = self.base_url if base_url is None else base_url\n        overview = self.api.doc\n        if overview:\n            documentation['overview'] = overview\n\n        version_dict = OrderedDict()\n        versions = self.versions\n        versions_list = list(versions)\n        if None in versions_list:\n            versions_list.remove(None)\n        if False in versions_list:\n            versions_list.remove(False)\n        if api_version is None and len(versions_list) > 0:\n            api_version = max(versions_list)\n            documentation['version'] = api_version\n        elif api_version is not None:\n            documentation['version'] = api_version\n        if versions_list:\n            documentation['versions'] = versions_list\n        for router_base_url, routes in self.routes.items():\n            for url, methods in routes.items():\n                for method, method_versions in methods.items():\n                    for version, handler in method_versions.items():\n                        if getattr(handler, 'private', False):\n                            continue\n                        if version is None:\n                            applies_to = versions\n                        else:\n                            applies_to = (version, )\n                        for version in applies_to:\n                            if api_version and version != api_version:\n                                continue\n                            if base_url and router_base_url != base_url:\n                                continue\n                            doc = version_dict.setdefault(url, OrderedDict())\n                            doc[method] = handler.documentation(doc.get(method, None), version=version, prefix=prefix,\n                                                                base_url=router_base_url, url=url)\n        documentation['handlers'] = version_dict\n        return documentation", "response": "Generates and returns documentation for this API endpoint"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nrunning the basic hug development server against this API", "response": "def serve(self, host='', port=8000, no_documentation=False, display_intro=True):\n        \"\"\"Runs the basic hug development server against this API\"\"\"\n        if no_documentation:\n            api = self.server(None)\n        else:\n            api = self.server()\n\n        if display_intro:\n            print(INTRO)\n\n        httpd = make_server(host, port, api)\n        print(\"Serving on {0}:{1}...\".format(host, port))\n        httpd.serve_forever()"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ndetermine the appropriate version given the set api_version the request header and URL query params", "response": "def determine_version(self, request, api_version=None):\n        \"\"\"Determines the appropriate version given the set api_version, the request header, and URL query params\"\"\"\n        if api_version is False:\n            api_version = None\n            for version in self.versions:\n                if version and \"v{0}\".format(version) in request.path:\n                    api_version = version\n                    break\n\n        request_version = set()\n        if api_version is not None:\n            request_version.add(api_version)\n\n        version_header = request.get_header(\"X-API-VERSION\")\n        if version_header:\n            request_version.add(version_header)\n\n        version_param = request.get_param('api_version')\n        if version_param is not None:\n            request_version.add(version_param)\n\n        if len(request_version) > 1:\n            raise ValueError('You are requesting conflicting versions')\n\n        return next(iter(request_version or (None, )))"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef documentation_404(self, base_url=None):\n        base_url = self.base_url if base_url is None else base_url\n\n        def handle_404(request, response, *args, **kwargs):\n            url_prefix = request.forwarded_uri[:-1]\n            if request.path and request.path != \"/\":\n                url_prefix = request.forwarded_uri.split(request.path)[0]\n\n            to_return = OrderedDict()\n            to_return['404'] = (\"The API call you tried to make was not defined. \"\n                                \"Here's a definition of the API to help you get going :)\")\n            to_return['documentation'] = self.documentation(base_url, self.determine_version(request, False),\n                                                            prefix=url_prefix)\n\n            if self.output_format == hug.output_format.json:\n                response.data = hug.output_format.json(to_return, indent=4, separators=(',', ': '))\n                response.content_type = 'application/json; charset=utf-8'\n            else:\n                response.data = self.output_format(to_return, request=request, response=response)\n                response.content_type = self.output_format.content_type\n\n            response.status = falcon.HTTP_NOT_FOUND\n\n        handle_404.interface = True\n        return handle_404", "response": "Returns a smart 404 page that contains documentation for the written API"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef version_router(self, request, response, api_version=None, versions={}, not_found=None, **kwargs):\n        request_version = self.determine_version(request, api_version)\n        if request_version:\n            request_version = int(request_version)\n        versions.get(request_version or False, versions.get(None, not_found))(request, response,\n                                                                              api_version=api_version,\n                                                                              **kwargs)", "response": "Intelligently routes a request to the correct handler based on the version being requested"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a WSGI compatible API server for the given Hug API module.", "response": "def server(self, default_not_found=True, base_url=None):\n        \"\"\"Returns a WSGI compatible API server for the given Hug API module\"\"\"\n        falcon_api = falcon.API(middleware=self.middleware)\n        default_not_found = self.documentation_404() if default_not_found is True else None\n        base_url = self.base_url if base_url is None else base_url\n\n        not_found_handler = default_not_found\n        self.api._ensure_started()\n        if self.not_found_handlers:\n            if len(self.not_found_handlers) == 1 and None in self.not_found_handlers:\n                not_found_handler = self.not_found_handlers[None]\n            else:\n                not_found_handler = partial(self.version_router, api_version=False,\n                                            versions=self.not_found_handlers, not_found=default_not_found)\n                not_found_handler.interface = True\n\n        if not_found_handler:\n            falcon_api.add_sink(not_found_handler)\n            self._not_found = not_found_handler\n\n        for sink_base_url, sinks in self.sinks.items():\n            for url, extra_sink in sinks.items():\n                falcon_api.add_sink(extra_sink, sink_base_url + url + '(?P<path>.*)')\n\n        for router_base_url, routes in self.routes.items():\n            for url, methods in routes.items():\n                router = {}\n                for method, versions in methods.items():\n                    method_function = \"on_{0}\".format(method.lower())\n                    if len(versions) == 1 and None in versions.keys():\n                        router[method_function] = versions[None]\n                    else:\n                        router[method_function] = partial(self.version_router, versions=versions,\n                                                          not_found=not_found_handler)\n\n                router = namedtuple('Router', router.keys())(**router)\n                falcon_api.add_route(router_base_url + url, router)\n                if self.versions and self.versions != (None, ):\n                    falcon_api.add_route(router_base_url + '/v{api_version}' + url, router)\n\n        def error_serializer(request, response, error):\n            response.content_type = self.output_format.content_type\n            response.body = self.output_format({\"errors\": {error.title: error.description}},\n                                               request, response)\n\n        falcon_api.set_error_serializer(error_serializer)\n        return falcon_api"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef extend(self, cli_api, command_prefix=\"\", sub_command=\"\", **kwargs):\n        if sub_command and command_prefix:\n            raise ValueError('It is not currently supported to provide both a command_prefix and sub_command')\n\n        if sub_command:\n            self.commands[sub_command] = cli_api\n        else:\n            for name, command in cli_api.commands.items():\n                self.commands[\"{}{}\".format(command_prefix, name)] = command", "response": "Extends this CLI api with the commands present in the provided CLI api object."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning all directives applicable to this Hug API", "response": "def directives(self):\n        \"\"\"Returns all directives applicable to this Hug API\"\"\"\n        directive_sources = chain(hug.defaults.directives.items(), getattr(self, '_directives', {}).items())\n        return {'hug_' + directive_name: directive for directive_name, directive in directive_sources}"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn the loaded directive with the specified name or default if it is not present.", "response": "def directive(self, name, default=None):\n        \"\"\"Returns the loaded directive with the specified name, or default if passed name is not present\"\"\"\n        return getattr(self, '_directives', {}).get(name,  hug.defaults.directives.get(name, default))"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef handlers(self):\n        if getattr(self, '_http'):\n            yield from self.http.handlers()\n        if getattr(self, '_cli'):\n            yield from self.cli.handlers()", "response": "Returns all registered handlers attached to this API"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nadd handlers from a different Hug API to this one - to create a single API", "response": "def extend(self, api, route=\"\", base_url=\"\", http=True, cli=True, **kwargs):\n        \"\"\"Adds handlers from a different Hug API to this one - to create a single API\"\"\"\n        api = API(api)\n\n        if http and hasattr(api, '_http'):\n            self.http.extend(api.http, route, base_url, **kwargs)\n\n        if cli and hasattr(api, '_cli'):\n            self.cli.extend(api.cli, **kwargs)\n\n        for directive in getattr(api, '_directives', {}).values():\n            self.add_directive(directive)\n\n        for startup_handler in (api.startup_handlers or ()):\n            self.add_startup_handler(startup_handler)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef add_startup_handler(self, handler):\n        if not self.startup_handlers:\n            self._startup_handlers = []\n\n        self.startup_handlers.append(handler)", "response": "Adds a startup handler to the hug api"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nmarks the API as started and runs all startup handlers", "response": "def _ensure_started(self):\n        \"\"\"Marks the API as started and runs all startup handlers\"\"\"\n        if not self.started:\n            async_handlers = [startup_handler for startup_handler in self.startup_handlers if\n                              introspect.is_coroutine(startup_handler)]\n            if async_handlers:\n                loop = asyncio.get_event_loop()\n                loop.run_until_complete(asyncio.gather(*[handler(self) for handler in async_handlers], loop=loop))\n            for startup_handler in self.startup_handlers:\n                if not startup_handler in async_handlers:\n                    startup_handler(self)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nload the IMDB dataset for binary sentiment classification.", "response": "def imdb_dataset(directory='data/',\n                 train=False,\n                 test=False,\n                 train_directory='train',\n                 test_directory='test',\n                 extracted_name='aclImdb',\n                 check_files=['aclImdb/README'],\n                 url='http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz',\n                 sentiments=['pos', 'neg']):\n    \"\"\"\n    Load the IMDB dataset (Large Movie Review Dataset v1.0).\n\n    This is a dataset for binary sentiment classification containing substantially more data than\n    previous benchmark datasets. Provided a set of 25,000 highly polar movie reviews for\n    training, and 25,000 for testing. There is additional unlabeled data for use as well. Raw text\n    and already processed bag of words formats are provided.\n\n    Note:\n        The order examples are returned is not guaranteed due to ``iglob``.\n\n    **Reference:** http://ai.stanford.edu/~amaas/data/sentiment/\n\n    Args:\n        directory (str, optional): Directory to cache the dataset.\n        train (bool, optional): If to load the training split of the dataset.\n        test (bool, optional): If to load the test split of the dataset.\n        train_directory (str, optional): The directory of the training split.\n        test_directory (str, optional): The directory of the test split.\n        extracted_name (str, optional): Name of the extracted dataset directory.\n        check_files (str, optional): Check if these files exist, then this download was successful.\n        url (str, optional): URL of the dataset ``tar.gz`` file.\n        sentiments (list of str, optional): Sentiments to load from the dataset.\n\n    Returns:\n        :class:`tuple` of :class:`torchnlp.datasets.Dataset` or :class:`torchnlp.datasets.Dataset`:\n        Returns between one and all dataset splits (train, dev and test) depending on if their\n        respective boolean argument is ``True``.\n\n    Example:\n        >>> from torchnlp.datasets import imdb_dataset  # doctest: +SKIP\n        >>> train = imdb_dataset(train=True)  # doctest: +SKIP\n        >>> train[0:2]  # doctest: +SKIP\n        [{\n          'text': 'For a movie that gets no respect there sure are a lot of memorable quotes...',\n          'sentiment': 'pos'\n        }, {\n          'text': 'Bizarre horror movie filled with famous faces but stolen by Cristina Raines...',\n          'sentiment': 'pos'\n        }]\n    \"\"\"\n    download_file_maybe_extract(url=url, directory=directory, check_files=check_files)\n\n    ret = []\n    splits = [\n        dir_ for (requested, dir_) in [(train, train_directory), (test, test_directory)]\n        if requested\n    ]\n    for split_directory in splits:\n        full_path = os.path.join(directory, extracted_name, split_directory)\n        examples = []\n        for sentiment in sentiments:\n            for filename in glob.iglob(os.path.join(full_path, sentiment, '*.txt')):\n                with open(filename, 'r', encoding=\"utf-8\") as f:\n                    text = f.readline()\n                examples.append({\n                    'text': text,\n                    'sentiment': sentiment,\n                })\n        ret.append(Dataset(examples))\n\n    if len(ret) == 1:\n        return ret[0]\n    else:\n        return tuple(ret)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef trec_dataset(directory='data/trec/',\n                 train=False,\n                 test=False,\n                 train_filename='train_5500.label',\n                 test_filename='TREC_10.label',\n                 check_files=['train_5500.label'],\n                 urls=[\n                     'http://cogcomp.org/Data/QA/QC/train_5500.label',\n                     'http://cogcomp.org/Data/QA/QC/TREC_10.label'\n                 ],\n                 fine_grained=False):\n    \"\"\"\n    Load the Text REtrieval Conference (TREC) Question Classification dataset.\n\n    TREC dataset contains 5500 labeled questions in training set and another 500 for test set. The\n    dataset has 6 labels, 50 level-2 labels. Average length of each sentence is 10, vocabulary size\n    of 8700.\n\n    References:\n        * https://nlp.stanford.edu/courses/cs224n/2004/may-steinberg-project.pdf\n        * http://cogcomp.org/Data/QA/QC/\n        * http://www.aclweb.org/anthology/C02-1150\n\n    **Citation:**\n    Xin Li, Dan Roth, Learning Question Classifiers. COLING'02, Aug., 2002.\n\n    Args:\n        directory (str, optional): Directory to cache the dataset.\n        train (bool, optional): If to load the training split of the dataset.\n        test (bool, optional): If to load the test split of the dataset.\n        train_filename (str, optional): The filename of the training split.\n        test_filename (str, optional): The filename of the test split.\n        check_files (str, optional): Check if these files exist, then this download was successful.\n        urls (str, optional): URLs to download.\n\n    Returns:\n        :class:`tuple` of :class:`torchnlp.datasets.Dataset` or :class:`torchnlp.datasets.Dataset`:\n        Returns between one and all dataset splits (train, dev and test) depending on if their\n        respective boolean argument is ``True``.\n\n    Example:\n        >>> from torchnlp.datasets import trec_dataset  # doctest: +SKIP\n        >>> train = trec_dataset(train=True)  # doctest: +SKIP\n        >>> train[:2]  # doctest: +SKIP\n        [{\n          'label': 'DESC',\n          'text': 'How did serfdom develop in and then leave Russia ?'\n        }, {\n          'label': 'ENTY',\n          'text': 'What films featured the character Popeye Doyle ?'\n        }]\n    \"\"\"\n    download_files_maybe_extract(urls=urls, directory=directory, check_files=check_files)\n\n    ret = []\n    splits = [(train, train_filename), (test, test_filename)]\n    splits = [f for (requested, f) in splits if requested]\n    for filename in splits:\n        full_path = os.path.join(directory, filename)\n        examples = []\n        for line in open(full_path, 'rb'):\n            # there is one non-ASCII byte: sisterBADBYTEcity; replaced with space\n            label, _, text = line.replace(b'\\xf0', b' ').strip().decode().partition(' ')\n            label, _, label_fine = label.partition(':')\n            if fine_grained:\n                examples.append({'label': label_fine, 'text': text})\n            else:\n                examples.append({'label': label, 'text': text})\n        ret.append(Dataset(examples))\n\n    if len(ret) == 1:\n        return ret[0]\n    else:\n        return tuple(ret)", "response": "Load the TREC dataset."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nload the Stanford Natural Language Inference dataset.", "response": "def snli_dataset(directory='data/',\n                 train=False,\n                 dev=False,\n                 test=False,\n                 train_filename='snli_1.0_train.jsonl',\n                 dev_filename='snli_1.0_dev.jsonl',\n                 test_filename='snli_1.0_test.jsonl',\n                 extracted_name='snli_1.0',\n                 check_files=['snli_1.0/snli_1.0_train.jsonl'],\n                 url='http://nlp.stanford.edu/projects/snli/snli_1.0.zip'):\n    \"\"\"\n    Load the Stanford Natural Language Inference (SNLI) dataset.\n\n    The SNLI corpus (version 1.0) is a collection of 570k human-written English sentence pairs\n    manually labeled for balanced classification with the labels entailment, contradiction, and\n    neutral, supporting the task of natural language inference (NLI), also known as recognizing\n    textual entailment (RTE). We aim for it to serve both as a benchmark for evaluating\n    representational systems for text, especially including those induced by representation\n    learning methods, as well as a resource for developing NLP models of any kind.\n\n    **Reference:** https://nlp.stanford.edu/projects/snli/\n\n    **Citation:**\n    Samuel R. Bowman, Gabor Angeli, Christopher Potts, and Christopher D. Manning. 2015. A large\n    annotated corpus for learning natural language inference. In Proceedings of the 2015 Conference\n    on Empirical Methods in Natural Language Processing (EMNLP).\n\n    Args:\n        directory (str, optional): Directory to cache the dataset.\n        train (bool, optional): If to load the training split of the dataset.\n        dev (bool, optional): If to load the development split of the dataset.\n        test (bool, optional): If to load the test split of the dataset.\n        train_filename (str, optional): The filename of the training split.\n        dev_filename (str, optional): The filename of the development split.\n        test_filename (str, optional): The filename of the test split.\n        extracted_name (str, optional): Name of the extracted dataset directory.\n        check_files (str, optional): Check if these files exist, then this download was successful.\n        url (str, optional): URL of the dataset `tar.gz` file.\n\n    Returns:\n        :class:`tuple` of :class:`torchnlp.datasets.Dataset` or :class:`torchnlp.datasets.Dataset`:\n        Returns between one and all dataset splits (train, dev and test) depending on if their\n        respective boolean argument is ``True``.\n\n    Example:\n        >>> from torchnlp.datasets import snli_dataset  # doctest: +SKIP\n        >>> train = snli_dataset(train=True)  # doctest: +SKIP\n        >>> train[0]  # doctest: +SKIP\n        {\n          'premise': 'Kids are on a amusement ride.',\n          'hypothesis': 'A car is broke down on the side of the road.',\n          'label': 'contradiction',\n          'premise_transitions': ['shift', 'shift', 'shift', 'shift', 'shift', 'shift', ...],\n          'hypothesis_transitions': ['shift', 'shift', 'shift', 'shift', 'shift', 'shift', ...],\n        }\n    \"\"\"\n    download_file_maybe_extract(url=url, directory=directory, check_files=check_files)\n\n    get_transitions = lambda parse: ['reduce' if t == ')' else 'shift' for t in parse if t != '(']\n    ret = []\n    splits = [(train, train_filename), (dev, dev_filename), (test, test_filename)]\n    splits = [f for (requested, f) in splits if requested]\n    for filename in splits:\n        full_path = os.path.join(directory, extracted_name, filename)\n        examples = []\n        with io.open(full_path, encoding='utf-8') as f:\n            for line in f:\n                line = line.strip()\n                line = json.loads(line)\n                examples.append({\n                    'premise': line['sentence1'],\n                    'hypothesis': line['sentence2'],\n                    'label': line['gold_label'],\n                    'premise_transitions': get_transitions(line['sentence1_binary_parse']),\n                    'hypothesis_transitions': get_transitions(line['sentence2_binary_parse'])\n                })\n        ret.append(Dataset(examples))\n\n    if len(ret) == 1:\n        return ret[0]\n    else:\n        return tuple(ret)"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nforwards the attention mechanism to the next entry in the sequence of entries in the context.", "response": "def forward(self, query, context):\n        \"\"\"\n        Args:\n            query (:class:`torch.FloatTensor` [batch size, output length, dimensions]): Sequence of\n                queries to query the context.\n            context (:class:`torch.FloatTensor` [batch size, query length, dimensions]): Data\n                overwhich to apply the attention mechanism.\n\n        Returns:\n            :class:`tuple` with `output` and `weights`:\n            * **output** (:class:`torch.LongTensor` [batch size, output length, dimensions]):\n              Tensor containing the attended features.\n            * **weights** (:class:`torch.FloatTensor` [batch size, output length, query length]):\n              Tensor containing attention weights.\n        \"\"\"\n        batch_size, output_len, dimensions = query.size()\n        query_len = context.size(1)\n\n        if self.attention_type == \"general\":\n            query = query.view(batch_size * output_len, dimensions)\n            query = self.linear_in(query)\n            query = query.view(batch_size, output_len, dimensions)\n\n        # TODO: Include mask on PADDING_INDEX?\n\n        # (batch_size, output_len, dimensions) * (batch_size, query_len, dimensions) ->\n        # (batch_size, output_len, query_len)\n        attention_scores = torch.bmm(query, context.transpose(1, 2).contiguous())\n\n        # Compute weights across every context sequence\n        attention_scores = attention_scores.view(batch_size * output_len, query_len)\n        attention_weights = self.softmax(attention_scores)\n        attention_weights = attention_weights.view(batch_size, output_len, query_len)\n\n        # (batch_size, output_len, query_len) * (batch_size, query_len, dimensions) ->\n        # (batch_size, output_len, dimensions)\n        mix = torch.bmm(attention_weights, context)\n\n        # concat -> (batch_size * output_len, 2*dimensions)\n        combined = torch.cat((mix, query), dim=2)\n        combined = combined.view(batch_size * output_len, 2 * dimensions)\n\n        # Apply linear_out on every 2nd dimension of concat\n        # output -> (batch_size, output_len, dimensions)\n        output = self.linear_out(combined).view(batch_size, output_len, dimensions)\n        output = self.tanh(output)\n\n        return output, attention_weights"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nloading the IWSLT 2017 translation dataset.", "response": "def iwslt_dataset(\n        directory='data/iwslt/',\n        train=False,\n        dev=False,\n        test=False,\n        language_extensions=['en', 'de'],\n        train_filename='{source}-{target}/train.{source}-{target}.{lang}',\n        dev_filename='{source}-{target}/IWSLT16.TED.tst2013.{source}-{target}.{lang}',\n        test_filename='{source}-{target}/IWSLT16.TED.tst2014.{source}-{target}.{lang}',\n        check_files=['{source}-{target}/train.tags.{source}-{target}.{source}'],\n        url='https://wit3.fbk.eu/archive/2016-01/texts/{source}/{target}/{source}-{target}.tgz'):\n    \"\"\"\n    Load the International Workshop on Spoken Language Translation (IWSLT) 2017 translation dataset.\n\n    In-domain training, development and evaluation sets were supplied through the website of the\n    WIT3 project, while out-of-domain training data were linked in the workshop\u2019s website. With\n    respect to edition 2016 of the evaluation campaign, some of the talks added to the TED\n    repository during the last year have been used to define the evaluation sets (tst2017), while\n    the remaining new talks have been included in the training sets.\n\n    The English data that participants were asked to recognize and translate consists in part of\n    TED talks as in the years before, and in part of real-life lectures and talks that have been\n    mainly recorded in lecture halls at KIT and Carnegie Mellon University. TED talks are\n    challenging due to their variety in topics, but are very benign as they are very thoroughly\n    rehearsed and planned, leading to easy to recognize and translate language.\n\n    Note:\n        The order examples are returned is not guaranteed due to ``iglob``.\n\n    References:\n      * http://workshop2017.iwslt.org/downloads/iwslt2017_proceeding_v2.pdf\n      * http://workshop2017.iwslt.org/\n\n    **Citation:**\n    M. Cettolo, C. Girardi, and M. Federico. 2012. WIT3: Web Inventory of Transcribed and Translated\n    Talks. In Proc. of EAMT, pp. 261-268, Trento, Italy.\n\n    Args:\n        directory (str, optional): Directory to cache the dataset.\n        train (bool, optional): If to load the training split of the dataset.\n        dev (bool, optional): If to load the dev split of the dataset.\n        test (bool, optional): If to load the test split of the dataset.\n        language_extensions (:class:`list` of :class:`str`): Two language extensions\n            ['en'|'de'|'it'|'ni'|'ro'] to load.\n        train_filename (str, optional): The filename of the training split.\n        dev_filename (str, optional): The filename of the dev split.\n        test_filename (str, optional): The filename of the test split.\n        check_files (str, optional): Check if these files exist, then this download was successful.\n        url (str, optional): URL of the dataset file.\n\n    Returns:\n        :class:`tuple` of :class:`torchnlp.datasets.Dataset` or :class:`torchnlp.datasets.Dataset`:\n        Returns between one and all dataset splits (train, dev and test) depending on if their\n        respective boolean argument is ``True``.\n\n    Example:\n        >>> from torchnlp.datasets import iwslt_dataset  # doctest: +SKIP\n        >>> train = iwslt_dataset(train=True)  # doctest: +SKIP\n        >>> train[:2]  # doctest: +SKIP\n        [{\n          'en': \"David Gallo: This is Bill Lange. I'm Dave Gallo.\",\n          'de': 'David Gallo: Das ist Bill Lange. Ich bin Dave Gallo.'\n        }, {\n          'en': \"And we're going to tell you some stories from the sea here in video.\",\n          'de': 'Wir werden Ihnen einige Geschichten \u00fcber das Meer in Videoform erz\u00e4hlen.'\n        }]\n    \"\"\"\n    if len(language_extensions) != 2:\n        raise ValueError(\"`language_extensions` must be two language extensions \"\n                         \"['en'|'de'|'it'|'ni'|'ro'] to load.\")\n\n    # Format Filenames\n    source, target = tuple(language_extensions)\n    check_files = [s.format(source=source, target=target) for s in check_files]\n    url = url.format(source=source, target=target)\n\n    download_file_maybe_extract(url=url, directory=directory, check_files=check_files)\n\n    iwslt_clean(os.path.join(directory, '{source}-{target}'.format(source=source, target=target)))\n\n    ret = []\n    splits = [(train, train_filename), (dev, dev_filename), (test, test_filename)]\n    splits = [f for (requested, f) in splits if requested]\n    for filename in splits:\n        examples = []\n        for extension in language_extensions:\n            path = os.path.join(directory,\n                                filename.format(lang=extension, source=source, target=target))\n            with open(path, 'r', encoding='utf-8') as f:\n                language_specific_examples = [l.strip() for l in f]\n\n            if len(examples) == 0:\n                examples = [{} for _ in range(len(language_specific_examples))]\n            for i, example in enumerate(language_specific_examples):\n                examples[i][extension] = example\n\n        ret.append(Dataset(examples))\n\n    if len(ret) == 1:\n        return ret[0]\n    else:\n        return tuple(ret)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef encode(self, object_):\n        if self.enforce_reversible:\n            self.enforce_reversible = False\n            if self.decode(self.encode(object_)) != object_:\n                raise ValueError('Encoding is not reversible for \"%s\"' % object_)\n            self.enforce_reversible = True\n\n        return object_", "response": "Encodes an object into a base64 - encoded version of the object."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nencode a list of objects into a single list of bytes.", "response": "def batch_encode(self, iterator, *args, **kwargs):\n        \"\"\"\n        Args:\n            batch (list): Batch of objects to encode.\n            *args: Arguments passed to ``encode``.\n            **kwargs: Keyword arguments passed to ``encode``.\n\n        Returns:\n            list: Batch of encoded objects.\n        \"\"\"\n        return [self.encode(object_, *args, **kwargs) for object_ in iterator]"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ndecode an object. Args: object_ (object): Encoded object. Returns: object: Object decoded.", "response": "def decode(self, encoded):\n        \"\"\" Decodes an object.\n\n        Args:\n            object_ (object): Encoded object.\n\n        Returns:\n            object: Object decoded.\n        \"\"\"\n        if self.enforce_reversible:\n            self.enforce_reversible = False\n            if self.encode(self.decode(encoded)) != encoded:\n                raise ValueError('Decoding is not reversible for \"%s\"' % encoded)\n            self.enforce_reversible = True\n\n        return encoded"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ndecoding a list of encoded objects.", "response": "def batch_decode(self, iterator, *args, **kwargs):\n        \"\"\"\n        Args:\n            iterator (list): Batch of encoded objects.\n            *args: Arguments passed to ``decode``.\n            **kwargs: Keyword arguments passed to ``decode``.\n\n        Returns:\n            list: Batch of decoded objects.\n        \"\"\"\n        return [self.decode(encoded, *args, **kwargs) for encoded in iterator]"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nforward computation of the sequence.", "response": "def forward(self, tokens, mask=None):\n        \"\"\"\n        Args:\n            tokens (:class:`torch.FloatTensor` [batch_size, num_tokens, input_dim]): Sequence\n                matrix to encode.\n            mask (:class:`torch.FloatTensor`): Broadcastable matrix to `tokens` used as a mask.\n        Returns:\n            (:class:`torch.FloatTensor` [batch_size, output_dim]): Encoding of sequence.\n        \"\"\"\n        if mask is not None:\n            tokens = tokens * mask.unsqueeze(-1).float()\n\n        # Our input is expected to have shape `(batch_size, num_tokens, embedding_dim)`.  The\n        # convolution layers expect input of shape `(batch_size, in_channels, sequence_length)`,\n        # where the conv layer `in_channels` is our `embedding_dim`.  We thus need to transpose the\n        # tensor first.\n        tokens = torch.transpose(tokens, 1, 2)\n        # Each convolution layer returns output of size `(batch_size, num_filters, pool_length)`,\n        # where `pool_length = num_tokens - ngram_size + 1`.  We then do an activation function,\n        # then do max pooling over each filter for the whole input sequence.  Because our max\n        # pooling is simple, we just use `torch.max`.  The resultant tensor of has shape\n        # `(batch_size, num_conv_layers * num_filters)`, which then gets projected using the\n        # projection layer, if requested.\n\n        filter_outputs = []\n        for i in range(len(self._convolution_layers)):\n            convolution_layer = getattr(self, 'conv_layer_{}'.format(i))\n            filter_outputs.append(self._activation(convolution_layer(tokens)).max(dim=2)[0])\n\n        # Now we have a list of `num_conv_layers` tensors of shape `(batch_size, num_filters)`.\n        # Concatenating them gives us a tensor of shape\n        # `(batch_size, num_filters * num_conv_layers)`.\n        maxpool_output = torch.cat(\n            filter_outputs, dim=1) if len(filter_outputs) > 1 else filter_outputs[0]\n\n        if self.projection_layer:\n            result = self.projection_layer(maxpool_output)\n        else:\n            result = maxpool_output\n        return result"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _unescape_token(escaped_token):\n\n    def match(m):\n        if m.group(1) is None:\n            return u\"_\" if m.group(0) == u\"\\\\u\" else u\"\\\\\"\n\n        try:\n            return six.unichr(int(m.group(1)))\n        except (ValueError, OverflowError):\n            return \"\"\n\n    trimmed = escaped_token[:-1] if escaped_token.endswith(\"_\") else escaped_token\n    return _UNESCAPE_REGEX.sub(match, trimmed)", "response": "Unescapes a unicode string."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nconvert a list of tokens to a list of subtoken strings.", "response": "def _tokens_to_subtoken(self, tokens):\n        \"\"\" Converts a list of tokens to a list of subtoken.\n\n        Args:\n          tokens: a list of strings.\n        Returns:\n          a list of integers in the range [0, vocab_size)\n        \"\"\"\n        ret = []\n        for token in tokens:\n            ret.extend(\n                self._escaped_token_to_subtoken_strings(_escape_token(token, self._alphabet)))\n        return ret"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nconvert a list of subtoken to a list of tokens.", "response": "def _subtoken_to_tokens(self, subtokens):\n        \"\"\" Converts a list of subtoken to a list of tokens.\n\n        Args:\n          subtokens: a list of integers in the range [0, vocab_size)\n\n        Returns:\n          a list of strings.\n        \"\"\"\n        concatenated = \"\".join(subtokens)\n        split = concatenated.split(\"_\")\n        return [_unescape_token(t + \"_\") for t in split if t]"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _escaped_token_to_subtoken_strings(self, escaped_token):\n        # NOTE: This algorithm is greedy; it won't necessarily produce the \"best\"\n        # list of subtokens.\n        ret = []\n        start = 0\n        token_len = len(escaped_token)\n        while start < token_len:\n            for end in xrange(min(token_len, start + self._max_subtoken_len), start, -1):\n                subtoken = escaped_token[start:end]\n                if subtoken in self._all_subtoken_strings:\n                    ret.append(subtoken)\n                    start = end\n                    break\n\n            else:  # Did not break\n                # If there is no possible encoding of the escaped token then one of the\n                # characters in the token is not in the alphabet. This should be\n                # impossible and would be indicative of a bug.\n                assert False, \"Token substring not found in subtoken vocabulary.\"\n\n        return ret", "response": "Converts an escaped token string to a list of subtoken strings."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nbuild a SubwordTextTokenizer instance that has vocab_size near target_size.", "response": "def build_to_target_size_from_token_counts(cls,\n                                               target_size,\n                                               token_counts,\n                                               min_val,\n                                               max_val,\n                                               num_iterations=4):\n        \"\"\"Builds a SubwordTextTokenizer that has `vocab_size` near `target_size`.\n\n        Uses simple recursive binary search to find a minimum token count that most\n        closely matches the `target_size`.\n\n        Args:\n          target_size: Desired vocab_size to approximate.\n          token_counts: A dictionary of token counts, mapping string to int.\n          min_val: An integer; lower bound for the minimum token count.\n          max_val: An integer; upper bound for the minimum token count.\n          num_iterations: An integer; how many iterations of refinement.\n\n        Returns:\n          A SubwordTextTokenizer instance.\n\n        Raises:\n          ValueError: If `min_val` is greater than `max_val`.\n        \"\"\"\n        if min_val > max_val:\n            raise ValueError(\"Lower bound for the minimum token count \"\n                             \"is greater than the upper bound.\")\n\n        def bisect(min_val, max_val):\n            \"\"\"Bisection to find the right size.\"\"\"\n            present_count = (max_val + min_val) // 2\n            logger.info(\"Trying min_count %d\" % present_count)\n            subtokenizer = cls()\n            subtokenizer.build_from_token_counts(token_counts, present_count, num_iterations)\n            logger.info(\"min_count %d attained a %d vocab_size\", present_count,\n                        subtokenizer.vocab_size)\n\n            # If min_val == max_val, we can't do any better than this.\n            if subtokenizer.vocab_size == target_size or min_val >= max_val:\n                return subtokenizer\n\n            if subtokenizer.vocab_size > target_size:\n                other_subtokenizer = bisect(present_count + 1, max_val)\n            else:\n                other_subtokenizer = bisect(min_val, present_count - 1)\n\n            if other_subtokenizer is None:\n                return subtokenizer\n\n            if (abs(other_subtokenizer.vocab_size - target_size) <\n                    abs(subtokenizer.vocab_size - target_size)):\n                return other_subtokenizer\n            return subtokenizer\n\n        return bisect(min_val, max_val)"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ntrain a SubwordTextTokenizer based on a dictionary of word counts.", "response": "def build_from_token_counts(self, token_counts, min_count, num_iterations=4):\n        \"\"\"Train a SubwordTextTokenizer based on a dictionary of word counts.\n\n        Args:\n          token_counts: a dictionary of Unicode strings to int.\n          min_count: an integer - discard subtokens with lower counts.\n          num_iterations: an integer; how many iterations of refinement.\n        \"\"\"\n        self._init_alphabet_from_tokens(six.iterkeys(token_counts))\n\n        # Bootstrap the initial list of subtokens with the characters from the\n        # alphabet plus the escaping characters.\n        self._init_subtokens_from_list(list(self._alphabet))\n\n        # We build iteratively.  On each iteration, we segment all the words,\n        # then count the resulting potential subtokens, keeping the ones\n        # with high enough counts for our new vocabulary.\n        if min_count < 1:\n            min_count = 1\n        for i in xrange(num_iterations):\n\n            # Collect all substrings of the encoded token that break along current\n            # subtoken boundaries.\n            subtoken_counts = collections.defaultdict(int)\n            for token, count in six.iteritems(token_counts):\n                escaped_token = _escape_token(token, self._alphabet)\n                subtokens = self._escaped_token_to_subtoken_strings(escaped_token)\n                start = 0\n                for subtoken in subtokens:\n                    for end in xrange(start + 1, len(escaped_token) + 1):\n                        new_subtoken = escaped_token[start:end]\n                        subtoken_counts[new_subtoken] += count\n                    start += len(subtoken)\n\n            # Array of sets of candidate subtoken strings, by length.\n            len_to_subtoken_strings = []\n            for subtoken_string, count in six.iteritems(subtoken_counts):\n                lsub = len(subtoken_string)\n                if count >= min_count:\n                    while len(len_to_subtoken_strings) <= lsub:\n                        len_to_subtoken_strings.append(set())\n                    len_to_subtoken_strings[lsub].add(subtoken_string)\n\n            # Consider the candidates longest to shortest, so that if we accept\n            # a longer subtoken string, we can decrement the counts of its\n            # prefixes.\n            new_subtoken_strings = []\n            for lsub in xrange(len(len_to_subtoken_strings) - 1, 0, -1):\n                subtoken_strings = len_to_subtoken_strings[lsub]\n                for subtoken_string in subtoken_strings:\n                    count = subtoken_counts[subtoken_string]\n                    if count >= min_count:\n                        # Exclude alphabet tokens here, as they must be included later,\n                        # explicitly, regardless of count.\n                        if subtoken_string not in self._alphabet:\n                            new_subtoken_strings.append((count, subtoken_string))\n                        for l in xrange(1, lsub):\n                            subtoken_counts[subtoken_string[:l]] -= count\n\n            # Include the alphabet explicitly to guarantee all strings are\n            # encodable.\n            new_subtoken_strings.extend((subtoken_counts.get(a, 0), a) for a in self._alphabet)\n            new_subtoken_strings.sort(reverse=True)\n\n            # Reinitialize to the candidate vocabulary.\n            self._init_subtokens_from_list([subtoken for _, subtoken in new_subtoken_strings])"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _init_subtokens_from_list(self, subtoken_strings):\n        # we remember the maximum length of any subtoken to avoid having to\n        # check arbitrarily long strings.\n        self._all_subtoken_strings = set([s for s in subtoken_strings if s])\n        self._max_subtoken_len = max([len(s) for s in subtoken_strings])", "response": "Initialize token information from a list of subtoken strings."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _init_alphabet_from_tokens(self, tokens):\n        # Include all characters from all tokens in the alphabet to guarantee that\n        # any token can be encoded. Additionally, include all escaping\n        # characters.\n        self._alphabet = {c for token in tokens for c in token}\n        self._alphabet |= _ESCAPE_CHARS", "response": "Initialize the alphabet from an iterable of token or subtoken strings."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ngetting the BLEU score using the MOSES multi - BLEU script.", "response": "def get_moses_multi_bleu(hypotheses, references, lowercase=False):\n    \"\"\"Get the BLEU score using the moses `multi-bleu.perl` script.\n\n    **Script:**\n    https://raw.githubusercontent.com/moses-smt/mosesdecoder/master/scripts/generic/multi-bleu.perl\n\n    Args:\n      hypotheses (list of str): List of predicted values\n      references (list of str): List of target values\n      lowercase (bool): If true, pass the \"-lc\" flag to the `multi-bleu.perl` script\n\n    Returns:\n      (:class:`np.float32`) The BLEU score as a float32 value.\n\n    Example:\n\n      >>> hypotheses = [\n      ...   \"The brown fox jumps over the dog \u7b11\",\n      ...   \"The brown fox jumps over the dog 2 \u7b11\"\n      ... ]\n      >>> references = [\n      ...   \"The quick brown fox jumps over the lazy dog \u7b11\",\n      ...   \"The quick brown fox jumps over the lazy dog \u7b11\"\n      ... ]\n      >>> get_moses_multi_bleu(hypotheses, references, lowercase=True)\n      46.51\n    \"\"\"\n    if isinstance(hypotheses, list):\n        hypotheses = np.array(hypotheses)\n    if isinstance(references, list):\n        references = np.array(references)\n\n    if np.size(hypotheses) == 0:\n        return np.float32(0.0)\n\n    # Get MOSES multi-bleu script\n    try:\n        multi_bleu_path, _ = urllib.request.urlretrieve(\n            \"https://raw.githubusercontent.com/moses-smt/mosesdecoder/\"\n            \"master/scripts/generic/multi-bleu.perl\")\n        os.chmod(multi_bleu_path, 0o755)\n    except:\n        logger.warning(\"Unable to fetch multi-bleu.perl script\")\n        return None\n\n    # Dump hypotheses and references to tempfiles\n    hypothesis_file = tempfile.NamedTemporaryFile()\n    hypothesis_file.write(\"\\n\".join(hypotheses).encode(\"utf-8\"))\n    hypothesis_file.write(b\"\\n\")\n    hypothesis_file.flush()\n    reference_file = tempfile.NamedTemporaryFile()\n    reference_file.write(\"\\n\".join(references).encode(\"utf-8\"))\n    reference_file.write(b\"\\n\")\n    reference_file.flush()\n\n    # Calculate BLEU using multi-bleu script\n    with open(hypothesis_file.name, \"r\") as read_pred:\n        bleu_cmd = [multi_bleu_path]\n        if lowercase:\n            bleu_cmd += [\"-lc\"]\n        bleu_cmd += [reference_file.name]\n        try:\n            bleu_out = subprocess.check_output(bleu_cmd, stdin=read_pred, stderr=subprocess.STDOUT)\n            bleu_out = bleu_out.decode(\"utf-8\")\n            bleu_score = re.search(r\"BLEU = (.+?),\", bleu_out).group(1)\n            bleu_score = float(bleu_score)\n            bleu_score = np.float32(bleu_score)\n        except subprocess.CalledProcessError as error:\n            if error.output is not None:\n                logger.warning(\"multi-bleu.perl script returned non-zero exit code\")\n                logger.warning(error.output)\n            bleu_score = None\n\n    # Close temp files\n    hypothesis_file.close()\n    reference_file.close()\n\n    return bleu_score"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nloads the Universal Dependencies Treebank dataset.", "response": "def ud_pos_dataset(directory='data/',\n                   train=False,\n                   dev=False,\n                   test=False,\n                   train_filename='en-ud-tag.v2.train.txt',\n                   dev_filename='en-ud-tag.v2.dev.txt',\n                   test_filename='en-ud-tag.v2.test.txt',\n                   extracted_name='en-ud-v2',\n                   check_files=['en-ud-v2/en-ud-tag.v2.train.txt'],\n                   url='https://bitbucket.org/sivareddyg/public/downloads/en-ud-v2.zip'):\n    \"\"\"\n    Load the Universal Dependencies - English Dependency Treebank dataset.\n\n    Corpus of sentences annotated using Universal Dependencies annotation. The corpus comprises\n    254,830 words and 16,622 sentences, taken from various web media including weblogs, newsgroups,\n    emails, reviews, and Yahoo! answers.\n\n    References:\n        * http://universaldependencies.org/\n        * https://github.com/UniversalDependencies/UD_English\n\n    **Citation:**\n    Natalia Silveira and Timothy Dozat and Marie-Catherine de Marneffe and Samuel Bowman and\n    Miriam Connor and John Bauer and Christopher D. Manning (2014).\n    A Gold Standard Dependency Corpus for {E}nglish\n\n    Args:\n        directory (str, optional): Directory to cache the dataset.\n        train (bool, optional): If to load the training split of the dataset.\n        dev (bool, optional): If to load the development split of the dataset.\n        test (bool, optional): If to load the test split of the dataset.\n        train_filename (str, optional): The filename of the training split.\n        dev_filename (str, optional): The filename of the development split.\n        test_filename (str, optional): The filename of the test split.\n        extracted_name (str, optional): Name of the extracted dataset directory.\n        check_files (str, optional): Check if these files exist, then this download was successful.\n        url (str, optional): URL of the dataset `tar.gz` file.\n\n    Returns:\n        :class:`tuple` of :class:`torchnlp.datasets.Dataset` or :class:`torchnlp.datasets.Dataset`:\n        Returns between one and all dataset splits (train, dev and test) depending on if their\n        respective boolean argument is ``True``.\n\n    Example:\n        >>> from torchnlp.datasets import ud_pos_dataset  # doctest: +SKIP\n        >>> train = ud_pos_dataset(train=True)  # doctest: +SKIP\n        >>> train[17]  # doctest: +SKIP\n        {\n          'tokens': ['Guerrillas', 'killed', 'an', 'engineer', ',', 'Asi', 'Ali', ',', 'from',\n                     'Tikrit', '.'],\n          'ud_tags': ['NOUN', 'VERB', 'DET', 'NOUN', 'PUNCT', 'PROPN', 'PROPN', 'PUNCT', 'ADP',\n                      'PROPN', 'PUNCT'],\n          'ptb_tags': ['NNS', 'VBD', 'DT', 'NN', ',', 'NNP', 'NNP', ',', 'IN', 'NNP', '.']\n        }\n    \"\"\"\n    download_file_maybe_extract(url=url, directory=directory, check_files=check_files)\n\n    ret = []\n    splits = [(train, train_filename), (dev, dev_filename), (test, test_filename)]\n    splits = [f for (requested, f) in splits if requested]\n    for filename in splits:\n        full_path = os.path.join(directory, extracted_name, filename)\n        examples = []\n        with io.open(full_path, encoding='utf-8') as f:\n            sentence = {'tokens': [], 'ud_tags': [], 'ptb_tags': []}\n            for line in f:\n                line = line.strip()\n                if line == '' and len(sentence['tokens']) > 0:\n                    examples.append(sentence)\n                    sentence = {'tokens': [], 'ud_tags': [], 'ptb_tags': []}\n                elif line != '':\n                    token, ud_tag, ptb_tag = tuple(line.split('\\t'))\n                    sentence['tokens'].append(token)\n                    sentence['ud_tags'].append(ud_tag)\n                    sentence['ptb_tags'].append(ptb_tag)\n        ret.append(Dataset(examples))\n\n    if len(ret) == 1:\n        return ret[0]\n    else:\n        return tuple(ret)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef makedirs(name):\n\n    import os, errno\n\n    try:\n        os.makedirs(name)\n    except OSError as ex:\n        if ex.errno == errno.EEXIST and os.path.isdir(name):\n            # ignore existing directory\n            pass\n        else:\n            # a different error happened\n            raise", "response": "helper function for python 2 and 3 to call os. makedirs"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef collate_fn(batch, train=True):\n    premise_batch, _ = pad_batch([row['premise'] for row in batch])\n    hypothesis_batch, _ = pad_batch([row['hypothesis'] for row in batch])\n    label_batch = torch.stack([row['label'] for row in batch])\n\n    # PyTorch RNN requires batches to be transposed for speed and integration with CUDA\n    transpose = (lambda b: b.t_().squeeze(0).contiguous())\n\n    return (transpose(premise_batch), transpose(hypothesis_batch), transpose(label_batch))", "response": "Returns a list of tensors to a batch tensors"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef smt_dataset(directory='data/',\n                train=False,\n                dev=False,\n                test=False,\n                train_filename='train.txt',\n                dev_filename='dev.txt',\n                test_filename='test.txt',\n                extracted_name='trees',\n                check_files=['trees/train.txt'],\n                url='http://nlp.stanford.edu/sentiment/trainDevTestTrees_PTB.zip',\n                fine_grained=False,\n                subtrees=False):\n    \"\"\"\n    Load the Stanford Sentiment Treebank dataset.\n\n    Semantic word spaces have been very useful but cannot express the meaning of longer phrases in\n    a principled way. Further progress towards understanding compositionality in tasks such as\n    sentiment detection requires richer supervised training and evaluation resources and more\n    powerful models of composition. To remedy this, we introduce a Sentiment Treebank. It includes\n    fine grained sentiment labels for 215,154 phrases in the parse trees of 11,855 sentences and\n    presents new challenges for sentiment compositionality.\n\n    **Reference**:\n    https://nlp.stanford.edu/sentiment/index.html\n\n    **Citation:**\n    Richard Socher, Alex Perelygin, Jean Y. Wu, Jason Chuang, Christopher D. Manning,\n    Andrew Y. Ng and Christopher Potts. Recursive Deep Models for Semantic Compositionality Over a\n    Sentiment Treebank\n\n    Args:\n        directory (str, optional): Directory to cache the dataset.\n        train (bool, optional): If to load the training split of the dataset.\n        dev (bool, optional): If to load the development split of the dataset.\n        test (bool, optional): If to load the test split of the dataset.\n        train_filename (str, optional): The filename of the training split.\n        dev_filename (str, optional): The filename of the development split.\n        test_filename (str, optional): The filename of the test split.\n        extracted_name (str, optional): Name of the extracted dataset directory.\n        check_files (str, optional): Check if these files exist, then this download was successful.\n        url (str, optional): URL of the dataset `tar.gz` file.\n        subtrees (bool, optional): Whether to include sentiment-tagged subphrases in addition to\n            complete examples.\n        fine_grained (bool, optional): Whether to use 5-class instead of 3-class labeling.\n\n    Returns:\n        :class:`tuple` of :class:`torchnlp.datasets.Dataset`: Tuple with the training tokens, dev\n        tokens and test tokens in order if their respective boolean argument is true.\n\n    Example:\n        >>> from torchnlp.datasets import smt_dataset  # doctest: +SKIP\n        >>> train = smt_dataset(train=True)  # doctest: +SKIP\n        >>> train[5]  # doctest: +SKIP\n        {\n          'text': \"Whether or not you 're enlightened by any of Derrida 's lectures on ...\",\n          'label': 'positive'\n        }\n    \"\"\"\n    download_file_maybe_extract(url=url, directory=directory, check_files=check_files)\n\n    ret = []\n    splits = [(train, train_filename), (dev, dev_filename), (test, test_filename)]\n    splits = [f for (requested, f) in splits if requested]\n    for filename in splits:\n        full_path = os.path.join(directory, extracted_name, filename)\n        examples = []\n        with io.open(full_path, encoding='utf-8') as f:\n            for line in f:\n                line = line.strip()\n                if subtrees:\n                    examples.extend(parse_tree(line, subtrees=subtrees, fine_grained=fine_grained))\n                else:\n                    examples.append(parse_tree(line, subtrees=subtrees, fine_grained=fine_grained))\n        ret.append(Dataset(examples))\n\n    if len(ret) == 1:\n        return ret[0]\n    else:\n        return tuple(ret)", "response": "Loads the Sentiment Treebank dataset and returns a dict of Sentiment Treebank objects."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nloading the SimpleQuestions dataset.", "response": "def simple_qa_dataset(\n        directory='data/',\n        train=False,\n        dev=False,\n        test=False,\n        extracted_name='SimpleQuestions_v2',\n        train_filename='annotated_fb_data_train.txt',\n        dev_filename='annotated_fb_data_valid.txt',\n        test_filename='annotated_fb_data_test.txt',\n        check_files=['SimpleQuestions_v2/annotated_fb_data_train.txt'],\n        url='https://www.dropbox.com/s/tohrsllcfy7rch4/SimpleQuestions_v2.tgz?raw=1',\n):  # pragma: no cover\n    \"\"\"\n    Load the SimpleQuestions dataset.\n\n    Single-relation factoid questions (simple questions) are common in many settings\n    (e.g. Microsoft\u2019s search query logs and WikiAnswers questions). The SimpleQuestions dataset is\n    one of the most commonly used benchmarks for studying single-relation factoid questions.\n\n    **Reference:**\n    https://research.fb.com/publications/large-scale-simple-question-answering-with-memory-networks/\n\n    Args:\n        directory (str, optional): Directory to cache the dataset.\n        train (bool, optional): If to load the training split of the dataset.\n        dev (bool, optional): If to load the development split of the dataset.\n        test (bool, optional): If to load the test split of the dataset.\n        extracted_name (str, optional): Name of the extracted dataset directory.\n        train_filename (str, optional): The filename of the training split.\n        dev_filename (str, optional): The filename of the development split.\n        test_filename (str, optional): The filename of the test split.\n        check_files (str, optional): Check if these files exist, then this download was successful.\n        url (str, optional): URL of the dataset `tar.gz` file.\n\n    Returns:\n        :class:`tuple` of :class:`torchnlp.datasets.Dataset` or :class:`torchnlp.datasets.Dataset`:\n        Returns between one and all dataset splits (train, dev and test) depending on if their\n        respective boolean argument is ``True``.\n\n    Example:\n        >>> from torchnlp.datasets import simple_qa_dataset  # doctest: +SKIP\n        >>> train = simple_qa_dataset(train=True)  # doctest: +SKIP\n        SimpleQuestions_v2.tgz:  15%|\u258f| 62.3M/423M [00:09<00:41, 8.76MB/s]\n        >>> train[0:2]  # doctest: +SKIP\n        [{\n          'question': 'what is the book e about',\n          'relation': 'www.freebase.com/book/written_work/subjects',\n          'object': 'www.freebase.com/m/01cj3p',\n          'subject': 'www.freebase.com/m/04whkz5'\n        }, {\n          'question': 'to what release does the release track cardiac arrest come from',\n          'relation': 'www.freebase.com/music/release_track/release',\n          'object': 'www.freebase.com/m/0sjc7c1',\n          'subject': 'www.freebase.com/m/0tp2p24'\n        }]\n    \"\"\"\n    download_file_maybe_extract(url=url, directory=directory, check_files=check_files)\n\n    ret = []\n    splits = [(train, train_filename), (dev, dev_filename), (test, test_filename)]\n    splits = [f for (requested, f) in splits if requested]\n    for filename in splits:\n        full_path = os.path.join(directory, extracted_name, filename)\n        data = pd.read_csv(\n            full_path, header=None, sep='\\t', names=['subject', 'relation', 'object', 'question'])\n        ret.append(\n            Dataset([{\n                'question': row['question'],\n                'relation': row['relation'],\n                'object': row['object'],\n                'subject': row['subject'],\n            } for _, row in data.iterrows()]))\n\n    if len(ret) == 1:\n        return ret[0]\n    else:\n        return tuple(ret)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef forward(self, x):\n        if not self.training or not self.p:\n            return x\n        x = x.clone()\n        mask = x.new_empty(1, x.size(1), x.size(2), requires_grad=False).bernoulli_(1 - self.p)\n        mask = mask.div_(1 - self.p)\n        mask = mask.expand_as(x)\n        return x * mask", "response": "Forward computation of the n - grams."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef pad_tensor(tensor, length, padding_index=DEFAULT_PADDING_INDEX):\n    n_padding = length - tensor.shape[0]\n    assert n_padding >= 0\n    if n_padding == 0:\n        return tensor\n    padding = tensor.new(n_padding, *tensor.shape[1:]).fill_(padding_index)\n    return torch.cat((tensor, padding), dim=0)", "response": "Pad a tensor up to length with padding_index."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\npad a list of tensors with padding_index.", "response": "def stack_and_pad_tensors(batch, padding_index=DEFAULT_PADDING_INDEX, dim=0):\n    \"\"\" Pad a :class:`list` of ``tensors`` (``batch``) with ``padding_index``.\n\n    Args:\n        batch (:class:`list` of :class:`torch.Tensor`): Batch of tensors to pad.\n        padding_index (int, optional): Index to pad tensors with.\n        dim (int, optional): Dimension on to which to concatenate the batch of tensors.\n\n    Returns\n        torch.Tensor, torch.Tensor: Padded tensors and original lengths of tensors.\n    \"\"\"\n    lengths = [tensor.shape[0] for tensor in batch]\n    max_len = max(lengths)\n    padded = [pad_tensor(tensor, max_len, padding_index) for tensor in batch]\n    lengths = torch.tensor(lengths)\n    padded = torch.stack(padded, dim=dim).contiguous()\n    for _ in range(dim):\n        lengths = lengths.unsqueeze(0)\n    return padded, lengths"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn a batch of text encoded into a single tensor.", "response": "def batch_encode(self, iterator, *args, dim=0, **kwargs):\n        \"\"\"\n        Args:\n            iterator (iterator): Batch of text to encode.\n            *args: Arguments passed onto ``Encoder.__init__``.\n            dim (int, optional): Dimension along which to concatenate tensors.\n            **kwargs: Keyword arguments passed onto ``Encoder.__init__``.\n\n        Returns\n            torch.Tensor, list of int: Encoded and padded batch of sequences; Original lengths of\n                sequences.\n        \"\"\"\n        return stack_and_pad_tensors(\n            super().batch_encode(iterator), padding_index=self.padding_index, dim=dim)"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ndecodes a batch of encoded sequences.", "response": "def batch_decode(self, tensor, lengths, dim=0, *args, **kwargs):\n        \"\"\"\n        Args:\n            batch (list of :class:`torch.Tensor`): Batch of encoded sequences.\n            lengths (list of int): Original lengths of sequences.\n            dim (int, optional): Dimension along which to split tensors.\n            *args: Arguments passed to ``decode``.\n            **kwargs: Key word arguments passed to ``decode``.\n\n        Returns:\n            list: Batch of decoded sequences.\n        \"\"\"\n        return super().batch_decode(\n            [t.squeeze(0)[:l] for t, l in zip(tensor.split(1, dim=dim), lengths)])"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nencode a sequence into a new version of the sequence.", "response": "def encode(self, sequence):\n        \"\"\" Encodes a ``sequence``.\n\n        Args:\n            sequence (str): String ``sequence`` to encode.\n\n        Returns:\n            torch.Tensor: Encoding of the ``sequence``.\n        \"\"\"\n        sequence = super().encode(sequence)\n        sequence = self.tokenize(sequence)\n        vector = [self.stoi.get(token, self.unknown_index) for token in sequence]\n        if self.append_eos:\n            vector.append(self.eos_index)\n        return torch.tensor(vector)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef decode(self, encoded):\n        encoded = super().decode(encoded)\n        tokens = [self.itos[index] for index in encoded]\n        return self.detokenize(tokens)", "response": "Decodes a tensor into a sequence."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _reporthook(t):\n    last_b = [0]\n\n    def inner(b=1, bsize=1, tsize=None):\n        \"\"\"\n        Args:\n            b (int, optional): Number of blocks just transferred [default: 1].\n            bsize (int, optional): Size of each block (in tqdm units) [default: 1].\n            tsize (int, optional): Total size (in tqdm units). If [default: None] remains unchanged.\n        \"\"\"\n        if tsize is not None:\n            t.total = tsize\n        t.update((b - last_b[0]) * bsize)\n        last_b[0] = b\n\n    return inner", "response": "A function that returns a function that returns a tuple of tuples with the first element of the tuple as first argument and the second element as second argument."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ndownloads a file from Google drive.", "response": "def _download_file_from_drive(filename, url):  # pragma: no cover\n    \"\"\" Download filename from google drive unless it's already in directory.\n\n    Args:\n        filename (str): Name of the file to download to (do nothing if it already exists).\n        url (str): URL to download from.\n    \"\"\"\n    confirm_token = None\n\n    # Since the file is big, drive will scan it for virus and take it to a\n    # warning page. We find the confirm token on this page and append it to the\n    # URL to start the download process.\n    confirm_token = None\n    session = requests.Session()\n    response = session.get(url, stream=True)\n    for k, v in response.cookies.items():\n        if k.startswith(\"download_warning\"):\n            confirm_token = v\n\n    if confirm_token:\n        url = url + \"&confirm=\" + confirm_token\n\n    logger.info(\"Downloading %s to %s\" % (url, filename))\n\n    response = session.get(url, stream=True)\n    # Now begin the download.\n    chunk_size = 16 * 1024\n    with open(filename, \"wb\") as f:\n        for chunk in response.iter_content(chunk_size):\n            if chunk:\n                f.write(chunk)\n\n    # Print newline to clear the carriage return from the download progress\n    statinfo = os.stat(filename)\n    logger.info(\"Successfully downloaded %s, %s bytes.\" % (filename, statinfo.st_size))"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nextracting a compressed file to a directory.", "response": "def _maybe_extract(compressed_filename, directory, extension=None):\n    \"\"\" Extract a compressed file to ``directory``.\n\n    Args:\n        compressed_filename (str): Compressed file.\n        directory (str): Extract to directory.\n        extension (str, optional): Extension of the file; Otherwise, attempts to extract extension\n            from the filename.\n    \"\"\"\n    logger.info('Extracting {}'.format(compressed_filename))\n\n    if extension is None:\n        basename = os.path.basename(compressed_filename)\n        extension = basename.split('.', 1)[1]\n\n    if 'zip' in extension:\n        with zipfile.ZipFile(compressed_filename, \"r\") as zip_:\n            zip_.extractall(directory)\n    elif 'tar' in extension or 'tgz' in extension:\n        with tarfile.open(compressed_filename, mode='r') as tar:\n            tar.extractall(path=directory)\n\n    logger.info('Extracted {}'.format(compressed_filename))"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _get_filename_from_url(url):\n    parse = urlparse(url)\n    return os.path.basename(parse.path)", "response": "Return a filename from a URL"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ndownloading the file at url to directory if tar or zip.", "response": "def download_file_maybe_extract(url, directory, filename=None, extension=None, check_files=[]):\n    \"\"\" Download the file at ``url`` to ``directory``. Extract to ``directory`` if tar or zip.\n\n    Args:\n        url (str): Url of file.\n        directory (str): Directory to download to.\n        filename (str, optional): Name of the file to download; Otherwise, a filename is extracted\n            from the url.\n        extension (str, optional): Extension of the file; Otherwise, attempts to extract extension\n            from the filename.\n        check_files (list of str): Check if these files exist, ensuring the download succeeded.\n            If these files exist before the download, the download is skipped.\n\n    Returns:\n        (str): Filename of download file.\n\n    Raises:\n        ValueError: Error if one of the ``check_files`` are not found following the download.\n    \"\"\"\n    if filename is None:\n        filename = _get_filename_from_url(url)\n\n    filepath = os.path.join(directory, filename)\n    check_files = [os.path.join(directory, f) for f in check_files]\n\n    if len(check_files) > 0 and _check_download(*check_files):\n        return filepath\n\n    if not os.path.isdir(directory):\n        os.makedirs(directory)\n\n    logger.info('Downloading {}'.format(filename))\n\n    # Download\n    if 'drive.google.com' in url:\n        _download_file_from_drive(filepath, url)\n    else:\n        with tqdm(unit='B', unit_scale=True, miniters=1, desc=filename) as t:\n            urllib.request.urlretrieve(url, filename=filepath, reporthook=_reporthook(t))\n\n    _maybe_extract(compressed_filename=filepath, directory=directory, extension=extension)\n\n    if not _check_download(*check_files):\n        raise ValueError('[DOWNLOAD FAILED] `*check_files` not found')\n\n    return filepath"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef download_files_maybe_extract(urls, directory, check_files=[]):\n    check_files = [os.path.join(directory, f) for f in check_files]\n    if _check_download(*check_files):\n        return\n\n    for url in urls:\n        download_file_maybe_extract(url=url, directory=directory)\n\n    if not _check_download(*check_files):\n        raise ValueError('[DOWNLOAD FAILED] `*check_files` not found')", "response": "Download the files at urls to directory if tar or zip."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nget the accuracy between two tensors.", "response": "def get_accuracy(targets, outputs, k=1, ignore_index=None):\n    \"\"\" Get the accuracy top-k accuracy between two tensors.\n\n    Args:\n      targets (1 - 2D :class:`torch.Tensor`): Target or true vector against which to measure\n          saccuracy\n      outputs (1 - 3D :class:`torch.Tensor`): Prediction or output vector\n      ignore_index (int, optional): Specifies a target index that is ignored\n\n    Returns:\n      :class:`tuple` consisting of accuracy (:class:`float`), number correct (:class:`int`) and\n      total (:class:`int`)\n\n    Example:\n\n        >>> import torch\n        >>> from torchnlp.metrics import get_accuracy\n        >>> targets = torch.LongTensor([1, 2, 3, 4, 5])\n        >>> outputs = torch.LongTensor([1, 2, 2, 3, 5])\n        >>> accuracy, n_correct, n_total = get_accuracy(targets, outputs, ignore_index=3)\n        >>> accuracy\n        0.8\n        >>> n_correct\n        4\n        >>> n_total\n        5\n    \"\"\"\n    n_correct = 0.0\n    for target, output in zip(targets, outputs):\n        if not torch.is_tensor(target) or is_scalar(target):\n            target = torch.LongTensor([target])\n\n        if not torch.is_tensor(output) or is_scalar(output):\n            output = torch.LongTensor([[output]])\n\n        predictions = output.topk(k=min(k, len(output)), dim=0)[0]\n        for prediction in predictions:\n            if torch_equals_ignore_index(\n                    target.squeeze(), prediction.squeeze(), ignore_index=ignore_index):\n                n_correct += 1\n                break\n\n    return n_correct / len(targets), int(n_correct), len(targets)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_token_accuracy(targets, outputs, ignore_index=None):\n    n_correct = 0.0\n    n_total = 0.0\n    for target, output in zip(targets, outputs):\n        if not torch.is_tensor(target) or is_scalar(target):\n            target = torch.LongTensor([target])\n\n        if not torch.is_tensor(output) or is_scalar(output):\n            output = torch.LongTensor([[output]])\n\n        if len(target.size()) != len(output.size()):\n            prediction = output.max(dim=0)[0].view(-1)\n        else:\n            prediction = output\n\n        if ignore_index is not None:\n            mask = target.ne(ignore_index)\n            n_correct += prediction.eq(target).masked_select(mask).sum().item()\n            n_total += mask.sum().item()\n        else:\n            n_total += len(target)\n            n_correct += prediction.eq(target).sum().item()\n\n    return n_correct / n_total, n_correct, n_total", "response": "Get the accuracy token accuracy between two tensors."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get_tensors(object_):\n    if torch.is_tensor(object_):\n        return [object_]\n    elif isinstance(object_, (str, float, int)):\n        return []\n\n    tensors = set()\n\n    if isinstance(object_, collections.abc.Mapping):\n        for value in object_.values():\n            tensors.update(get_tensors(value))\n    elif isinstance(object_, collections.abc.Iterable):\n        for value in object_:\n            tensors.update(get_tensors(value))\n    else:\n        members = [\n            value for key, value in inspect.getmembers(object_)\n            if not isinstance(value, (collections.abc.Callable, type(None)))\n        ]\n        tensors.update(get_tensors(members))\n\n    return tensors", "response": "Returns a list of tensors that are associated with object_."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef sampler_to_iterator(dataset, sampler):\n    for sample in sampler:\n        if isinstance(sample, (list, tuple)):\n            # yield a batch\n            yield [dataset[i] for i in sample]\n        else:\n            # yield a single example\n            yield dataset[sample]", "response": "Given a batch sampler or sampler returns examples instead of indices\nAttributeNames"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef resplit_datasets(dataset, other_dataset, random_seed=None, split=None):\n    # Prevent circular dependency\n    from torchnlp.datasets import Dataset\n\n    concat = dataset.rows + other_dataset.rows\n    shuffle(concat, random_seed=random_seed)\n    if split is None:\n        return Dataset(concat[:len(dataset)]), Dataset(concat[len(dataset):])\n    else:\n        split = max(min(round(len(concat) * split), len(concat)), 0)\n        return Dataset(concat[:split]), Dataset(concat[split:])", "response": "Deterministic shuffle and split algorithm."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef torch_equals_ignore_index(tensor, tensor_other, ignore_index=None):\n    if ignore_index is not None:\n        assert tensor.size() == tensor_other.size()\n        mask_arr = tensor.ne(ignore_index)\n        tensor = tensor.masked_select(mask_arr)\n        tensor_other = tensor_other.masked_select(mask_arr)\n\n    return torch.equal(tensor, tensor_other)", "response": "Compute torch. equal with the optional mask parameter."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ngives a list of lengths create a batch mask.", "response": "def lengths_to_mask(*lengths, **kwargs):\n    \"\"\" Given a list of lengths, create a batch mask.\n\n    Example:\n        >>> lengths_to_mask([1, 2, 3])\n        tensor([[1, 0, 0],\n                [1, 1, 0],\n                [1, 1, 1]], dtype=torch.uint8)\n        >>> lengths_to_mask([1, 2, 2], [1, 2, 2])\n        tensor([[[1, 0],\n                 [0, 0]],\n        <BLANKLINE>\n                [[1, 1],\n                 [1, 1]],\n        <BLANKLINE>\n                [[1, 1],\n                 [1, 1]]], dtype=torch.uint8)\n\n    Args:\n        *lengths (list of int or torch.Tensor)\n        **kwargs: Keyword arguments passed to ``torch.zeros`` upon initially creating the returned\n          tensor.\n\n    Returns:\n        torch.ByteTensor\n    \"\"\"\n    # Squeeze to deal with random additional dimensions\n    lengths = [l.squeeze().tolist() if torch.is_tensor(l) else l for l in lengths]\n\n    # For cases where length is a scalar, this needs to convert it to a list.\n    lengths = [l if isinstance(l, list) else [l] for l in lengths]\n    assert all(len(l) == len(lengths[0]) for l in lengths)\n    batch_size = len(lengths[0])\n    other_dimensions = tuple([int(max(l)) for l in lengths])\n    mask = torch.zeros(batch_size, *other_dimensions, **kwargs)\n    for i, length in enumerate(zip(*tuple(lengths))):\n        mask[i][[slice(int(l)) for l in length]].fill_(1)\n    return mask.byte()"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncollates a list of types k with tensors.", "response": "def collate_tensors(batch, stack_tensors=torch.stack):\n    \"\"\" Collate a list of type ``k`` (dict, namedtuple, list, etc.) with tensors.\n\n    Inspired by:\n    https://github.com/pytorch/pytorch/blob/master/torch/utils/data/_utils/collate.py#L31\n\n    Args:\n        batch (list of k): List of rows of type ``k``.\n        stack_tensors (callable): Function to stack tensors into a batch.\n\n    Returns:\n        k: Collated batch of type ``k``.\n\n    Example use case:\n        This is useful with ``torch.utils.data.dataloader.DataLoader`` which requires a collate\n        function. Typically, when collating sequences you'd set\n        ``collate_fn=partial(collate_tensors, stack_tensors=encoders.text.stack_and_pad_tensors)``.\n\n    Example:\n\n        >>> import torch\n        >>> batch = [\n        ...   { 'column_a': torch.randn(5), 'column_b': torch.randn(5) },\n        ...   { 'column_a': torch.randn(5), 'column_b': torch.randn(5) },\n        ... ]\n        >>> collated = collate_tensors(batch)\n        >>> {k: t.size() for (k, t) in collated.items()}\n        {'column_a': torch.Size([2, 5]), 'column_b': torch.Size([2, 5])}\n    \"\"\"\n    if all([torch.is_tensor(b) for b in batch]):\n        return stack_tensors(batch)\n    if (all([isinstance(b, dict) for b in batch]) and\n            all([b.keys() == batch[0].keys() for b in batch])):\n        return {key: collate_tensors([d[key] for d in batch], stack_tensors) for key in batch[0]}\n    elif all([is_namedtuple(b) for b in batch]):  # Handle ``namedtuple``\n        return batch[0].__class__(**collate_tensors([b._asdict() for b in batch], stack_tensors))\n    elif all([isinstance(b, list) for b in batch]):\n        # Handle list of lists such each list has some column to be batched, similar to:\n        # [['a', 'b'], ['a', 'b']] \u2192 [['a', 'a'], ['b', 'b']]\n        transposed = zip(*batch)\n        return [collate_tensors(samples, stack_tensors) for samples in transposed]\n    else:\n        return batch"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\napply torch. Tensor. to a list of tensors and returns a new list of tuples.", "response": "def tensors_to(tensors, *args, **kwargs):\n    \"\"\" Apply ``torch.Tensor.to`` to tensors in a generic data structure.\n\n    Inspired by:\n    https://github.com/pytorch/pytorch/blob/master/torch/utils/data/_utils/collate.py#L31\n\n    Args:\n        tensors (tensor, dict, list, namedtuple or tuple): Data structure with tensor values to\n            move.\n        *args: Arguments passed to ``torch.Tensor.to``.\n        **kwargs: Keyword arguments passed to ``torch.Tensor.to``.\n\n    Example use case:\n        This is useful as a complementary function to ``collate_tensors``. Following collating,\n        it's important to move your tensors to the appropriate device.\n\n    Returns:\n        The inputted ``tensors`` with ``torch.Tensor.to`` applied.\n\n    Example:\n\n        >>> import torch\n        >>> batch = [\n        ...   { 'column_a': torch.randn(5), 'column_b': torch.randn(5) },\n        ...   { 'column_a': torch.randn(5), 'column_b': torch.randn(5) },\n        ... ]\n        >>> tensors_to(batch, torch.device('cpu'))  # doctest: +ELLIPSIS\n        [{'column_a': tensor(...}]\n    \"\"\"\n    if torch.is_tensor(tensors):\n        return tensors.to(*args, **kwargs)\n    elif isinstance(tensors, dict):\n        return {k: tensors_to(v, *args, **kwargs) for k, v in tensors.items()}\n    elif hasattr(tensors, '_asdict') and isinstance(tensors, tuple):  # Handle ``namedtuple``\n        return tensors.__class__(**tensors_to(tensors._asdict(), *args, **kwargs))\n    elif isinstance(tensors, list):\n        return [tensors_to(t, *args, **kwargs) for t in tensors]\n    elif isinstance(tensors, tuple):\n        return tuple([tensors_to(t, *args, **kwargs) for t in tensors])\n    else:\n        return tensors"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nload the Zero dataset.", "response": "def zero_dataset(train=False, dev=False, test=False, train_rows=256, dev_rows=64, test_rows=64):\n    \"\"\"\n    Load the Zero dataset.\n\n    The Zero dataset is a simple task of predicting zero from zero. This dataset is useful for\n    integration testing. The extreme simplicity of the dataset allows for models to learn the task\n    quickly allowing for quick end-to-end testing.\n\n    Args:\n        train (bool, optional): If to load the training split of the dataset.\n        dev (bool, optional): If to load the development split of the dataset.\n        test (bool, optional): If to load the test split of the dataset.\n        train_rows (int, optional): Number of training rows to generate.\n        dev_rows (int, optional): Number of development rows to generate.\n        test_rows (int, optional): Number of test rows to generate.\n\n    Returns:\n        :class:`tuple` of :class:`torchnlp.datasets.Dataset` or :class:`torchnlp.datasets.Dataset`:\n        Returns between one and all dataset splits (train, dev and test) depending on if their\n        respective boolean argument is ``True``.\n\n    Example:\n        >>> from torchnlp.datasets import zero_dataset\n        >>> train = zero_dataset(train=True)\n        >>> train[0:2]\n        [{'source': '0', 'target': '0'}, {'source': '0', 'target': '0'}]\n    \"\"\"\n    ret = []\n    for is_requested, n_rows in [(train, train_rows), (dev, dev_rows), (test, test_rows)]:\n        if not is_requested:\n            continue\n        rows = [{'source': str(0), 'target': str(0)} for i in range(n_rows)]\n        ret.append(Dataset(rows))\n\n    if len(ret) == 1:\n        return ret[0]\n    else:\n        return tuple(ret)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ndecode a tensor into a sequence.", "response": "def decode(self, encoded):\n        \"\"\" Decodes a tensor into a sequence.\n\n        Args:\n            encoded (torch.Tensor): Encoded sequence.\n\n        Returns:\n            str: Sequence decoded from ``encoded``.\n        \"\"\"\n        encoded = super().decode(encoded)\n        return self.tokenizer.decode([self.itos[index] for index in encoded])"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef encode(self, label):\n        label = super().encode(label)\n\n        return torch.tensor(self.stoi.get(label, self.unknown_index))", "response": "Encodes a label into a new version of the label."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef batch_encode(self, iterator, *args, dim=0, **kwargs):\n        return torch.stack(super().batch_encode(iterator, *args, **kwargs), dim=dim)", "response": "Returns a batch of encoded labels."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef decode(self, encoded):\n        encoded = super().decode(encoded)\n\n        if encoded.numel() > 1:\n            raise ValueError(\n                '``decode`` decodes one label at a time, use ``batch_decode`` instead.')\n\n        return self.itos[encoded.squeeze().item()]", "response": "Decodes an encoded label."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nbatch decoding of a single object.", "response": "def batch_decode(self, tensor, *args, dim=0, **kwargs):\n        \"\"\"\n        Args:\n            tensor (torch.Tensor): Batch of tensors.\n            *args: Arguments passed to ``Encoder.batch_decode``.\n            dim (int, optional): Dimension along which to split tensors.\n            **kwargs: Keyword arguments passed to ``Encoder.batch_decode``.\n\n        Returns:\n            list: Batch of decoded labels.\n        \"\"\"\n        return super().batch_decode([t.squeeze(0) for t in tensor.split(1, dim=dim)])"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nloading the Count dataset.", "response": "def count_dataset(train=False,\n                  dev=False,\n                  test=False,\n                  train_rows=10000,\n                  dev_rows=1000,\n                  test_rows=1000,\n                  seq_max_length=10):\n    \"\"\"\n    Load the Count dataset.\n\n    The Count dataset is a simple task of counting the number of integers in a sequence. This\n    dataset is useful for testing implementations of sequence to label models.\n\n    Args:\n        train (bool, optional): If to load the training split of the dataset.\n        dev (bool, optional): If to load the development split of the dataset.\n        test (bool, optional): If to load the test split of the dataset.\n        train_rows (int, optional): Number of training rows to generate.\n        dev_rows (int, optional): Number of development rows to generate.\n        test_rows (int, optional): Number of test rows to generate.\n        seq_max_length (int, optional): Maximum sequence length.\n\n    Returns:\n        :class:`tuple` of :class:`torchnlp.datasets.Dataset` or :class:`torchnlp.datasets.Dataset`:\n        Returns between one and all dataset splits (train, dev and test) depending on if their\n        respective boolean argument is ``True``.\n\n    Example:\n        >>> import random\n        >>> random.seed(321)\n        >>>\n        >>> from torchnlp.datasets import count_dataset\n        >>> train = count_dataset(train=True)\n        >>> train[0:2]\n        [{'numbers': '6 2 5 8 7', 'count': '5'}, {'numbers': '3 9 7 6 6 7', 'count': '6'}]\n    \"\"\"\n    ret = []\n    for is_requested, n_rows in [(train, train_rows), (dev, dev_rows), (test, test_rows)]:\n        rows = []\n        for i in range(n_rows):\n            length = random.randint(1, seq_max_length)\n            seq = []\n            for _ in range(length):\n                seq.append(str(random.randint(0, 9)))\n            input_ = ' '.join(seq)\n            rows.append({'numbers': input_, 'count': str(length)})\n\n        # NOTE: Given that `random.randint` is deterministic with the same `random_seed` we need\n        # to allow the random generator to create the train, dev and test dataset in order.\n        # Otherwise, `reverse(train=True)` and `reverse(test=True)` would share the first 1000 rows.\n        if not is_requested:\n            continue\n\n        ret.append(Dataset(rows))\n\n    if len(ret) == 1:\n        return ret[0]\n    else:\n        return tuple(ret)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef multi30k_dataset(directory='data/multi30k/',\n                     train=False,\n                     dev=False,\n                     test=False,\n                     train_filename='train',\n                     dev_filename='val',\n                     test_filename='test',\n                     check_files=['train.de', 'val.de'],\n                     urls=[\n                         'http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/training.tar.gz',\n                         'http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/validation.tar.gz',\n                         'http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/mmt16_task1_test.tar.gz'\n                     ]):\n    \"\"\"\n    Load the WMT 2016 machine translation dataset.\n\n    As a translation task, this task consists in translating English sentences that describe an\n    image into German, given the English sentence itself. As training and development data, we\n    provide 29,000 and 1,014 triples respectively, each containing an English source sentence, its\n    German human translation. As test data, we provide a new set of 1,000 tuples containing an\n    English description.\n\n    Status:\n        Host ``www.quest.dcs.shef.ac.uk`` forgot to update their SSL\n        certificate; therefore, this dataset does not download securely.\n\n    References:\n        * http://www.statmt.org/wmt16/multimodal-task.html\n        * http://shannon.cs.illinois.edu/DenotationGraph/\n\n    **Citation**\n    ::\n\n        @article{elliott-EtAl:2016:VL16,\n            author    = {{Elliott}, D. and {Frank}, S. and {Sima'an}, K. and {Specia}, L.},\n            title     = {Multi30K: Multilingual English-German Image Descriptions},\n            booktitle = {Proceedings of the 5th Workshop on Vision and Language},\n            year      = {2016},\n            pages     = {70--74},\n            year      = 2016\n        }\n\n    Args:\n        directory (str, optional): Directory to cache the dataset.\n        train (bool, optional): If to load the training split of the dataset.\n        dev (bool, optional): If to load the dev split of the dataset.\n        test (bool, optional): If to load the test split of the dataset.\n        train_directory (str, optional): The directory of the training split.\n        dev_directory (str, optional): The directory of the dev split.\n        test_directory (str, optional): The directory of the test split.\n        check_files (str, optional): Check if these files exist, then this download was successful.\n        urls (str, optional): URLs to download.\n\n    Returns:\n        :class:`tuple` of :class:`torchnlp.datasets.Dataset` or :class:`torchnlp.datasets.Dataset`:\n        Returns between one and all dataset splits (train, dev and test) depending on if their\n        respective boolean argument is ``True``.\n\n    Example:\n        >>> from torchnlp.datasets import multi30k_dataset  # doctest: +SKIP\n        >>> train = multi30k_dataset(train=True)  # doctest: +SKIP\n        >>> train[:2]  # doctest: +SKIP\n        [{\n          'en': 'Two young, White males are outside near many bushes.',\n          'de': 'Zwei junge wei\u00dfe M\u00e4nner sind im Freien in der N\u00e4he vieler B\u00fcsche.'\n        }, {\n          'en': 'Several men in hard hatsare operating a giant pulley system.',\n          'de': 'Mehrere M\u00e4nner mit Schutzhelmen bedienen ein Antriebsradsystem.'\n        }]\n    \"\"\"\n    download_files_maybe_extract(urls=urls, directory=directory, check_files=check_files)\n\n    ret = []\n    splits = [(train, train_filename), (dev, dev_filename), (test, test_filename)]\n    splits = [f for (requested, f) in splits if requested]\n\n    for filename in splits:\n        examples = []\n\n        en_path = os.path.join(directory, filename + '.en')\n        de_path = os.path.join(directory, filename + '.de')\n        en_file = [l.strip() for l in open(en_path, 'r', encoding='utf-8')]\n        de_file = [l.strip() for l in open(de_path, 'r', encoding='utf-8')]\n        assert len(en_file) == len(de_file)\n        for i in range(len(en_file)):\n            if en_file[i] != '' and de_file[i] != '':\n                examples.append({'en': en_file[i], 'de': de_file[i]})\n\n        ret.append(Dataset(examples))\n\n    if len(ret) == 1:\n        return ret[0]\n    else:\n        return tuple(ret)", "response": "Load the multi30k dataset."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nload a Penn Treebank dataset.", "response": "def penn_treebank_dataset(\n        directory='data/penn-treebank',\n        train=False,\n        dev=False,\n        test=False,\n        train_filename='ptb.train.txt',\n        dev_filename='ptb.valid.txt',\n        test_filename='ptb.test.txt',\n        check_files=['ptb.train.txt'],\n        urls=[\n            'https://raw.githubusercontent.com/wojzaremba/lstm/master/data/ptb.train.txt',\n            'https://raw.githubusercontent.com/wojzaremba/lstm/master/data/ptb.valid.txt',\n            'https://raw.githubusercontent.com/wojzaremba/lstm/master/data/ptb.test.txt'\n        ],\n        unknown_token=DEFAULT_UNKNOWN_TOKEN,\n        eos_token=DEFAULT_EOS_TOKEN):\n    \"\"\"\n    Load the Penn Treebank dataset.\n\n    This is the Penn Treebank Project: Release 2 CDROM, featuring a million words of 1989 Wall\n    Street Journal material.\n\n    **Reference:** https://catalog.ldc.upenn.edu/ldc99t42\n\n    **Citation:**\n    Marcus, Mitchell P., Marcinkiewicz, Mary Ann & Santorini, Beatrice (1993).\n    Building a Large Annotated Corpus of English: The Penn Treebank\n\n    Args:\n        directory (str, optional): Directory to cache the dataset.\n        train (bool, optional): If to load the training split of the dataset.\n        dev (bool, optional): If to load the development split of the dataset.\n        test (bool, optional): If to load the test split of the dataset.\n        train_filename (str, optional): The filename of the training split.\n        dev_filename (str, optional): The filename of the development split.\n        test_filename (str, optional): The filename of the test split.\n        name (str, optional): Name of the dataset directory.\n        check_files (str, optional): Check if these files exist, then this download was successful.\n        urls (str, optional): URLs to download.\n        unknown_token (str, optional): Token to use for unknown words.\n        eos_token (str, optional): Token to use at the end of sentences.\n\n    Returns:\n        :class:`tuple` of :class:`torchnlp.datasets.Dataset` or :class:`torchnlp.datasets.Dataset`:\n        Returns between one and all dataset splits (train, dev and test) depending on if their\n        respective boolean argument is ``True``.\n\n    Example:\n        >>> from torchnlp.datasets import penn_treebank_dataset  # doctest: +SKIP\n        >>> train = penn_treebank_dataset(train=True)  # doctest: +SKIP\n        >>> train[:10]  # doctest: +SKIP\n        ['aer', 'banknote', 'berlitz', 'calloway', 'centrust', 'cluett', 'fromstein', 'gitano',\n        'guterman', 'hydro-quebec']\n    \"\"\"\n    download_files_maybe_extract(urls=urls, directory=directory, check_files=check_files)\n\n    ret = []\n    splits = [(train, train_filename), (dev, dev_filename), (test, test_filename)]\n    splits = [f for (requested, f) in splits if requested]\n    for filename in splits:\n        full_path = os.path.join(directory, filename)\n        text = []\n        with io.open(full_path, encoding='utf-8') as f:\n            for line in f:\n                text.extend(line.replace('<unk>', unknown_token).split())\n                text.append(eos_token)\n        ret.append(text)\n\n    if len(ret) == 1:\n        return ret[0]\n    else:\n        return tuple(ret)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef repackage_hidden(h):\n    if isinstance(h, torch.Tensor):\n        return h.detach()\n    else:\n        return tuple(repackage_hidden(v) for v in h)", "response": "Wraps hidden states in new Tensors to detach them from their history."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef parse_to_ast(source_code: str) -> List[ast.stmt]:\n    class_types, reformatted_code = pre_parse(source_code)\n\n    if '\\x00' in reformatted_code:\n        raise ParserException('No null bytes (\\\\x00) allowed in the source code.')\n\n    parsed_ast = ast.parse(reformatted_code)\n    annotate_and_optimize_ast(parsed_ast, reformatted_code, class_types)\n\n    return parsed_ast.body", "response": "Parses the given vyper source code and returns a list of python AST objects for each statement in the source code."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef pack_args_by_32(holder, maxlen, arg, typ, context, placeholder,\n                    dynamic_offset_counter=None, datamem_start=None, zero_pad_i=None, pos=None):\n    \"\"\"\n    Copy necessary variables to pre-allocated memory section.\n\n    :param holder: Complete holder for all args\n    :param maxlen: Total length in bytes of the full arg section (static + dynamic).\n    :param arg: Current arg to pack\n    :param context: Context of arg\n    :param placeholder: Static placeholder for static argument part.\n    :param dynamic_offset_counter: position counter stored in static args.\n    :param dynamic_placeholder: pointer to current position in memory to write dynamic values to.\n    :param datamem_start: position where the whole datemem section starts.\n    \"\"\"\n\n    if isinstance(typ, BaseType):\n        if isinstance(arg, LLLnode):\n            value = unwrap_location(arg)\n        else:\n            value = parse_expr(arg, context)\n            value = base_type_conversion(value, value.typ, typ, pos)\n        holder.append(LLLnode.from_list(['mstore', placeholder, value], typ=typ, location='memory'))\n    elif isinstance(typ, ByteArrayLike):\n\n        if isinstance(arg, LLLnode):  # Is prealloacted variable.\n            source_lll = arg\n        else:\n            source_lll = parse_expr(arg, context)\n\n        # Set static offset, in arg slot.\n        holder.append(LLLnode.from_list(['mstore', placeholder, ['mload', dynamic_offset_counter]]))\n        # Get the biginning to write the ByteArray to.\n        dest_placeholder = LLLnode.from_list(\n            ['add', datamem_start, ['mload', dynamic_offset_counter]],\n            typ=typ, location='memory', annotation=\"pack_args_by_32:dest_placeholder\")\n        copier = make_byte_array_copier(dest_placeholder, source_lll, pos=pos)\n        holder.append(copier)\n        # Add zero padding.\n        new_maxlen = ceil32(source_lll.typ.maxlen)\n\n        holder.append([\n            'with', '_ceil32_end', ['ceil32', ['mload', dest_placeholder]], [\n                'seq', ['with', '_bytearray_loc', dest_placeholder, [\n                    'seq', ['repeat', zero_pad_i, ['mload', '_bytearray_loc'], new_maxlen, [\n                        'seq',\n                        # stay within allocated bounds\n                        ['if', ['ge', ['mload', zero_pad_i], '_ceil32_end'], 'break'],\n                        [\n                            'mstore8',\n                            ['add', ['add', '_bytearray_loc', 32], ['mload', zero_pad_i]],\n                            0,\n                        ],\n                    ]],\n                ]],\n            ]\n        ])\n\n        # Increment offset counter.\n        increment_counter = LLLnode.from_list([\n            'mstore', dynamic_offset_counter,\n            [\n                'add',\n                ['add', ['mload', dynamic_offset_counter], ['ceil32', ['mload', dest_placeholder]]],\n                32,\n            ],\n        ], annotation='Increment dynamic offset counter')\n        holder.append(increment_counter)\n    elif isinstance(typ, ListType):\n        maxlen += (typ.count - 1) * 32\n        typ = typ.subtype\n\n        def check_list_type_match(provided):  # Check list types match.\n            if provided != typ:\n                raise TypeMismatchException(\n                    \"Log list type '%s' does not match provided, expected '%s'\" % (provided, typ)\n                )\n\n        # List from storage\n        if isinstance(arg, ast.Attribute) and arg.value.id == 'self':\n            stor_list = context.globals[arg.attr]\n            check_list_type_match(stor_list.typ.subtype)\n            size = stor_list.typ.count\n            mem_offset = 0\n            for i in range(0, size):\n                storage_offset = i\n                arg2 = LLLnode.from_list(\n                    ['sload', ['add', ['sha3_32', Expr(arg, context).lll_node], storage_offset]],\n                    typ=typ,\n                )\n                holder, maxlen = pack_args_by_32(\n                    holder,\n                    maxlen,\n                    arg2,\n                    typ,\n                    context,\n                    placeholder + mem_offset,\n                    pos=pos,\n                )\n                mem_offset += get_size_of_type(typ) * 32\n\n        # List from variable.\n        elif isinstance(arg, ast.Name):\n            size = context.vars[arg.id].size\n            pos = context.vars[arg.id].pos\n            check_list_type_match(context.vars[arg.id].typ.subtype)\n            mem_offset = 0\n            for _ in range(0, size):\n                arg2 = LLLnode.from_list(pos + mem_offset, typ=typ, location='memory')\n                holder, maxlen = pack_args_by_32(\n                    holder,\n                    maxlen,\n                    arg2,\n                    typ,\n                    context,\n                    placeholder + mem_offset,\n                    pos=pos,\n                )\n                mem_offset += get_size_of_type(typ) * 32\n\n        # List from list literal.\n        else:\n            mem_offset = 0\n            for arg2 in arg.elts:\n                holder, maxlen = pack_args_by_32(\n                    holder,\n                    maxlen,\n                    arg2,\n                    typ,\n                    context,\n                    placeholder + mem_offset,\n                    pos=pos,\n                )\n                mem_offset += get_size_of_type(typ) * 32\n    return holder, maxlen", "response": "Packs the arguments into a new memory section."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef validate_version_pragma(version_str: str, start: ParserPosition) -> None:\n    from vyper import (\n        __version__,\n    )\n\n    version_arr = version_str.split('@version')\n\n    file_version = version_arr[1].strip()\n    file_major, file_minor, file_patch = _parse_version_str(file_version, start)\n    compiler_major, compiler_minor, compiler_patch = _parse_version_str(__version__, start)\n\n    if (file_major, file_minor) != (compiler_major, compiler_minor):\n        raise VersionException(\n            f'File version \"{file_version}\" is not compatible '\n            f'with compiler version \"{__version__}\"',\n            start,\n        )", "response": "Validates a version pragma directive against the current compiler version."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef pre_parse(code: str) -> Tuple[ClassTypes, str]:\n    result = []\n    previous_keyword = None\n    class_types: ClassTypes = {}\n\n    try:\n        code_bytes = code.encode('utf-8')\n        g = tokenize(io.BytesIO(code_bytes).readline)\n\n        for token in g:\n            toks = [token]\n\n            typ = token.type\n            string = token.string\n            start = token.start\n            end = token.end\n            line = token.line\n\n            if typ == COMMENT and \"@version\" in string:\n                validate_version_pragma(string[1:], start)\n\n            if typ == NAME and string == \"class\" and start[1] == 0:\n                raise StructureException(\n                    \"The `class` keyword is not allowed. Perhaps you meant `contract` or `struct`?\",\n                    start,\n                )\n\n            # Make note of contract or struct name along with the type keyword\n            # that preceded it\n            if typ == NAME and previous_keyword is not None:\n                class_types[string] = previous_keyword\n                previous_keyword = None\n\n            # Translate vyper-specific class keywords into python \"class\"\n            # keyword\n            if typ == NAME and string in VYPER_CLASS_TYPES and start[1] == 0:\n                toks = [TokenInfo(NAME, \"class\", start, end, line)]\n                previous_keyword = string\n\n            if (typ, string) == (OP, \";\"):\n                raise StructureException(\"Semi-colon statements not allowed.\", start)\n\n            result.extend(toks)\n    except TokenError as e:\n        raise StructureException(e.args[0], e.args[1]) from e\n\n    return class_types, untokenize(result).decode('utf-8')", "response": "Re - formats a vyper source string into a python source string and performs some validation."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nchecks if a variable name is valid.", "response": "def is_valid_varname(self, name, item):\n        \"\"\" Valid variable name, checked against global context. \"\"\"\n        check_valid_varname(name, self._custom_units, self._structs, self._constants, item)\n        if name in self._globals:\n            raise VariableDeclarationException(\n                'Invalid name \"%s\", previously defined as global.' % name, item\n            )\n        return True"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ngets the counter for a nonrentrant lock.", "response": "def get_nonrentrant_counter(self, key):\n        \"\"\"\n        Nonrentrant locks use a prefix with a counter to minimise deployment cost of a contract.\n        \"\"\"\n        prefix = NONRENTRANT_STORAGE_OFFSET\n\n        if key in self._nonrentrant_keys:\n            return self._nonrentrant_keys[key]\n        else:\n            counter = prefix + self._nonrentrant_counter\n            self._nonrentrant_keys[key] = counter\n            self._nonrentrant_counter += 1\n            return counter"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ntest if the current statement is a type of list.", "response": "def _is_list_iter(self):\n        \"\"\"\n        Test if the current statement is a type of list, used in for loops.\n        \"\"\"\n\n        # Check for literal or memory list.\n        iter_var_type = (\n            self.context.vars.get(self.stmt.iter.id).typ\n            if isinstance(self.stmt.iter, ast.Name)\n            else None\n        )\n        if isinstance(self.stmt.iter, ast.List) or isinstance(iter_var_type, ListType):\n            return True\n\n        # Check for storage list.\n        if isinstance(self.stmt.iter, ast.Attribute):\n            iter_var_type = self.context.globals.get(self.stmt.iter.attr)\n            if iter_var_type and isinstance(iter_var_type.typ, ListType):\n                return True\n\n        return False"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nlooks up the most accurate signature for a given method name in a given context.", "response": "def lookup_sig(cls, sigs, method_name, expr_args, stmt_or_expr, context):\n        \"\"\"\n        Using a list of args, determine the most accurate signature to use from\n        the given context\n        \"\"\"\n\n        def synonymise(s):\n            return s.replace('int128', 'num').replace('uint256', 'num')\n\n        # for sig in sigs['self']\n        full_sig = cls.get_full_sig(\n            stmt_or_expr.func.attr,\n            expr_args,\n            None,\n            context.custom_units,\n            context.structs,\n            context.constants,\n        )\n        method_names_dict = dict(Counter([x.split('(')[0] for x in context.sigs['self']]))\n        if method_name not in method_names_dict:\n            raise FunctionDeclarationException(\n                \"Function not declared yet (reminder: functions cannot \"\n                \"call functions later in code than themselves): %s\" % method_name\n            )\n\n        if method_names_dict[method_name] == 1:\n            return next(\n                sig\n                for name, sig\n                in context.sigs['self'].items()\n                if name.split('(')[0] == method_name\n            )\n        if full_sig in context.sigs['self']:\n            return context.sigs['self'][full_sig]\n        else:\n            synonym_sig = synonymise(full_sig)\n            syn_sigs_test = [synonymise(k) for k in context.sigs.keys()]\n            if len(syn_sigs_test) != len(set(syn_sigs_test)):\n                raise Exception(\n                    'Incompatible default parameter signature,'\n                    'can not tell the number type of literal', stmt_or_expr\n                )\n            synonym_sigs = [(synonymise(k), v) for k, v in context.sigs['self'].items()]\n            ssig = [s[1] for s in synonym_sigs if s[0] == synonym_sig]\n            if len(ssig) == 0:\n                raise FunctionDeclarationException(\n                    \"Function not declared yet (reminder: functions cannot \"\n                    \"call functions later in code than themselves): %s\" % method_name\n                )\n            return ssig[0]"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef annotate_and_optimize_ast(\n    parsed_ast: ast.Module,\n    source_code: str,\n    class_types: Optional[ClassTypes] = None,\n) -> None:\n    \"\"\"\n    Performs annotation and optimization on a parsed python AST by doing the\n    following:\n\n    * Annotating all AST nodes with the originating source code of the AST\n    * Annotating class definition nodes with their original class type\n      (\"contract\" or \"struct\")\n    * Substituting negative values for unary subtractions\n\n    :param parsed_ast: The AST to be annotated and optimized.\n    :param source_code: The originating source code of the AST.\n    :param class_types: A mapping of class names to original class types.\n    :return: The annotated and optmized AST.\n    \"\"\"\n    AnnotatingVisitor(source_code, class_types).visit(parsed_ast)\n    RewriteUnarySubVisitor().visit(parsed_ast)\n    EnsureSingleExitChecker().visit(parsed_ast)", "response": "Annotates the parsed python AST with the source code of the source code."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef check_valid_varname(varname,\n                        custom_units,\n                        custom_structs,\n                        constants,\n                        pos,\n                        error_prefix=\"Variable name invalid.\",\n                        exc=None):\n    \"\"\" Handle invalid variable names \"\"\"\n    exc = VariableDeclarationException if exc is None else exc\n\n    valid_varname, msg = is_varname_valid(varname, custom_units, custom_structs, constants)\n    if not valid_varname:\n        raise exc(error_prefix + msg, pos)\n\n    return True", "response": "Check if a variable name is valid."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nallocates a MID which has not been used yet.", "response": "def allocate_mid(mids):\n    \"\"\"\n    Allocate a MID which has not been used yet.\n    \"\"\"\n    i = 0\n    while True:\n        mid = str(i)\n        if mid not in mids:\n            mids.add(mid)\n            return mid\n        i += 1"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nadd a new RTCIceCandidate received from the remote peer.", "response": "def addIceCandidate(self, candidate):\n        \"\"\"\n        Add a new :class:`RTCIceCandidate` received from the remote peer.\n\n        The specified candidate must have a value for either `sdpMid` or `sdpMLineIndex`.\n        \"\"\"\n        if candidate.sdpMid is None and candidate.sdpMLineIndex is None:\n            raise ValueError('Candidate must have either sdpMid or sdpMLineIndex')\n\n        for transceiver in self.__transceivers:\n            if candidate.sdpMid == transceiver.mid and not transceiver._bundled:\n                iceTransport = transceiver._transport.transport\n                iceTransport.addRemoteCandidate(candidate)\n                return\n\n        if self.__sctp and candidate.sdpMid == self.__sctp.mid and not self.__sctp._bundled:\n            iceTransport = self.__sctp.transport.transport\n            iceTransport.addRemoteCandidate(candidate)"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef addTrack(self, track):\n        # check state is valid\n        self.__assertNotClosed()\n        if track.kind not in ['audio', 'video']:\n            raise InternalError('Invalid track kind \"%s\"' % track.kind)\n\n        # don't add track twice\n        self.__assertTrackHasNoSender(track)\n\n        for transceiver in self.__transceivers:\n            if transceiver.kind == track.kind:\n                if transceiver.sender.track is None:\n                    transceiver.sender.replaceTrack(track)\n                    transceiver.direction = or_direction(transceiver.direction, 'sendonly')\n                    return transceiver.sender\n\n        transceiver = self.__createTransceiver(\n            direction='sendrecv',\n            kind=track.kind,\n            sender_track=track)\n        return transceiver.sender", "response": "Adds a MediaStreamTrack to the set of media tracks which can be transmitted to the remote peer."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef addTransceiver(self, trackOrKind, direction='sendrecv'):\n        self.__assertNotClosed()\n\n        # determine track or kind\n        if hasattr(trackOrKind, 'kind'):\n            kind = trackOrKind.kind\n            track = trackOrKind\n        else:\n            kind = trackOrKind\n            track = None\n        if kind not in ['audio', 'video']:\n            raise InternalError('Invalid track kind \"%s\"' % kind)\n\n        # check direction\n        if direction not in sdp.DIRECTIONS:\n            raise InternalError('Invalid direction \"%s\"' % direction)\n\n        # don't add track twice\n        if track:\n            self.__assertTrackHasNoSender(track)\n\n        return self.__createTransceiver(\n            direction=direction,\n            kind=kind,\n            sender_track=track)", "response": "Add a new RTCRtpTransceiver."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nstop the ICE agent and all its associated sockets.", "response": "async def close(self):\n        \"\"\"\n        Terminate the ICE agent, ending ICE processing and streams.\n        \"\"\"\n        if self.__isClosed:\n            return\n        self.__isClosed = True\n        self.__setSignalingState('closed')\n\n        # stop senders / receivers\n        for transceiver in self.__transceivers:\n            await transceiver.stop()\n        if self.__sctp:\n            await self.__sctp.stop()\n\n        # stop transports\n        for transceiver in self.__transceivers:\n            await transceiver._transport.stop()\n            await transceiver._transport.transport.stop()\n        if self.__sctp:\n            await self.__sctp.transport.stop()\n            await self.__sctp.transport.transport.stop()\n        self.__updateIceConnectionState()\n\n        # no more events will be emitted, so remove all event listeners\n        # to facilitate garbage collection.\n        self.remove_all_listeners()"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\nasync def createAnswer(self):\n        # check state is valid\n        self.__assertNotClosed()\n        if self.signalingState not in ['have-remote-offer', 'have-local-pranswer']:\n            raise InvalidStateError('Cannot create answer in signaling state \"%s\"' %\n                                    self.signalingState)\n\n        # create description\n        ntp_seconds = clock.current_ntp_time() >> 32\n        description = sdp.SessionDescription()\n        description.origin = '- %d %d IN IP4 0.0.0.0' % (ntp_seconds, ntp_seconds)\n        description.msid_semantic.append(sdp.GroupDescription(\n            semantic='WMS',\n            items=['*']))\n        description.type = 'answer'\n\n        for remote_m in self.__remoteDescription().media:\n            if remote_m.kind in ['audio', 'video']:\n                transceiver = self.__getTransceiverByMid(remote_m.rtp.muxId)\n                description.media.append(create_media_description_for_transceiver(\n                    transceiver,\n                    cname=self.__cname,\n                    direction=and_direction(transceiver.direction, transceiver._offerDirection),\n                    mid=transceiver.mid))\n            else:\n                description.media.append(create_media_description_for_sctp(\n                    self.__sctp, legacy=self._sctpLegacySdp, mid=self.__sctp.mid))\n\n        bundle = sdp.GroupDescription(semantic='BUNDLE', items=[])\n        for media in description.media:\n            bundle.items.append(media.rtp.muxId)\n        description.group.append(bundle)\n\n        return wrap_session_description(description)", "response": "Create an SDP answer to an offer received from a remote peer during the offer negotiation of a WebRTC connection."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef createDataChannel(self, label, maxPacketLifeTime=None, maxRetransmits=None,\n                          ordered=True, protocol='', negotiated=False, id=None):\n        \"\"\"\n        Create a data channel with the given label.\n\n        :rtype: :class:`RTCDataChannel`\n        \"\"\"\n        if maxPacketLifeTime is not None and maxRetransmits is not None:\n            raise ValueError('Cannot specify both maxPacketLifeTime and maxRetransmits')\n\n        if not self.__sctp:\n            self.__createSctpTransport()\n\n        parameters = RTCDataChannelParameters(\n            id=id,\n            label=label,\n            maxPacketLifeTime=maxPacketLifeTime,\n            maxRetransmits=maxRetransmits,\n            negotiated=negotiated,\n            ordered=ordered,\n            protocol=protocol)\n        return RTCDataChannel(self.__sctp, parameters)", "response": "Create a new data channel with the given label."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\nasync def createOffer(self):\n        # check state is valid\n        self.__assertNotClosed()\n\n        if not self.__sctp and not self.__transceivers:\n            raise InternalError('Cannot create an offer with no media and no data channels')\n\n        # offer codecs\n        for transceiver in self.__transceivers:\n            transceiver._codecs = filter_preferred_codecs(\n                CODECS[transceiver.kind][:],\n                transceiver._preferred_codecs,\n            )\n            transceiver._headerExtensions = HEADER_EXTENSIONS[transceiver.kind][:]\n\n        mids = self.__seenMids.copy()\n\n        # create description\n        ntp_seconds = clock.current_ntp_time() >> 32\n        description = sdp.SessionDescription()\n        description.origin = '- %d %d IN IP4 0.0.0.0' % (ntp_seconds, ntp_seconds)\n        description.msid_semantic.append(sdp.GroupDescription(\n            semantic='WMS',\n            items=['*']))\n        description.type = 'offer'\n\n        def get_media(description):\n            return description.media if description else []\n\n        def get_media_section(media, i):\n            return media[i] if i < len(media) else None\n\n        # handle existing transceivers / sctp\n        local_media = get_media(self.__localDescription())\n        remote_media = get_media(self.__remoteDescription())\n        for i in range(max(len(local_media), len(remote_media))):\n            local_m = get_media_section(local_media, i)\n            remote_m = get_media_section(remote_media, i)\n            media_kind = local_m.kind if local_m else remote_m.kind\n            mid = local_m.rtp.muxId if local_m else remote_m.rtp.muxId\n            if media_kind in ['audio', 'video']:\n                transceiver = self.__getTransceiverByMid(mid)\n                transceiver._set_mline_index(i)\n                description.media.append(create_media_description_for_transceiver(\n                    transceiver,\n                    cname=self.__cname,\n                    direction=transceiver.direction,\n                    mid=mid))\n            elif media_kind == 'application':\n                self.__sctp_mline_index = i\n                description.media.append(create_media_description_for_sctp(\n                    self.__sctp, legacy=self._sctpLegacySdp, mid=mid))\n\n        # handle new transceivers / sctp\n        def next_mline_index():\n            return len(description.media)\n\n        for transceiver in filter(lambda x: x.mid is None and not x.stopped, self.__transceivers):\n            transceiver._set_mline_index(next_mline_index())\n            description.media.append(create_media_description_for_transceiver(\n                transceiver,\n                cname=self.__cname,\n                direction=transceiver.direction,\n                mid=allocate_mid(mids)))\n        if self.__sctp and self.__sctp.mid is None:\n            self.__sctp_mline_index = next_mline_index()\n            description.media.append(create_media_description_for_sctp(\n                self.__sctp, legacy=self._sctpLegacySdp, mid=allocate_mid(mids)))\n\n        bundle = sdp.GroupDescription(semantic='BUNDLE', items=[])\n        for media in description.media:\n            bundle.items.append(media.rtp.muxId)\n        description.group.append(bundle)\n\n        return wrap_session_description(description)", "response": "Create an SDP offer for the purpose of starting a new WebRTC connection to a remote peer."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning statistics for the connection.", "response": "async def getStats(self):\n        \"\"\"\n        Returns statistics for the connection.\n\n        :rtype: :class:`RTCStatsReport`\n        \"\"\"\n        merged = RTCStatsReport()\n        coros = [x.getStats() for x in (self.getSenders() + self.getReceivers())]\n        for report in await asyncio.gather(*coros):\n            merged.update(report)\n        return merged"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nchanges the local description associated with the connection.", "response": "async def setLocalDescription(self, sessionDescription):\n        \"\"\"\n        Change the local description associated with the connection.\n\n        :param: sessionDescription: An :class:`RTCSessionDescription` generated\n                                    by :meth:`createOffer` or :meth:`createAnswer()`.\n        \"\"\"\n        # parse and validate description\n        description = sdp.SessionDescription.parse(sessionDescription.sdp)\n        description.type = sessionDescription.type\n        self.__validate_description(description, is_local=True)\n\n        # update signaling state\n        if description.type == 'offer':\n            self.__setSignalingState('have-local-offer')\n        elif description.type == 'answer':\n            self.__setSignalingState('stable')\n\n        # assign MID\n        for i, media in enumerate(description.media):\n            mid = media.rtp.muxId\n            self.__seenMids.add(mid)\n            if media.kind in ['audio', 'video']:\n                transceiver = self.__getTransceiverByMLineIndex(i)\n                transceiver._set_mid(mid)\n            elif media.kind == 'application':\n                self.__sctp.mid = mid\n\n        # set ICE role\n        if self.__initialOfferer is None:\n            self.__initialOfferer = (description.type == 'offer')\n            for iceTransport in self.__iceTransports:\n                iceTransport._connection.ice_controlling = self.__initialOfferer\n\n        # configure direction\n        for t in self.__transceivers:\n            if description.type in ['answer', 'pranswer']:\n                t._currentDirection = and_direction(t.direction, t._offerDirection)\n\n        # gather candidates\n        await self.__gather()\n        for i, media in enumerate(description.media):\n            if media.kind in ['audio', 'video']:\n                transceiver = self.__getTransceiverByMLineIndex(i)\n                add_transport_description(media, transceiver._transport)\n            elif media.kind == 'application':\n                add_transport_description(media, self.__sctp.transport)\n\n        # connect\n        asyncio.ensure_future(self.__connect())\n\n        # replace description\n        if description.type == 'answer':\n            self.__currentLocalDescription = description\n            self.__pendingLocalDescription = None\n        else:\n            self.__pendingLocalDescription = description"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nchange the remote description associated with the connection. :param: sessionDescription: An :class:`RTCSessionDescription` created from information received over the signaling channel.", "response": "async def setRemoteDescription(self, sessionDescription):\n        \"\"\"\n        Changes the remote description associated with the connection.\n\n        :param: sessionDescription: An :class:`RTCSessionDescription` created from\n                                    information received over the signaling channel.\n        \"\"\"\n        # parse and validate description\n        description = sdp.SessionDescription.parse(sessionDescription.sdp)\n        description.type = sessionDescription.type\n        self.__validate_description(description, is_local=False)\n\n        # apply description\n        trackEvents = []\n        for i, media in enumerate(description.media):\n            self.__seenMids.add(media.rtp.muxId)\n            if media.kind in ['audio', 'video']:\n                # find transceiver\n                transceiver = None\n                for t in self.__transceivers:\n                    if t.kind == media.kind and t.mid in [None, media.rtp.muxId]:\n                        transceiver = t\n                if transceiver is None:\n                    transceiver = self.__createTransceiver(direction='recvonly', kind=media.kind)\n                if transceiver.mid is None:\n                    transceiver._set_mid(media.rtp.muxId)\n                    transceiver._set_mline_index(i)\n\n                # negotiate codecs\n                common = filter_preferred_codecs(\n                    find_common_codecs(CODECS[media.kind], media.rtp.codecs),\n                    transceiver._preferred_codecs)\n                assert len(common)\n                transceiver._codecs = common\n                transceiver._headerExtensions = find_common_header_extensions(\n                    HEADER_EXTENSIONS[media.kind], media.rtp.headerExtensions)\n\n                # configure transport\n                iceTransport = transceiver._transport.transport\n                add_remote_candidates(iceTransport, media)\n                self.__remoteDtls[transceiver] = media.dtls\n                self.__remoteIce[transceiver] = media.ice\n\n                # configure direction\n                direction = reverse_direction(media.direction)\n                if description.type in ['answer', 'pranswer']:\n                    transceiver._currentDirection = direction\n                else:\n                    transceiver._offerDirection = direction\n\n                # create remote stream track\n                if direction in ['recvonly', 'sendrecv'] and not transceiver.receiver._track:\n                    transceiver.receiver._track = RemoteStreamTrack(kind=media.kind)\n                    trackEvents.append(RTCTrackEvent(\n                        receiver=transceiver.receiver,\n                        track=transceiver.receiver._track,\n                        transceiver=transceiver,\n                    ))\n\n            elif media.kind == 'application':\n                if not self.__sctp:\n                    self.__createSctpTransport()\n                if self.__sctp.mid is None:\n                    self.__sctp.mid = media.rtp.muxId\n                    self.__sctp_mline_index = i\n\n                # configure sctp\n                if media.profile == 'DTLS/SCTP':\n                    self._sctpLegacySdp = True\n                    self.__sctpRemotePort = int(media.fmt[0])\n                else:\n                    self._sctpLegacySdp = False\n                    self.__sctpRemotePort = media.sctp_port\n                self.__sctpRemoteCaps = media.sctpCapabilities\n\n                # configure transport\n                iceTransport = self.__sctp.transport.transport\n                add_remote_candidates(iceTransport, media)\n                self.__remoteDtls[self.__sctp] = media.dtls\n                self.__remoteIce[self.__sctp] = media.ice\n\n        # remove bundled transports\n        bundle = next((x for x in description.group if x.semantic == 'BUNDLE'), None)\n        if bundle and bundle.items:\n            # find main media stream\n            masterMid = bundle.items[0]\n            masterTransport = None\n            for transceiver in self.__transceivers:\n                if transceiver.mid == masterMid:\n                    masterTransport = transceiver._transport\n                    break\n            if self.__sctp and self.__sctp.mid == masterMid:\n                masterTransport = self.__sctp.transport\n\n            # replace transport for bundled media\n            oldTransports = set()\n            slaveMids = bundle.items[1:]\n            for transceiver in self.__transceivers:\n                if transceiver.mid in slaveMids and not transceiver._bundled:\n                    oldTransports.add(transceiver._transport)\n                    transceiver.receiver.setTransport(masterTransport)\n                    transceiver.sender.setTransport(masterTransport)\n                    transceiver._bundled = True\n                    transceiver._transport = masterTransport\n            if self.__sctp and self.__sctp.mid in slaveMids:\n                oldTransports.add(self.__sctp.transport)\n                self.__sctp.setTransport(masterTransport)\n                self.__sctp._bundled = True\n\n            # stop and discard old ICE transports\n            for dtlsTransport in oldTransports:\n                await dtlsTransport.stop()\n                await dtlsTransport.transport.stop()\n                self.__iceTransports.discard(dtlsTransport.transport)\n            self.__updateIceGatheringState()\n            self.__updateIceConnectionState()\n\n        # FIXME: in aiortc 1.0.0 emit RTCTrackEvent directly\n        for event in trackEvents:\n            self.emit('track', event.track)\n\n        # connect\n        asyncio.ensure_future(self.__connect())\n\n        # update signaling state\n        if description.type == 'offer':\n            self.__setSignalingState('have-remote-offer')\n        elif description.type == 'answer':\n            self.__setSignalingState('stable')\n\n        # replace description\n        if description.type == 'answer':\n            self.__currentRemoteDescription = description\n            self.__pendingRemoteDescription = None\n        else:\n            self.__pendingRemoteDescription = description"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef pack_remb_fci(bitrate, ssrcs):\n    data = b'REMB'\n    exponent = 0\n    mantissa = bitrate\n    while mantissa > 0x3ffff:\n        mantissa >>= 1\n        exponent += 1\n    data += pack('!BBH',\n                 len(ssrcs),\n                 (exponent << 2) | (mantissa >> 16),\n                 (mantissa & 0xffff))\n    for ssrc in ssrcs:\n        data += pack('!L', ssrc)\n    return data", "response": "Pack the FCI for a Receiver Estimated Maximum Bitrate report."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nunpack the FCI for a Receiver Estimated Maximum Bitrate report.", "response": "def unpack_remb_fci(data):\n    \"\"\"\n    Unpack the FCI for a Receiver Estimated Maximum Bitrate report.\n\n    https://tools.ietf.org/html/draft-alvestrand-rmcat-remb-03\n    \"\"\"\n    if len(data) < 8 or data[0:4] != b'REMB':\n        raise ValueError('Invalid REMB prefix')\n\n    exponent = (data[5] & 0xfc) >> 2\n    mantissa = ((data[5] & 0x03) << 16) | (data[6] << 8) | data[7]\n    bitrate = mantissa << exponent\n\n    pos = 8\n    ssrcs = []\n    for r in range(data[4]):\n        ssrcs.append(unpack_from('!L', data, pos)[0])\n        pos += 4\n\n    return (bitrate, ssrcs)"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nparses header extensions according to RFC 5285.", "response": "def unpack_header_extensions(extension_profile: int,\n                             extension_value: bytes) -> List[Tuple[int, bytes]]:\n    \"\"\"\n    Parse header extensions according to RFC 5285.\n    \"\"\"\n    extensions = []\n    pos = 0\n\n    if extension_profile == 0xBEDE:\n        # One-Byte Header\n        while pos < len(extension_value):\n            # skip padding byte\n            if extension_value[pos] == 0:\n                pos += 1\n                continue\n\n            x_id = (extension_value[pos] & 0xf0) >> 4\n            x_length = (extension_value[pos] & 0x0f) + 1\n            pos += 1\n\n            if len(extension_value) < pos + x_length:\n                raise ValueError('RTP one-byte header extension value is truncated')\n            x_value = extension_value[pos:pos + x_length]\n            extensions.append((x_id,  x_value))\n            pos += x_length\n    elif extension_profile == 0x1000:\n        # Two-Byte Header\n        while pos < len(extension_value):\n            # skip padding byte\n            if extension_value[pos] == 0:\n                pos += 1\n                continue\n\n            if len(extension_value) < pos + 2:\n                raise ValueError('RTP two-byte header extension is truncated')\n            x_id, x_length = unpack_from('!BB', extension_value, pos)\n            pos += 2\n\n            if len(extension_value) < pos + x_length:\n                raise ValueError('RTP two-byte header extension value is truncated')\n            x_value = extension_value[pos:pos + x_length]\n            extensions.append((x_id,  x_value))\n            pos += x_length\n\n    return extensions"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef pack_header_extensions(extensions: List[Tuple[int, bytes]]) -> Tuple[int, bytes]:\n    extension_profile = 0\n    extension_value = b''\n\n    if not extensions:\n        return extension_profile, extension_value\n\n    one_byte = True\n    for x_id, x_value in extensions:\n        x_length = len(x_value)\n        assert x_id > 0 and x_id < 256\n        assert x_length >= 0 and x_length < 256\n        if x_id > 14 or x_length == 0 or x_length > 16:\n            one_byte = False\n\n    if one_byte:\n        # One-Byte Header\n        extension_profile = 0xBEDE\n        extension_value = b''\n        for x_id, x_value in extensions:\n            x_length = len(x_value)\n            extension_value += pack('!B', (x_id << 4) | (x_length - 1))\n            extension_value += x_value\n    else:\n        # Two-Byte Header\n        extension_profile = 0x1000\n        extension_value = b''\n        for x_id, x_value in extensions:\n            x_length = len(x_value)\n            extension_value += pack('!BB', x_id, x_length)\n            extension_value += x_value\n\n    extension_value += b'\\x00' * padl(len(extension_value))\n    return extension_profile, extension_value", "response": "Serializes header extensions according to RFC 5285."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nrecovers initial packet from a retransmission packet.", "response": "def unwrap_rtx(rtx, payload_type, ssrc):\n    \"\"\"\n    Recover initial packet from a retransmission packet.\n    \"\"\"\n    packet = RtpPacket(\n        payload_type=payload_type,\n        marker=rtx.marker,\n        sequence_number=unpack('!H', rtx.payload[0:2])[0],\n        timestamp=rtx.timestamp,\n        ssrc=ssrc,\n        payload=rtx.payload[2:])\n    packet.csrc = rtx.csrc\n    packet.extensions = rtx.extensions\n    return packet"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef wrap_rtx(packet, payload_type, sequence_number, ssrc):\n    rtx = RtpPacket(\n        payload_type=payload_type,\n        marker=packet.marker,\n        sequence_number=sequence_number,\n        timestamp=packet.timestamp,\n        ssrc=ssrc,\n        payload=pack('!H', packet.sequence_number) + packet.payload)\n    rtx.csrc = packet.csrc\n    rtx.extensions = packet.extensions\n    return rtx", "response": "Create a retransmission packet from a lost packet."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning statistics about the RTP sender.", "response": "async def getStats(self):\n        \"\"\"\n        Returns statistics about the RTP sender.\n\n        :rtype: :class:`RTCStatsReport`\n        \"\"\"\n        self.__stats.add(RTCOutboundRtpStreamStats(\n            # RTCStats\n            timestamp=clock.current_datetime(),\n            type='outbound-rtp',\n            id='outbound-rtp_' + str(id(self)),\n            # RTCStreamStats\n            ssrc=self._ssrc,\n            kind=self.__kind,\n            transportId=self.transport._stats_id,\n            # RTCSentRtpStreamStats\n            packetsSent=self.__packet_count,\n            bytesSent=self.__octet_count,\n            # RTCOutboundRtpStreamStats\n            trackId=str(id(self.track)),\n        ))\n        self.__stats.update(self.transport._get_stats())\n\n        return self.__stats"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\nasync def send(self, parameters: RTCRtpSendParameters):\n        if not self.__started:\n            self.__cname = parameters.rtcp.cname\n            self.__mid = parameters.muxId\n\n            # make note of the RTP header extension IDs\n            self.__transport._register_rtp_sender(self, parameters)\n            self.__rtp_header_extensions_map.configure(parameters)\n\n            # make note of RTX payload type\n            for codec in parameters.codecs:\n                if is_rtx(codec) and codec.parameters['apt'] == parameters.codecs[0].payloadType:\n                    self.__rtx_payload_type = codec.payloadType\n                    break\n\n            self.__rtp_task = asyncio.ensure_future(self._run_rtp(parameters.codecs[0]))\n            self.__rtcp_task = asyncio.ensure_future(self._run_rtcp())\n            self.__started = True", "response": "Sends the media to the remote peer."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\nasync def stop(self):\n        if self.__started:\n            self.__transport._unregister_rtp_sender(self)\n            self.__rtp_task.cancel()\n            self.__rtcp_task.cancel()\n            await asyncio.gather(\n                self.__rtp_exited.wait(),\n                self.__rtcp_exited.wait())", "response": "Irreversibly stop the sender."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef uint16_gt(a: int, b: int) -> bool:\n    half_mod = 0x8000\n    return (((a < b) and ((b - a) > half_mod)) or\n            ((a > b) and ((a - b) < half_mod)))", "response": "Return True if uint16_gt a and b."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef uint16_gte(a: int, b: int) -> bool:\n    return (a == b) or uint16_gt(a, b)", "response": "Return true if uint16_gt a and b are not equal."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning True if uint32_gt a and b.", "response": "def uint32_gt(a: int, b: int) -> bool:\n    \"\"\"\n    Return a > b.\n    \"\"\"\n    half_mod = 0x80000000\n    return (((a < b) and ((b - a) > half_mod)) or\n            ((a > b) and ((a - b) < half_mod)))"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns True if uint32_gt a or greater than b.", "response": "def uint32_gte(a: int, b: int) -> bool:\n    \"\"\"\n    Return a >= b.\n    \"\"\"\n    return (a == b) or uint32_gt(a, b)"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nbrings up interface. Equivalent to ifconfig [ iface up.", "response": "def up(self):\n        ''' Bring up interface. Equivalent to ifconfig [iface] up. '''\n        # Set new flags\n        flags = self.ifflags | IFF_UP\n        self.ifflags = flags\n        self.get_mtu()"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef open(self):\n        ''' Open file corresponding to the TUN device. '''\n        self.fd = open('/dev/net/tun', 'rb+', buffering=0)\n        tun_flags = IFF_TAP | IFF_NO_PI | IFF_PERSIST\n        ifr = struct.pack('16sH', self.name, tun_flags)\n        fcntl.ioctl(self.fd, TUNSETIFF, ifr)\n        fcntl.ioctl(self.fd, TUNSETOWNER, os.getuid())\n        self.ifflags = self.ifflags | IFF_RUNNING", "response": "Open the file corresponding to the TUN device."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nretrieving the ICE parameters of the ICE gatherer.", "response": "def getLocalParameters(self):\n        \"\"\"\n        Retrieve the ICE parameters of the ICE gatherer.\n\n        :rtype: RTCIceParameters\n        \"\"\"\n        return RTCIceParameters(\n            usernameFragment=self._connection.local_username,\n            password=self._connection.local_password)"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nadds a remote candidate to the AIOIC.", "response": "def addRemoteCandidate(self, candidate):\n        \"\"\"\n        Add a remote candidate.\n        \"\"\"\n        # FIXME: don't use private member!\n        if not self._connection._remote_candidates_end:\n            if candidate is None:\n                self._connection.add_remote_candidate(None)\n            else:\n                self._connection.add_remote_candidate(candidate_to_aioice(candidate))"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\nasync def start(self, remoteParameters):\n        if self.state == 'closed':\n            raise InvalidStateError('RTCIceTransport is closed')\n\n        # handle the case where start is already in progress\n        if self.__start is not None:\n            return await self.__start.wait()\n        self.__start = asyncio.Event()\n\n        self.__setState('checking')\n        self._connection.remote_username = remoteParameters.usernameFragment\n        self._connection.remote_password = remoteParameters.password\n        try:\n            await self._connection.connect()\n        except ConnectionError:\n            self.__setState('failed')\n        else:\n            self.__setState('completed')\n        self.__start.set()", "response": "Initiate connectivity checks.\n\n        :param: remoteParameters: The :class:`RTCIceParameters` associated with\n                                  the remote :class:`RTCIceTransport`."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef send(self, data):\n        if self.readyState != 'open':\n            raise InvalidStateError\n\n        if not isinstance(data, (str, bytes)):\n            raise ValueError('Cannot send unsupported data type: %s' % type(data))\n\n        self.transport._data_channel_send(self, data)", "response": "Send data across the data channel."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef setCodecPreferences(self, codecs):\n        if not codecs:\n            self._preferred_codecs = []\n\n        capabilities = get_capabilities(self.kind).codecs\n        unique = []\n        for codec in reversed(codecs):\n            if codec not in capabilities:\n                raise ValueError('Codec is not in capabilities')\n            if codec not in unique:\n                unique.insert(0, codec)\n        self._preferred_codecs = unique", "response": "Override the default codec preferences."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\nasync def stop(self):\n        await self.__receiver.stop()\n        await self.__sender.stop()\n        self.__stopped = True", "response": "Permanently stops the :class:`RTCRtpTransceiver`."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\nasync def stop(self):\n        for task in self.__tracks.values():\n            if task is not None:\n                task.cancel()\n        self.__tracks = {}", "response": "Stop discarding all tracks."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef addTrack(self, track):\n        if track.kind == 'audio':\n            if self.__container.format.name == 'wav':\n                codec_name = 'pcm_s16le'\n            elif self.__container.format.name == 'mp3':\n                codec_name = 'mp3'\n            else:\n                codec_name = 'aac'\n            stream = self.__container.add_stream(codec_name)\n        else:\n            if self.__container.format.name == 'image2':\n                stream = self.__container.add_stream('png', rate=30)\n                stream.pix_fmt = 'rgb24'\n            else:\n                stream = self.__container.add_stream('libx264', rate=30)\n                stream.pix_fmt = 'yuv420p'\n        self.__tracks[track] = MediaRecorderContext(stream)", "response": "Adds a track to be recorded."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a datetime. datetime object representing the date and time after which the certificate will be considered invalid.", "response": "def expires(self):\n        \"\"\"\n        The date and time after which the certificate will be considered invalid.\n        \"\"\"\n        not_after = self._cert.get_notAfter().decode('ascii')\n        return datetime.datetime.strptime(not_after, '%Y%m%d%H%M%SZ').replace(\n            tzinfo=datetime.timezone.utc)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef generateCertificate(cls):\n        key = generate_key()\n        cert = generate_certificate(key)\n        return cls(key=key, cert=cert)", "response": "Create and return an X.509 certificate and corresponding private key."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\nasync def start(self, remoteParameters):\n        assert self._state == State.NEW\n        assert len(remoteParameters.fingerprints)\n\n        if self.transport.role == 'controlling':\n            self._role = 'server'\n            lib.SSL_set_accept_state(self.ssl)\n        else:\n            self._role = 'client'\n            lib.SSL_set_connect_state(self.ssl)\n\n        self._set_state(State.CONNECTING)\n        try:\n            while not self.encrypted:\n                result = lib.SSL_do_handshake(self.ssl)\n                await self._write_ssl()\n\n                if result > 0:\n                    self.encrypted = True\n                    break\n\n                error = lib.SSL_get_error(self.ssl, result)\n                if error == lib.SSL_ERROR_WANT_READ:\n                    await self._recv_next()\n                else:\n                    self.__log_debug('x DTLS handshake failed (error %d)', error)\n                    for info in get_error_queue():\n                        self.__log_debug('x %s', ':'.join(info))\n                    self._set_state(State.FAILED)\n                    return\n        except ConnectionError:\n            self.__log_debug('x DTLS handshake failed (connection error)')\n            self._set_state(State.FAILED)\n            return\n\n        # check remote fingerprint\n        x509 = lib.SSL_get_peer_certificate(self.ssl)\n        remote_fingerprint = certificate_digest(x509)\n        fingerprint_is_valid = False\n        for f in remoteParameters.fingerprints:\n            if f.algorithm.lower() == 'sha-256' and f.value.lower() == remote_fingerprint.lower():\n                fingerprint_is_valid = True\n                break\n        if not fingerprint_is_valid:\n            self.__log_debug('x DTLS handshake failed (fingerprint mismatch)')\n            self._set_state(State.FAILED)\n            return\n\n        # generate keying material\n        buf = ffi.new('unsigned char[]', 2 * (SRTP_KEY_LEN + SRTP_SALT_LEN))\n        extractor = b'EXTRACTOR-dtls_srtp'\n        _openssl_assert(lib.SSL_export_keying_material(\n            self.ssl, buf, len(buf), extractor, len(extractor), ffi.NULL, 0, 0) == 1)\n\n        view = ffi.buffer(buf)\n        if self._role == 'server':\n            srtp_tx_key = get_srtp_key_salt(view, 1)\n            srtp_rx_key = get_srtp_key_salt(view, 0)\n        else:\n            srtp_tx_key = get_srtp_key_salt(view, 0)\n            srtp_rx_key = get_srtp_key_salt(view, 1)\n\n        rx_policy = Policy(key=srtp_rx_key, ssrc_type=Policy.SSRC_ANY_INBOUND)\n        rx_policy.allow_repeat_tx = True\n        rx_policy.window_size = 1024\n        self._rx_srtp = Session(rx_policy)\n\n        tx_policy = Policy(key=srtp_tx_key, ssrc_type=Policy.SSRC_ANY_OUTBOUND)\n        tx_policy.allow_repeat_tx = True\n        tx_policy.window_size = 1024\n        self._tx_srtp = Session(tx_policy)\n\n        # start data pump\n        self.__log_debug('- DTLS handshake complete')\n        self._set_state(State.CONNECTED)\n        self._task = asyncio.ensure_future(self.__run())", "response": "Start DTLS negotiation with the parameters of the remote DTLS transport."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\nasync def stop(self):\n        if self._task is not None:\n            self._task.cancel()\n            self._task = None\n\n        if self._state in [State.CONNECTING, State.CONNECTED]:\n            lib.SSL_shutdown(self.ssl)\n            try:\n                await self._write_ssl()\n            except ConnectionError:\n                pass\n            self.__log_debug('- DTLS shutdown complete')", "response": "Stop and close the DTLS transport."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\nasync def _write_ssl(self):\n        pending = lib.BIO_ctrl_pending(self.write_bio)\n        if pending > 0:\n            result = lib.BIO_read(self.write_bio, self.write_cdata, len(self.write_cdata))\n            await self.transport._send(ffi.buffer(self.write_cdata)[0:result])\n            self.__tx_bytes += result\n            self.__tx_packets += 1", "response": "Write SSL data to the transport."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nset the bitrate for the current log entry.", "response": "def set_estimate(self, bitrate: int, now_ms: int):\n        \"\"\"\n        For testing purposes.\n        \"\"\"\n        self.current_bitrate = self._clamp_bitrate(bitrate, bitrate)\n        self.current_bitrate_initialized = True\n        self.last_change_ms = now_ms"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreceive the next audio frame.", "response": "async def recv(self):\n        \"\"\"\n        Receive the next :class:`~av.audio.frame.AudioFrame`.\n\n        The base implementation just reads silence, subclass\n        :class:`AudioStreamTrack` to provide a useful implementation.\n        \"\"\"\n        if self.readyState != 'live':\n            raise MediaStreamError\n\n        sample_rate = 8000\n        samples = int(AUDIO_PTIME * sample_rate)\n\n        if hasattr(self, '_timestamp'):\n            self._timestamp += samples\n            wait = self._start + (self._timestamp / sample_rate) - time.time()\n            await asyncio.sleep(wait)\n        else:\n            self._start = time.time()\n            self._timestamp = 0\n\n        frame = AudioFrame(format='s16', layout='mono', samples=samples)\n        for p in frame.planes:\n            p.update(bytes(p.buffer_size))\n        frame.pts = self._timestamp\n        frame.sample_rate = sample_rate\n        frame.time_base = fractions.Fraction(1, sample_rate)\n        return frame"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreceive the next : class : ~av. video. frame. VideoFrame.", "response": "async def recv(self):\n        \"\"\"\n        Receive the next :class:`~av.video.frame.VideoFrame`.\n\n        The base implementation just reads a 640x480 green frame at 30fps,\n        subclass :class:`VideoStreamTrack` to provide a useful implementation.\n        \"\"\"\n        pts, time_base = await self.next_timestamp()\n\n        frame = VideoFrame(width=640, height=480)\n        for p in frame.planes:\n            p.update(bytes(p.buffer_size))\n        frame.pts = pts\n        frame.time_base = time_base\n        return frame"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef prune_chunks(self, tsn):\n        pos = -1\n        size = 0\n        for i, chunk in enumerate(self.reassembly):\n            if uint32_gte(tsn, chunk.tsn):\n                pos = i\n                size += len(chunk.user_data)\n            else:\n                break\n\n        self.reassembly = self.reassembly[pos + 1:]\n        return size", "response": "Prune chunks up to the given TSN."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ngetting what extensions are supported by the remote party.", "response": "def _get_extensions(self, params):\n        \"\"\"\n        Gets what extensions are supported by the remote party.\n        \"\"\"\n        for k, v in params:\n            if k == SCTP_PRSCTP_SUPPORTED:\n                self._remote_partial_reliability = True\n            elif k == SCTP_SUPPORTED_CHUNK_EXT:\n                self._remote_extensions = list(v)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _set_extensions(self, params):\n        extensions = []\n        if self._local_partial_reliability:\n            params.append((SCTP_PRSCTP_SUPPORTED, b''))\n            extensions.append(ForwardTsnChunk.type)\n\n        extensions.append(ReconfigChunk.type)\n        params.append((SCTP_SUPPORTED_CHUNK_EXT, bytes(extensions)))", "response": "Sets what extensions are supported by the local party."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _get_inbound_stream(self, stream_id):\n        if stream_id not in self._inbound_streams:\n            self._inbound_streams[stream_id] = InboundStream()\n        return self._inbound_streams[stream_id]", "response": "Get or create the inbound stream with the specified ID."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nhandling data received from the network.", "response": "async def _handle_data(self, data):\n        \"\"\"\n        Handle data received from the network.\n        \"\"\"\n        try:\n            _, _, verification_tag, chunks = parse_packet(data)\n        except ValueError:\n            return\n\n        # is this an init?\n        init_chunk = len([x for x in chunks if isinstance(x, InitChunk)])\n        if init_chunk:\n            assert len(chunks) == 1\n            expected_tag = 0\n        else:\n            expected_tag = self._local_verification_tag\n\n        # verify tag\n        if verification_tag != expected_tag:\n            self.__log_debug('Bad verification tag %d vs %d',\n                             verification_tag, expected_tag)\n            return\n\n        # handle chunks\n        for chunk in chunks:\n            await self._receive_chunk(chunk)\n\n        # send SACK if needed\n        if self._sack_needed:\n            await self._send_sack()"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn True if the chunk needs to be abandoned.", "response": "def _maybe_abandon(self, chunk):\n        \"\"\"\n        Determine if a chunk needs to be marked as abandoned.\n\n        If it does, it marks the chunk and any other chunk belong to the same\n        message as abandoned.\n        \"\"\"\n        if chunk._abandoned:\n            return True\n\n        abandon = (\n            (chunk._max_retransmits is not None and chunk._sent_count > chunk._max_retransmits) or\n            (chunk._expiry is not None and chunk._expiry < time.time())\n        )\n        if not abandon:\n            return False\n\n        chunk_pos = self._sent_queue.index(chunk)\n        for pos in range(chunk_pos, -1, -1):\n            ochunk = self._sent_queue[pos]\n            ochunk._abandoned = True\n            ochunk._retransmit = False\n            if (ochunk.flags & SCTP_DATA_FIRST_FRAG):\n                break\n        for pos in range(chunk_pos, len(self._sent_queue)):\n            ochunk = self._sent_queue[pos]\n            ochunk._abandoned = True\n            ochunk._retransmit = False\n            if (ochunk.flags & SCTP_DATA_LAST_FRAG):\n                break\n\n        return True"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _mark_received(self, tsn):\n        # it's a duplicate\n        if uint32_gte(self._last_received_tsn, tsn) or tsn in self._sack_misordered:\n            self._sack_duplicates.append(tsn)\n            return True\n\n        # consolidate misordered entries\n        self._sack_misordered.add(tsn)\n        for tsn in sorted(self._sack_misordered):\n            if tsn == tsn_plus_one(self._last_received_tsn):\n                self._last_received_tsn = tsn\n            else:\n                break\n\n        # filter out obsolete entries\n        def is_obsolete(x):\n            return uint32_gt(x, self._last_received_tsn)\n        self._sack_duplicates = list(filter(is_obsolete, self._sack_duplicates))\n        self._sack_misordered = set(filter(is_obsolete, self._sack_misordered))", "response": "Mark an incoming data TSN as received."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\nasync def _receive(self, stream_id, pp_id, data):\n        await self._data_channel_receive(stream_id, pp_id, data)", "response": "Receive data from a ULP stream."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\nasync def _receive_chunk(self, chunk):\n        self.__log_debug('< %s', chunk)\n\n        # common\n        if isinstance(chunk, DataChunk):\n            await self._receive_data_chunk(chunk)\n        elif isinstance(chunk, SackChunk):\n            await self._receive_sack_chunk(chunk)\n        elif isinstance(chunk, ForwardTsnChunk):\n            await self._receive_forward_tsn_chunk(chunk)\n        elif isinstance(chunk, HeartbeatChunk):\n            ack = HeartbeatAckChunk()\n            ack.params = chunk.params\n            await self._send_chunk(ack)\n        elif isinstance(chunk, AbortChunk):\n            self.__log_debug('x Association was aborted by remote party')\n            self._set_state(self.State.CLOSED)\n        elif isinstance(chunk, ShutdownChunk):\n            self._t2_cancel()\n            self._set_state(self.State.SHUTDOWN_RECEIVED)\n            ack = ShutdownAckChunk()\n            await self._send_chunk(ack)\n            self._t2_start(ack)\n            self._set_state(self.State.SHUTDOWN_ACK_SENT)\n        elif (isinstance(chunk, ShutdownCompleteChunk) and\n              self._association_state == self.State.SHUTDOWN_ACK_SENT):\n            self._t2_cancel()\n            self._set_state(self.State.CLOSED)\n        elif (isinstance(chunk, ReconfigChunk) and\n              self._association_state == self.State.ESTABLISHED):\n            for param in chunk.params:\n                cls = RECONFIG_PARAM_TYPES.get(param[0])\n                if cls:\n                    await self._receive_reconfig_param(cls.parse(param[1]))\n\n        # server\n        elif isinstance(chunk, InitChunk) and self.is_server:\n            self._last_received_tsn = tsn_minus_one(chunk.initial_tsn)\n            self._reconfig_response_seq = tsn_minus_one(chunk.initial_tsn)\n            self._remote_verification_tag = chunk.initiate_tag\n            self._ssthresh = chunk.advertised_rwnd\n            self._get_extensions(chunk.params)\n\n            self.__log_debug('- Peer supports %d outbound streams, %d max inbound streams',\n                             chunk.outbound_streams, chunk.inbound_streams)\n            self._inbound_streams_count = min(chunk.outbound_streams, self._inbound_streams_max)\n            self._outbound_streams_count = min(self._outbound_streams_count, chunk.inbound_streams)\n\n            ack = InitAckChunk()\n            ack.initiate_tag = self._local_verification_tag\n            ack.advertised_rwnd = self._advertised_rwnd\n            ack.outbound_streams = self._outbound_streams_count\n            ack.inbound_streams = self._inbound_streams_max\n            ack.initial_tsn = self._local_tsn\n            self._set_extensions(ack.params)\n\n            # generate state cookie\n            cookie = pack('!L', self._get_timestamp())\n            cookie += hmac.new(self._hmac_key, cookie, 'sha1').digest()\n            ack.params.append((SCTP_STATE_COOKIE, cookie))\n            await self._send_chunk(ack)\n        elif isinstance(chunk, CookieEchoChunk) and self.is_server:\n            # check state cookie MAC\n            cookie = chunk.body\n            if (len(cookie) != COOKIE_LENGTH or\n               hmac.new(self._hmac_key, cookie[0:4], 'sha1').digest() != cookie[4:]):\n                self.__log_debug('x State cookie is invalid')\n                return\n\n            # check state cookie lifetime\n            now = self._get_timestamp()\n            stamp = unpack_from('!L', cookie)[0]\n            if stamp < now - COOKIE_LIFETIME or stamp > now:\n                self.__log_debug('x State cookie has expired')\n                error = ErrorChunk()\n                error.params.append((SCTP_CAUSE_STALE_COOKIE, b'\\x00' * 8))\n                await self._send_chunk(error)\n                return\n\n            ack = CookieAckChunk()\n            await self._send_chunk(ack)\n            self._set_state(self.State.ESTABLISHED)\n\n        # client\n        elif isinstance(chunk, InitAckChunk) and self._association_state == self.State.COOKIE_WAIT:\n            # cancel T1 timer and process chunk\n            self._t1_cancel()\n            self._last_received_tsn = tsn_minus_one(chunk.initial_tsn)\n            self._reconfig_response_seq = tsn_minus_one(chunk.initial_tsn)\n            self._remote_verification_tag = chunk.initiate_tag\n            self._ssthresh = chunk.advertised_rwnd\n            self._get_extensions(chunk.params)\n\n            self.__log_debug('- Peer supports %d outbound streams, %d max inbound streams',\n                             chunk.outbound_streams, chunk.inbound_streams)\n            self._inbound_streams_count = min(chunk.outbound_streams, self._inbound_streams_max)\n            self._outbound_streams_count = min(self._outbound_streams_count, chunk.inbound_streams)\n\n            echo = CookieEchoChunk()\n            for k, v in chunk.params:\n                if k == SCTP_STATE_COOKIE:\n                    echo.body = v\n                    break\n            await self._send_chunk(echo)\n\n            # start T1 timer and enter COOKIE-ECHOED state\n            self._t1_start(echo)\n            self._set_state(self.State.COOKIE_ECHOED)\n        elif (isinstance(chunk, CookieAckChunk) and\n              self._association_state == self.State.COOKIE_ECHOED):\n            # cancel T1 timer and enter ESTABLISHED state\n            self._t1_cancel()\n            self._set_state(self.State.ESTABLISHED)\n        elif (isinstance(chunk, ErrorChunk) and\n              self._association_state in [self.State.COOKIE_WAIT, self.State.COOKIE_ECHOED]):\n            self._t1_cancel()\n            self._set_state(self.State.CLOSED)\n            self.__log_debug('x Could not establish association')\n            return", "response": "Handle an incoming chunk."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\nasync def _receive_data_chunk(self, chunk):\n        self._sack_needed = True\n\n        # mark as received\n        if self._mark_received(chunk.tsn):\n            return\n\n        # find stream\n        inbound_stream = self._get_inbound_stream(chunk.stream_id)\n\n        # defragment data\n        inbound_stream.add_chunk(chunk)\n        self._advertised_rwnd -= len(chunk.user_data)\n        for message in inbound_stream.pop_messages():\n            self._advertised_rwnd += len(message[2])\n            await self._receive(*message)", "response": "Handle a DATA chunk."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nhandle a FORWARD TSN chunk.", "response": "async def _receive_forward_tsn_chunk(self, chunk):\n        \"\"\"\n        Handle a FORWARD TSN chunk.\n        \"\"\"\n        self._sack_needed = True\n\n        # it's a duplicate\n        if uint32_gte(self._last_received_tsn, chunk.cumulative_tsn):\n            return\n\n        def is_obsolete(x):\n            return uint32_gt(x, self._last_received_tsn)\n\n        # advance cumulative TSN\n        self._last_received_tsn = chunk.cumulative_tsn\n        self._sack_misordered = set(filter(is_obsolete, self._sack_misordered))\n        for tsn in sorted(self._sack_misordered):\n            if tsn == tsn_plus_one(self._last_received_tsn):\n                self._last_received_tsn = tsn\n            else:\n                break\n\n        # filter out obsolete entries\n        self._sack_duplicates = list(filter(is_obsolete, self._sack_duplicates))\n        self._sack_misordered = set(filter(is_obsolete, self._sack_misordered))\n\n        # update reassembly\n        for stream_id, stream_seq in chunk.streams:\n            inbound_stream = self._get_inbound_stream(stream_id)\n\n            # advance sequence number and perform delivery\n            inbound_stream.sequence_number = uint16_add(stream_seq, 1)\n            for message in inbound_stream.pop_messages():\n                self._advertised_rwnd += len(message[2])\n                await self._receive(*message)\n\n        # prune obsolete chunks\n        for stream_id, inbound_stream in self._inbound_streams.items():\n            self._advertised_rwnd += inbound_stream.prune_chunks(self._last_received_tsn)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nhandle a SACK chunk.", "response": "async def _receive_sack_chunk(self, chunk):\n        \"\"\"\n        Handle a SACK chunk.\n        \"\"\"\n        if uint32_gt(self._last_sacked_tsn, chunk.cumulative_tsn):\n            return\n\n        received_time = time.time()\n        self._last_sacked_tsn = chunk.cumulative_tsn\n        cwnd_fully_utilized = (self._flight_size >= self._cwnd)\n        done = 0\n        done_bytes = 0\n\n        # handle acknowledged data\n        while self._sent_queue and uint32_gte(self._last_sacked_tsn, self._sent_queue[0].tsn):\n            schunk = self._sent_queue.popleft()\n            done += 1\n            if not schunk._acked:\n                done_bytes += schunk._book_size\n                self._flight_size_decrease(schunk)\n\n            # update RTO estimate\n            if done == 1 and schunk._sent_count == 1:\n                self._update_rto(received_time - schunk._sent_time)\n\n        # handle gap blocks\n        loss = False\n        if chunk.gaps:\n            seen = set()\n            for gap in chunk.gaps:\n                for pos in range(gap[0], gap[1] + 1):\n                    highest_seen_tsn = (chunk.cumulative_tsn + pos) % SCTP_TSN_MODULO\n                    seen.add(highest_seen_tsn)\n\n            # determined Highest TSN Newly Acked (HTNA)\n            highest_newly_acked = chunk.cumulative_tsn\n            for schunk in self._sent_queue:\n                if uint32_gt(schunk.tsn, highest_seen_tsn):\n                    break\n                if schunk.tsn in seen and not schunk._acked:\n                    done_bytes += schunk._book_size\n                    schunk._acked = True\n                    self._flight_size_decrease(schunk)\n                    highest_newly_acked = schunk.tsn\n\n            # strike missing chunks prior to HTNA\n            for schunk in self._sent_queue:\n                if uint32_gt(schunk.tsn, highest_newly_acked):\n                    break\n                if schunk.tsn not in seen:\n                    schunk._misses += 1\n                    if schunk._misses == 3:\n                        schunk._misses = 0\n                        if not self._maybe_abandon(schunk):\n                            schunk._retransmit = True\n\n                        schunk._acked = False\n                        self._flight_size_decrease(schunk)\n\n                        loss = True\n\n        # adjust congestion window\n        if self._fast_recovery_exit is None:\n            if done and cwnd_fully_utilized:\n                if self._cwnd <= self._ssthresh:\n                    # slow start\n                    self._cwnd += min(done_bytes, USERDATA_MAX_LENGTH)\n                else:\n                    # congestion avoidance\n                    self._partial_bytes_acked += done_bytes\n                    if self._partial_bytes_acked >= self._cwnd:\n                        self._partial_bytes_acked -= self._cwnd\n                        self._cwnd += USERDATA_MAX_LENGTH\n            if loss:\n                self._ssthresh = max(self._cwnd // 2, 4 * USERDATA_MAX_LENGTH)\n                self._cwnd = self._ssthresh\n                self._partial_bytes_acked = 0\n                self._fast_recovery_exit = self._sent_queue[-1].tsn\n                self._fast_recovery_transmit = True\n        elif uint32_gte(chunk.cumulative_tsn, self._fast_recovery_exit):\n            self._fast_recovery_exit = None\n\n        if not self._sent_queue:\n            # there is no outstanding data, stop T3\n            self._t3_cancel()\n        elif done:\n            # the earliest outstanding chunk was acknowledged, restart T3\n            self._t3_restart()\n\n        self._update_advanced_peer_ack_point()\n        await self._data_channel_flush()\n        await self._transmit()"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\nasync def _receive_reconfig_param(self, param):\n        self.__log_debug('<< %s', param)\n\n        if isinstance(param, StreamResetOutgoingParam):\n            # mark closed inbound streams\n            for stream_id in param.streams:\n                self._inbound_streams.pop(stream_id, None)\n\n                # close data channel\n                channel = self._data_channels.get(stream_id)\n                if channel:\n                    self._data_channel_close(channel)\n\n            # send response\n            response_param = StreamResetResponseParam(\n                response_sequence=param.request_sequence,\n                result=1)\n            self._reconfig_response_seq = param.request_sequence\n\n            await self._send_reconfig_param(response_param)\n        elif isinstance(param, StreamAddOutgoingParam):\n            # increase inbound streams\n            self._inbound_streams_count += param.new_streams\n\n            # send response\n            response_param = StreamResetResponseParam(\n                response_sequence=param.request_sequence,\n                result=1)\n            self._reconfig_response_seq = param.request_sequence\n\n            await self._send_reconfig_param(response_param)\n        elif isinstance(param, StreamResetResponseParam):\n            if (self._reconfig_request and\n               param.response_sequence == self._reconfig_request.request_sequence):\n                # mark closed streams\n                for stream_id in self._reconfig_request.streams:\n                    self._outbound_stream_seq.pop(stream_id, None)\n                    self._data_channel_closed(stream_id)\n\n                self._reconfig_request = None\n                await self._transmit_reconfig()", "response": "Handle a RE - CONFIG parameter."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nsending data ULP to the stream.", "response": "async def _send(self, stream_id, pp_id, user_data,\n                    expiry=None, max_retransmits=None, ordered=True):\n        \"\"\"\n        Send data ULP -> stream.\n        \"\"\"\n        if ordered:\n            stream_seq = self._outbound_stream_seq.get(stream_id, 0)\n        else:\n            stream_seq = 0\n\n        fragments = math.ceil(len(user_data) / USERDATA_MAX_LENGTH)\n        pos = 0\n        for fragment in range(0, fragments):\n            chunk = DataChunk()\n            chunk.flags = 0\n            if not ordered:\n                chunk.flags = SCTP_DATA_UNORDERED\n            if fragment == 0:\n                chunk.flags |= SCTP_DATA_FIRST_FRAG\n            if fragment == fragments - 1:\n                chunk.flags |= SCTP_DATA_LAST_FRAG\n            chunk.tsn = self._local_tsn\n            chunk.stream_id = stream_id\n            chunk.stream_seq = stream_seq\n            chunk.protocol = pp_id\n            chunk.user_data = user_data[pos:pos + USERDATA_MAX_LENGTH]\n\n            # initialize counters\n            chunk._abandoned = False\n            chunk._acked = False\n            chunk._book_size = len(chunk.user_data)\n            chunk._expiry = expiry\n            chunk._max_retransmits = max_retransmits\n            chunk._misses = 0\n            chunk._retransmit = False\n            chunk._sent_count = 0\n            chunk._sent_time = None\n\n            pos += USERDATA_MAX_LENGTH\n            self._local_tsn = tsn_plus_one(self._local_tsn)\n            self._outbound_queue.append(chunk)\n\n        if ordered:\n            self._outbound_stream_seq[stream_id] = uint16_add(stream_seq, 1)\n\n        # transmit outbound data\n        if not self._t3_handle:\n            await self._transmit()"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ntransmits a single chunk.", "response": "async def _send_chunk(self, chunk):\n        \"\"\"\n        Transmit a chunk (no bundling for now).\n        \"\"\"\n        self.__log_debug('> %s', chunk)\n        await self.__transport._send_data(serialize_packet(\n            self._local_port,\n            self._remote_port,\n            self._remote_verification_tag,\n            chunk))"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\nasync def _send_sack(self):\n        gaps = []\n        gap_next = None\n        for tsn in sorted(self._sack_misordered):\n            pos = (tsn - self._last_received_tsn) % SCTP_TSN_MODULO\n            if tsn == gap_next:\n                gaps[-1][1] = pos\n            else:\n                gaps.append([pos, pos])\n            gap_next = tsn_plus_one(tsn)\n\n        sack = SackChunk()\n        sack.cumulative_tsn = self._last_received_tsn\n        sack.advertised_rwnd = max(0, self._advertised_rwnd)\n        sack.duplicates = self._sack_duplicates[:]\n        sack.gaps = [tuple(x) for x in gaps]\n\n        await self._send_chunk(sack)\n\n        self._sack_duplicates.clear()\n        self._sack_needed = False", "response": "Builds and sends a selective acknowledgement ( SACK chunk."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _set_state(self, state):\n        if state != self._association_state:\n            self.__log_debug('- %s -> %s', self._association_state, state)\n            self._association_state = state\n\n        if state == self.State.ESTABLISHED:\n            self.__state = 'connected'\n            for channel in list(self._data_channels.values()):\n                if channel.negotiated and channel.readyState != 'open':\n                    channel._setReadyState('open')\n            asyncio.ensure_future(self._data_channel_flush())\n        elif state == self.State.CLOSED:\n            self._t1_cancel()\n            self._t2_cancel()\n            self._t3_cancel()\n            self.__state = 'closed'\n\n            # close data channels\n            for stream_id in list(self._data_channels.keys()):\n                self._data_channel_closed(stream_id)\n\n            # no more events will be emitted, so remove all event listeners\n            # to facilitate garbage collection.\n            self.remove_all_listeners()", "response": "Transition the SCTP association to a new state."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\nasync def _transmit(self):\n        # send FORWARD TSN\n        if self._forward_tsn_chunk is not None:\n            await self._send_chunk(self._forward_tsn_chunk)\n            self._forward_tsn_chunk = None\n\n            # ensure T3 is running\n            if not self._t3_handle:\n                self._t3_start()\n\n        # limit burst size\n        if self._fast_recovery_exit is not None:\n            burst_size = 2 * USERDATA_MAX_LENGTH\n        else:\n            burst_size = 4 * USERDATA_MAX_LENGTH\n        cwnd = min(self._flight_size + burst_size, self._cwnd)\n\n        # retransmit\n        retransmit_earliest = True\n        for chunk in self._sent_queue:\n            if chunk._retransmit:\n                if self._fast_recovery_transmit:\n                    self._fast_recovery_transmit = False\n                elif self._flight_size >= cwnd:\n                    return\n                self._flight_size_increase(chunk)\n\n                chunk._misses = 0\n                chunk._retransmit = False\n                chunk._sent_count += 1\n                await self._send_chunk(chunk)\n                if retransmit_earliest:\n                    # restart the T3 timer as the earliest outstanding TSN\n                    # is being retransmitted\n                    self._t3_restart()\n            retransmit_earliest = False\n\n        while self._outbound_queue and self._flight_size < cwnd:\n            chunk = self._outbound_queue.popleft()\n            self._sent_queue.append(chunk)\n            self._flight_size_increase(chunk)\n\n            # update counters\n            chunk._sent_count += 1\n            chunk._sent_time = time.time()\n\n            await self._send_chunk(chunk)\n            if not self._t3_handle:\n                self._t3_start()", "response": "Transmit the current T3 message."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nupdating the internal internal", "response": "def _update_advanced_peer_ack_point(self):\n        \"\"\"\n        Try to advance \"Advanced.Peer.Ack.Point\" according to RFC 3758.\n        \"\"\"\n        if uint32_gt(self._last_sacked_tsn, self._advanced_peer_ack_tsn):\n            self._advanced_peer_ack_tsn = self._last_sacked_tsn\n\n        done = 0\n        streams = {}\n        while self._sent_queue and self._sent_queue[0]._abandoned:\n            chunk = self._sent_queue.popleft()\n            self._advanced_peer_ack_tsn = chunk.tsn\n            done += 1\n            if not (chunk.flags & SCTP_DATA_UNORDERED):\n                streams[chunk.stream_id] = chunk.stream_seq\n\n        if done:\n            # build FORWARD TSN\n            self._forward_tsn_chunk = ForwardTsnChunk()\n            self._forward_tsn_chunk.cumulative_tsn = self._advanced_peer_ack_tsn\n            self._forward_tsn_chunk.streams = list(streams.items())"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nupdate the RTO variable based on the new roundtrip measurement R.", "response": "def _update_rto(self, R):\n        \"\"\"\n        Update RTO given a new roundtrip measurement R.\n        \"\"\"\n        if self._srtt is None:\n            self._rttvar = R / 2\n            self._srtt = R\n        else:\n            self._rttvar = (1 - SCTP_RTO_BETA) * self._rttvar + SCTP_RTO_BETA * abs(self._srtt - R)\n            self._srtt = (1 - SCTP_RTO_ALPHA) * self._srtt + SCTP_RTO_ALPHA * R\n        self._rto = max(SCTP_RTO_MIN, min(self._srtt + 4 * self._rttvar, SCTP_RTO_MAX))"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _data_channel_close(self, channel, transmit=True):\n        if channel.readyState not in ['closing', 'closed']:\n            channel._setReadyState('closing')\n            self._reconfig_queue.append(channel.id)\n            if len(self._reconfig_queue) == 1:\n                asyncio.ensure_future(self._transmit_reconfig())", "response": "Closes the data channel."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nflush buffered data to the SCTP layer.", "response": "async def _data_channel_flush(self):\n        \"\"\"\n        Try to flush buffered data to the SCTP layer.\n\n        We wait until the association is established, as we need to know\n        whether we are a client or a server to correctly assign an odd/even ID\n        to the data channels.\n        \"\"\"\n        if self._association_state != self.State.ESTABLISHED:\n            return\n\n        while self._data_channel_queue and not self._outbound_queue:\n            channel, protocol, user_data = self._data_channel_queue.popleft()\n\n            # register channel if necessary\n            stream_id = channel.id\n            if stream_id is None:\n                while self._data_channel_id in self._data_channels:\n                    self._data_channel_id += 2\n                stream_id = self._data_channel_id\n                self._data_channels[stream_id] = channel\n                self._data_channel_id += 2\n                channel._setId(stream_id)\n\n            # send data\n            if protocol == WEBRTC_DCEP:\n                await self._send(stream_id, protocol, user_data)\n            else:\n                if channel.maxPacketLifeTime:\n                    expiry = time.time() + (channel.maxPacketLifeTime / 1000)\n                else:\n                    expiry = None\n                await self._send(stream_id, protocol, user_data,\n                                 expiry=expiry,\n                                 max_retransmits=channel.maxRetransmits,\n                                 ordered=channel.ordered)\n                channel._addBufferedAmount(-len(user_data))"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef add_signaling_arguments(parser):\n    parser.add_argument('--signaling', '-s', choices=[\n        'copy-and-paste', 'tcp-socket', 'unix-socket'])\n    parser.add_argument('--signaling-host', default='127.0.0.1',\n                        help='Signaling host (tcp-socket only)')\n    parser.add_argument('--signaling-port', default=1234,\n                        help='Signaling port (tcp-socket only)')\n    parser.add_argument('--signaling-path', default='aiortc.socket',\n                        help='Signaling socket path (unix-socket only)')", "response": "Adds signaling method arguments to an argparse. ArgumentParser."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef create_signaling(args):\n    if args.signaling == 'tcp-socket':\n        return TcpSocketSignaling(args.signaling_host, args.signaling_port)\n    elif args.signaling == 'unix-socket':\n        return UnixSocketSignaling(args.signaling_path)\n    else:\n        return CopyAndPasteSignaling()", "response": "Create a signaling method based on command - line arguments."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\nasync def recv(self):\n        if self.readyState != 'live':\n            raise MediaStreamError\n\n        frame = await self._queue.get()\n        if frame is None:\n            self.stop()\n            raise MediaStreamError\n        return frame", "response": "Receive the next frame from the queue."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn statistics about the receiver.", "response": "async def getStats(self):\n        \"\"\"\n        Returns statistics about the RTP receiver.\n\n        :rtype: :class:`RTCStatsReport`\n        \"\"\"\n        for ssrc, stream in self.__remote_streams.items():\n            self.__stats.add(RTCInboundRtpStreamStats(\n                # RTCStats\n                timestamp=clock.current_datetime(),\n                type='inbound-rtp',\n                id='inbound-rtp_' + str(id(self)),\n                # RTCStreamStats\n                ssrc=ssrc,\n                kind=self.__kind,\n                transportId=self.transport._stats_id,\n                # RTCReceivedRtpStreamStats\n                packetsReceived=stream.packets_received,\n                packetsLost=stream.packets_lost,\n                jitter=stream.jitter,\n                # RTPInboundRtpStreamStats\n            ))\n        self.__stats.update(self.transport._get_stats())\n\n        return self.__stats"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a list of RTCRtpSynchronizationSource objects for each unique SSRC identifier in the last 10 seconds.", "response": "def getSynchronizationSources(self):\n        \"\"\"\n        Returns a :class:`RTCRtpSynchronizationSource` for each unique SSRC identifier\n        received in the last 10 seconds.\n        \"\"\"\n        cutoff = clock.current_datetime() - datetime.timedelta(seconds=10)\n        sources = []\n        for source, timestamp in self.__active_ssrc.items():\n            if timestamp >= cutoff:\n                sources.append(RTCRtpSynchronizationSource(source=source, timestamp=timestamp))\n        return sources"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\nasync def receive(self, parameters: RTCRtpReceiveParameters):\n        if not self.__started:\n            for codec in parameters.codecs:\n                self.__codecs[codec.payloadType] = codec\n            for encoding in parameters.encodings:\n                if encoding.rtx:\n                    self.__rtx_ssrc[encoding.rtx.ssrc] = encoding.ssrc\n\n            # start decoder thread\n            self.__decoder_thread = threading.Thread(\n                target=decoder_worker,\n                name=self.__kind + '-decoder',\n                args=(asyncio.get_event_loop(), self.__decoder_queue, self._track._queue))\n            self.__decoder_thread.start()\n\n            self.__transport._register_rtp_receiver(self, parameters)\n            self.__rtcp_task = asyncio.ensure_future(self._run_rtcp())\n            self.__started = True", "response": "This method is called by the RTCP receiver when the media is received."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nhandle an incoming RTP packet.", "response": "async def _handle_rtp_packet(self, packet: RtpPacket, arrival_time_ms: int):\n        \"\"\"\n        Handle an incoming RTP packet.\n        \"\"\"\n        self.__log_debug('< %s', packet)\n\n        # feed bitrate estimator\n        if self.__remote_bitrate_estimator is not None:\n            if packet.extensions.abs_send_time is not None:\n                remb = self.__remote_bitrate_estimator.add(\n                    abs_send_time=packet.extensions.abs_send_time,\n                    arrival_time_ms=arrival_time_ms,\n                    payload_size=len(packet.payload) + packet.padding_size,\n                    ssrc=packet.ssrc,\n                )\n                if self.__rtcp_ssrc is not None and remb is not None:\n                    # send Receiver Estimated Maximum Bitrate feedback\n                    rtcp_packet = RtcpPsfbPacket(\n                        fmt=RTCP_PSFB_APP, ssrc=self.__rtcp_ssrc, media_ssrc=0)\n                    rtcp_packet.fci = pack_remb_fci(*remb)\n                    await self._send_rtcp(rtcp_packet)\n\n        # keep track of sources\n        self.__active_ssrc[packet.ssrc] = clock.current_datetime()\n\n        # check the codec is known\n        codec = self.__codecs.get(packet.payload_type)\n        if codec is None:\n            self.__log_debug('x RTP packet with unknown payload type %d', packet.payload_type)\n            return\n\n        # feed RTCP statistics\n        if packet.ssrc not in self.__remote_streams:\n            self.__remote_streams[packet.ssrc] = StreamStatistics(codec.clockRate)\n        self.__remote_streams[packet.ssrc].add(packet)\n\n        # unwrap retransmission packet\n        if is_rtx(codec):\n            original_ssrc = self.__rtx_ssrc.get(packet.ssrc)\n            if original_ssrc is None:\n                self.__log_debug('x RTX packet from unknown SSRC %d', packet.ssrc)\n                return\n\n            if len(packet.payload) < 2:\n                return\n\n            codec = self.__codecs[codec.parameters['apt']]\n            packet = unwrap_rtx(packet,\n                                payload_type=codec.payloadType,\n                                ssrc=original_ssrc)\n\n        # send NACKs for any missing any packets\n        if self.__nack_generator is not None and self.__nack_generator.add(packet):\n            await self._send_rtcp_nack(packet.ssrc, sorted(self.__nack_generator.missing))\n\n        # parse codec-specific information\n        try:\n            if packet.payload:\n                packet._data = depayload(codec, packet.payload)\n            else:\n                packet._data = b''\n        except ValueError as exc:\n            self.__log_debug('x RTP payload parsing failed: %s', exc)\n            return\n\n        # try to re-assemble encoded frame\n        encoded_frame = self.__jitter_buffer.add(packet)\n\n        # if we have a complete encoded frame, decode it\n        if encoded_frame is not None and self.__decoder_thread:\n            encoded_frame.timestamp = self.__timestamp_mapper.map(encoded_frame.timestamp)\n            self.__decoder_queue.put((codec, encoded_frame))"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nsends an RTCP packet to report missing RTP packets.", "response": "async def _send_rtcp_nack(self, media_ssrc, lost):\n        \"\"\"\n        Send an RTCP packet to report missing RTP packets.\n        \"\"\"\n        if self.__rtcp_ssrc is not None:\n            packet = RtcpRtpfbPacket(\n                fmt=RTCP_RTPFB_NACK, ssrc=self.__rtcp_ssrc, media_ssrc=media_ssrc)\n            packet.lost = lost\n            await self._send_rtcp(packet)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\nasync def _send_rtcp_pli(self, media_ssrc):\n        if self.__rtcp_ssrc is not None:\n            packet = RtcpPsfbPacket(fmt=RTCP_PSFB_PLI, ssrc=self.__rtcp_ssrc, media_ssrc=media_ssrc)\n            await self._send_rtcp(packet)", "response": "Send an RTCP packet to report picture loss."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef __stop_decoder(self):\n        if self.__decoder_thread:\n            self.__decoder_queue.put(None)\n            self.__decoder_thread.join()\n            self.__decoder_thread = None", "response": "Stop the decoder thread."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef init_domain_ledger(self):\n        if self.config.primaryStorage is None:\n            genesis_txn_initiator = GenesisTxnInitiatorFromFile(\n                self.genesis_dir, self.config.domainTransactionsFile)\n            return Ledger(\n                CompactMerkleTree(\n                    hashStore=self.getHashStore('domain')),\n                dataDir=self.dataLocation,\n                fileName=self.config.domainTransactionsFile,\n                ensureDurability=self.config.EnsureLedgerDurability,\n                genesis_txn_initiator=genesis_txn_initiator)\n        else:\n            return initStorage(self.config.primaryStorage,\n                               name=self.name + NODE_PRIMARY_STORAGE_SUFFIX,\n                               dataDir=self.dataLocation,\n                               config=self.config)", "response": "This method creates a new domain ledger."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nupdating the transaction with extra data that is omitted from the ledger", "response": "def update_txn_with_extra_data(self, txn):\n        \"\"\"\n        All the data of the transaction might not be stored in ledger so the\n        extra data that is omitted from ledger needs to be fetched from the\n        appropriate data store\n        :param txn:\n        :return:\n        \"\"\"\n        # For RAW and ENC attributes, only hash is stored in the ledger.\n        if get_type(txn) == ATTRIB:\n            txn_data = get_payload_data(txn)\n            # The key needs to be present and not None\n            key = RAW if (RAW in txn_data and txn_data[RAW] is not None) else \\\n                ENC if (ENC in txn_data and txn_data[ENC] is not None) else None\n            if key:\n                txn_data[key] = self.attributeStore.get(txn_data[key])\n        return txn"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef prepare_attr_for_state(txn, path_only=False):\n    assert get_type(txn) == ATTRIB\n    txn_data = get_payload_data(txn)\n    nym = txn_data[TARGET_NYM]\n    attr_type, attr_key, value = parse_attr_txn(txn_data)\n    path = make_state_path_for_attr(nym, attr_key, attr_type == HASH)\n    if path_only:\n        return path\n    hashed_value = hash_of(value) if value else ''\n    seq_no = get_seq_no(txn)\n    txn_time = get_txn_time(txn)\n    value_bytes = encode_state_value(hashed_value, seq_no, txn_time)\n    return attr_type, path, value, hashed_value, value_bytes", "response": "Prepare key - value pair for state attribute store."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nextracts the name and value of an attribute from a transaction data dictionary.", "response": "def _extract_attr_typed_value(txn_data):\n    \"\"\"\n    ATTR and GET_ATTR can have one of 'raw', 'enc' and 'hash' fields.\n    This method checks which of them presents and return it's name\n    and value in it.\n    \"\"\"\n    existing_keys = [key for key in ALL_ATR_KEYS if key in txn_data]\n    if len(existing_keys) == 0:\n        raise ValueError(\"ATTR should have one of the following fields: {}\"\n                         .format(ALL_ATR_KEYS))\n    if len(existing_keys) > 1:\n        raise ValueError(\"ATTR should have only one of the following fields: {}\"\n                         .format(ALL_ATR_KEYS))\n    existing_key = existing_keys[0]\n    return existing_key, txn_data[existing_key]"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _migrate_ledger(data_directory,\n                    old_ledger_file, new_ledger_file,\n                    serializer: MappingSerializer = None):\n    \"\"\"\n    Test for the directory, open old and new ledger, migrate data, rename directories\n    \"\"\"\n\n    # we should have ChunkedFileStorage implementation of the Ledger\n    if not os.path.isdir(os.path.join(data_directory, old_ledger_file)):\n        msg = 'Could not find directory {} for migration.'.format(\n            old_ledger_file)\n        logger.error(msg)\n        raise Exception(msg)\n\n    # open the current ledger using the specified serializer\n    old_txn_log_store = ChunkedFileStore(data_directory,\n                                         old_ledger_file,\n                                         isLineNoKey=True,\n                                         storeContentHash=False)\n    logger.info(\"Old ledger folder: {}, {}\".format(\n        data_directory, old_ledger_file))\n    old_ledger = Ledger(CompactMerkleTree(),\n                        dataDir=data_directory,\n                        txn_serializer=serializer,\n                        hash_serializer=serializer,\n                        fileName=old_ledger_file,\n                        transactionLogStore=old_txn_log_store)\n    logger.info(\"old size for {}: {}\".format(\n        old_ledger_file, str(old_ledger.size)))\n\n    # open the new ledger with new serialization\n    new_ledger_file_backup = new_ledger_file + \"_new\"\n    logger.info(\"New ledger folder: {}, {}\".format(\n        data_directory, new_ledger_file_backup))\n    new_ledger = Ledger(CompactMerkleTree(),\n                        dataDir=data_directory,\n                        fileName=new_ledger_file_backup)\n\n    # add all txns into the new ledger\n    for _, txn in old_ledger.getAllTxn():\n        if txn[TXN_TYPE] == SCHEMA:\n            if DATA in txn:\n                txn[DATA] = json.loads(txn[DATA])\n        if txn[TXN_TYPE] == CLAIM_DEF:\n            if DATA in txn:\n                txn[DATA] = json.loads(txn[DATA])\n        new_ledger.add(txn)\n    logger.info(\"new size for {}: {}\".format(\n        new_ledger_file, str(new_ledger.size)))\n\n    old_ledger.stop()\n    new_ledger.stop()\n\n    # now that everything succeeded, remove the old files and move the new\n    # files into place\n    shutil.rmtree(\n        os.path.join(data_directory, old_ledger_file))\n    os.rename(\n        os.path.join(data_directory, new_ledger_file_backup),\n        os.path.join(data_directory, new_ledger_file))\n\n    logger.info(\"Final new ledger folder: {}\".format(\n        os.path.join(data_directory, new_ledger_file)))", "response": "Open the old and new ledger and migrate data"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef getNym(self, nym, role=None, isCommitted=True):\n        try:\n            seqNo, txnTime, ta, actual_role, verkey = self.get(nym, isCommitted)\n        except KeyError:\n            return None\n        if role and role != actual_role:\n            return None\n        return {\n            ROLE: actual_role or None,\n            VERKEY: verkey or None,\n            f.IDENTIFIER.nm: ta or None,\n            f.SEQ_NO.nm: seqNo or None,\n            TXN_TIME: txnTime or None,\n        }", "response": "Get a nym from the cache."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _unscheduleAction(self):\n        logger.trace(\"{} unscheduling actions\".format(self))\n        self.aqStash = deque()\n        self.scheduledAction = None", "response": "Unschedule current action\n\n        Note that it does not add record to action log and does not do\n        required steps to resume previous action. If you need this - use\n        _cancelScheduledAction"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nfind rule_id for incoming action_id and return AuthConstraint instance", "response": "def get_auth_constraint(self, action_id: str) -> AbstractAuthConstraint:\n        \"\"\"\n        Find rule_id for incoming action_id and return AuthConstraint instance\n        \"\"\"\n        if self.anyone_can_write_map:\n            return self._find_auth_constraint(action_id, self.anyone_can_write_map, from_local=True)\n\n        return self._find_auth_constraint(action_id, self.auth_map)"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _append(self, ev_type: Enum, data: ActionLogData) -> None:\n        event = ActionLogEvent(None, ev_type, data, types=self._event_types)\n        with open(self._file_path, mode='a+', newline='') as f:\n            f.write(event.pack(delimiter=self._delimiter))\n        self._items.append(event)", "response": "Append an event to the log."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nupdates the state of the current entry in the state trie.", "response": "def update_state(self, txn, prev_result, is_committed=True) -> None:\n        \"\"\"\n        The state trie stores the hash of the whole attribute data at:\n            the did+attribute name if the data is plaintext (RAW)\n            the did+hash(attribute) if the data is encrypted (ENC)\n        If the attribute is HASH, then nothing is stored in attribute store,\n        the trie stores a blank value for the key did+hash\n        \"\"\"\n        self._validate_txn_type(txn)\n        attr_type, path, value, hashed_value, value_bytes = domain.prepare_attr_for_state(txn)\n        self.state.set(path, value_bytes)\n        if attr_type != HASH:\n            self.database_manager.attribute_store.set(hashed_value, value)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncreate an auth strategy based on the current configuration.", "response": "def create_auth_strategy(self):\n        \"\"\"depends on config\"\"\"\n        if self.config.authPolicy == LOCAL_AUTH_POLICY:\n            return LocalAuthStrategy(auth_map=self.auth_map,\n                                     anyone_can_write_map=self.anyone_can_write_map if self.anyone_can_write else None)\n        elif self.config.authPolicy == CONFIG_LEDGER_AUTH_POLICY:\n            return ConfigLedgerAuthStrategy(auth_map=self.auth_map,\n                                            state=self.config_state,\n                                            serializer=self.state_serializer,\n                                            anyone_can_write_map=self.anyone_can_write_map if self.anyone_can_write else None,\n                                            metrics=self.metrics)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ngenerate the path to state as str type", "response": "def gen_txn_path(self, txn):\n\n        \"\"\"Return path to state as 'str' type or None\"\"\"\n        txn_type = get_type(txn)\n        if txn_type not in self.state_update_handlers:\n            logger.error('Cannot generate id for txn of type {}'.format(txn_type))\n            return None\n\n        if txn_type == NYM:\n            nym = get_payload_data(txn).get(TARGET_NYM)\n            binary_digest = domain.make_state_path_for_nym(nym)\n            return hexlify(binary_digest).decode()\n        elif txn_type == ATTRIB:\n            path = domain.prepare_attr_for_state(txn, path_only=True)\n            return path.decode()\n        elif txn_type == SCHEMA:\n            path = domain.prepare_schema_for_state(txn, path_only=True)\n            return path.decode()\n        elif txn_type == CLAIM_DEF:\n            path = domain.prepare_claim_def_for_state(txn, path_only=True)\n            return path.decode()\n        elif txn_type == REVOC_REG_DEF:\n            path = domain.prepare_revoc_def_for_state(txn, path_only=True)\n            return path.decode()\n        elif txn_type == REVOC_REG_ENTRY:\n            path = domain.prepare_revoc_reg_entry_for_state(txn, path_only=True)\n            return path.decode()\n\n        raise NotImplementedError(\"path construction is not implemented for type {}\".format(txn_type))"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef handleGetRevocRegDelta(self, request: Request):\n        req_ts_from = request.operation.get(FROM, None)\n        req_ts_to = request.operation.get(TO)\n        revoc_reg_def_id = request.operation.get(REVOC_REG_DEF_ID)\n        reply = None\n        \"\"\"\n        Get root hash for \"to\" timestamp\n        Get REVOC_REG_ENTRY and ACCUM record for timestamp \"to\"\n        \"\"\"\n        path_to_reg_entry = domain.make_state_path_for_revoc_reg_entry(revoc_reg_def_id=revoc_reg_def_id)\n        path_to_reg_entry_accum = domain.make_state_path_for_revoc_reg_entry_accum(revoc_reg_def_id=revoc_reg_def_id)\n\n        entry_to = self._get_reg_entry_by_timestamp(req_ts_to, path_to_reg_entry)\n        accum_to = self._get_reg_entry_accum_by_timestamp(req_ts_to, path_to_reg_entry_accum)\n        entry_from = StateValue()\n        accum_from = StateValue()\n\n        if accum_to.value and entry_to.value:\n            \"\"\"Get issuance type from REVOC_REG_DEF\"\"\"\n            encoded_revoc_reg_def = self.state.get_for_root_hash(entry_to.root_hash,\n                                                                 revoc_reg_def_id)\n            if encoded_revoc_reg_def:\n                revoc_reg_def, _, _ = domain.decode_state_value(encoded_revoc_reg_def)\n                strategy_cls = self.get_revocation_strategy(revoc_reg_def[VALUE][ISSUANCE_TYPE])\n                issued_to = entry_to.value[VALUE].get(ISSUED, [])\n                revoked_to = entry_to.value[VALUE].get(REVOKED, [])\n                if req_ts_from:\n                    \"\"\"Get REVOC_REG_ENTRY and ACCUM records for timestamp from if exist\"\"\"\n                    entry_from = self._get_reg_entry_by_timestamp(req_ts_from, path_to_reg_entry)\n                    accum_from = self._get_reg_entry_accum_by_timestamp(req_ts_from, path_to_reg_entry_accum)\n                if req_ts_from and entry_from.value and accum_from.value:\n                    \"\"\"Compute issued/revoked lists corresponding with ISSUANCE_TYPE strategy\"\"\"\n                    issued_from = entry_from.value[VALUE].get(ISSUED, [])\n                    revoked_from = entry_from.value[VALUE].get(REVOKED, [])\n                    result_issued, result_revoked = strategy_cls.get_delta({ISSUED: issued_to,\n                                                                            REVOKED: revoked_to},\n                                                                           {ISSUED: issued_from,\n                                                                            REVOKED: revoked_from})\n                else:\n                    result_issued, result_revoked = strategy_cls.get_delta({ISSUED: issued_to,\n                                                                            REVOKED: revoked_to},\n                                                                           None)\n                reply = {\n                    REVOC_REG_DEF_ID: revoc_reg_def_id,\n                    REVOC_TYPE: revoc_reg_def.get(REVOC_TYPE),\n                    VALUE: {\n                        ACCUM_TO: accum_to.value if entry_from.value else entry_to.value,\n                        ISSUED: result_issued,\n                        REVOKED: result_revoked\n                    }\n\n                }\n                \"\"\"If we got \"from\" timestamp, then add state proof into \"data\" section of reply\"\"\"\n                if req_ts_from and accum_from.value:\n                    reply[STATE_PROOF_FROM] = accum_from.proof\n                    reply[VALUE][ACCUM_FROM] = accum_from.value\n\n        if accum_to and entry_to:\n            seq_no = accum_to.seq_no if entry_from.value else entry_to.seq_no\n            update_time = accum_to.update_time if entry_from.value else entry_to.update_time\n            proof = accum_to.proof if entry_from.value else entry_to.proof\n        else:\n            seq_no = None\n            update_time = None\n            proof = None\n\n        return self.make_domain_result(request=request,\n                                       data=reply,\n                                       last_seq_no=seq_no,\n                                       update_time=update_time,\n                                       proof=proof)", "response": "Handles the GET_REVOC_REG_DELTA request."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nquery state for data on specified path and returns the value last_seq_no last_update_time and proof.", "response": "def lookup(self, path, isCommitted=True, with_proof=False) -> (str, int):\n        \"\"\"\n        Queries state for data on specified path\n\n        :param path: path to data\n        :param isCommitted: queries the committed state root if True else the uncommitted root\n        :param with_proof: creates proof if True\n        :return: data\n        \"\"\"\n        assert path is not None\n        head_hash = self.state.committedHeadHash if isCommitted else self.state.headHash\n        encoded, proof = self.get_value_from_state(path, head_hash, with_proof=with_proof)\n        if encoded:\n            value, last_seq_no, last_update_time = domain.decode_state_value(encoded)\n            return value, last_seq_no, last_update_time, proof\n        return None, None, None, proof"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nadds an attribute to the state trie.", "response": "def _addAttr(self, txn, isCommitted=False) -> None:\n        \"\"\"\n        The state trie stores the hash of the whole attribute data at:\n            the did+attribute name if the data is plaintext (RAW)\n            the did+hash(attribute) if the data is encrypted (ENC)\n        If the attribute is HASH, then nothing is stored in attribute store,\n        the trie stores a blank value for the key did+hash\n        \"\"\"\n        assert get_type(txn) == ATTRIB\n        attr_type, path, value, hashed_value, value_bytes = domain.prepare_attr_for_state(txn)\n        self.state.set(path, value_bytes)\n        if attr_type != HASH:\n            self.attributeStore.set(hashed_value, value)"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ntransforms a transaction to be stored in the ledger", "response": "def transform_txn_for_ledger(txn):\n        \"\"\"\n        Some transactions need to be transformed before they can be stored in the\n        ledger, eg. storing certain payload in another data store and only its\n        hash in the ledger\n        \"\"\"\n        if get_type(txn) == ATTRIB:\n            txn = DomainReqHandler.transform_attrib_for_ledger(txn)\n        return txn"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef transform_attrib_for_ledger(txn):\n        txn = deepcopy(txn)\n        txn_data = get_payload_data(txn)\n        attr_type, _, value = domain.parse_attr_txn(txn_data)\n        if attr_type in [RAW, ENC]:\n            txn_data[attr_type] = domain.hash_of(value) if value else ''\n\n        return txn", "response": "Transform the attribute of a ledger transaction so that they can be used by the ledger."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef getSymmetricallyEncryptedVal(val, secretKey: Union[str, bytes] = None) -> \\\n        Tuple[str, str]:\n    \"\"\"\n    Encrypt the provided value with symmetric encryption\n\n    :param val: the value to encrypt\n    :param secretKey: Optional key, if provided should be either in hex or bytes\n    :return: Tuple of the encrypted value and secret key encoded in hex\n    \"\"\"\n\n    if isinstance(val, str):\n        val = val.encode(\"utf-8\")\n    if secretKey:\n        if isHex(secretKey):\n            secretKey = bytes(bytearray.fromhex(secretKey))\n        elif not isinstance(secretKey, bytes):\n            error(\"Secret key must be either in hex or bytes\")\n        box = libnacl.secret.SecretBox(secretKey)\n    else:\n        box = libnacl.secret.SecretBox()\n\n    return box.encrypt(val).hex(), box.sk.hex()", "response": "Encrypt the provided value with symmetric encryption with the specified secret key."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef getIndex(predicateFn: Callable[[T], bool], items: List[T]) -> int:\n    try:\n        return next(i for i, v in enumerate(items) if predicateFn(v))\n    except StopIteration:\n        return -1", "response": "Finds the index of an item in list which satisfies predicateFn"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef getImageByTag(tag):\n    '''Check if an image with a given tag exists. No side-effects. Idempotent.\n    Handles ImageNotFound and APIError exceptions, but only reraises APIError.\n    '''\n    require_str(\"tag\", tag)\n\n    image = None\n    try:\n        image = client.images.get(tag)\n        print(\"Found image\", tag, \"...\")\n    except ImageNotFound:\n        print(\"Image\", tag, \"does not exist ...\")\n    except APIError as exc:\n        eprint(\"Unhandled error while getting image\", tag)\n        raise exc\n\n    return image", "response": "Check if an image with a given tag exists. No side - effects. Idempotent."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef getImage(path, dockerfile, tag):\n    '''Check if an image with a given tag exists. If not, build an image from\n    using a given dockerfile in a given path, tagging it with a given tag.\n    No extra side effects. Handles and reraises BuildError, TypeError, and\n    APIError exceptions.\n    '''\n    image = getImageByTag(tag)\n\n    if not image:\n        # Build an Image using the dockerfile in the path\n        try:\n            image = client.images.build(\n              path=path,\n              dockerfile=dockerfile,\n              tag=tag\n            )\n        except BuildError as exc:\n            eprint(\"Failed to build docker image\")\n            raise exc\n        except TypeError as exc:\n            eprint(\"You must give a path to the build environemnt.\")\n            raise exc\n        except APIError as exc:\n            eprint(\"Unhandled error while building image\", tag)\n            raise exc\n\n    return image", "response": "Build an image from a given path using a given dockerfile in a given path."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef runContainer(image, **kwargs):\n    '''Run a docker container using a given image; passing keyword arguments\n    documented to be accepted by docker's client.containers.run function\n    No extra side effects. Handles and reraises ContainerError, ImageNotFound,\n    and APIError exceptions.\n    '''\n    container = None\n    try:\n        container = client.containers.run(image, **kwargs)\n        if \"name\" in kwargs.keys():\n          print(\"Container\", kwargs[\"name\"], \"is now running.\")\n    except ContainerError as exc:\n        eprint(\"Failed to run container\")\n        raise exc\n    except ImageNotFound as exc:\n        eprint(\"Failed to find image to run as a docker container\")\n        raise exc\n    except APIError as exc:\n        eprint(\"Unhandled error\")\n        raise exc\n\n    return container", "response": "Run a docker container using a given image"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nget the container with the given name or ID.", "response": "def getContainer(name_or_id):\n    '''Get the container with the given name or ID (str). No side effects.\n    Idempotent. Returns None if the container does not exist. Otherwise, the\n    continer is returned'''\n    require_str(\"name_or_id\", name_or_id)\n\n    container = None\n    try:\n        container = client.containers.get(name_or_id)\n    except NotFound as exc:\n        # Return None when the container is not found\n        pass\n    except APIError as exc:\n        eprint(\"Unhandled error\")\n        raise exc\n\n    return container"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef containerIsRunning(name_or_id):\n    '''Check if container with the given name or ID (str) is running. No side\n    effects. Idempotent. Returns True if running, False if not.'''\n    require_str(\"name_or_id\", name_or_id)\n\n    try:\n        container = getContainer(name_or_id)\n        # Refer to the latest status list here: https://docs.docker.com/engine/\n        #   api/v1.33/#operation/ContainerList\n        if container:\n            if container.status == 'created':\n              return False\n            elif container.status == 'restarting':\n              return True\n            elif container.status == 'running':\n              return True\n            elif container.status == 'removing':\n              return False\n            elif container.status == 'paused':\n              return False\n            elif container.status == 'exited':\n              return False\n            elif container.status == 'dead':\n              return False\n            else:\n              return False\n    except NotFound as exc:\n        return False\n\n    return False", "response": "Check if a container with the given name or ID is running. No side\n    effects. Idempotent. Returns True if running False if not."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nchecking if a container with a given tag exists. No side - effects. Idempotent. Returns None if the container is not found.", "response": "def getContainerByTag(tag):\n    '''Check if a container with a given tag exists. No side-effects.\n    Idempotent. Handles NotFound and APIError exceptions, but only reraises\n    APIError. Returns None if the container is not found. Otherwise, returns the\n    container.'''\n    require_str(\"tag\", tag)\n\n    container = None\n    try:\n        container = client.containers.get(tag)\n        print(\"Found container\", tag, \"...\")\n    except NotFound:\n        #print(\"Container\", tag, \"does not exist ...\")\n        pass\n    except APIError as exc:\n        eprint(\"Unhandled error while getting container\", tag)\n        raise exc\n\n    return container"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef removeContainer(tag):\n    '''Check if a container with a given tag exists. Kill it if it exists.\n    No extra side effects. Handles and reraises TypeError, and\n    APIError exceptions.\n    '''\n    container = getContainerByTag(tag)\n\n    if container:\n        # Build an Image using the dockerfile in the path\n        try:\n            container.remove(force=True)\n            #print(\"Removed container\", tag, \"...\")\n        except APIError as exc:\n            eprint(\"Unhandled error while removing container\", tag)\n            raise exc", "response": "Remove a container with a given tag."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nstart the indy_pool docker container iff it is not already running.", "response": "def startIndyPool(**kwargs):\n    '''Start the indy_pool docker container iff it is not already running. See\n    <indy-sdk>/ci/indy-pool.dockerfile for details. Idempotent. Simply ensures\n    that the indy_pool container is up and running.'''\n\n    # TODO: Decide if we need a separate docker container for testing and one for\n    #       development. The indy_sdk tests setup and teardown \"indy_pool\" on\n    #       ports 9701-9708. Perhaps we need an \"indy_dev_pool\" on 9709-9716? I'm\n    #       not quite sure where all of our dependencies are on port 9701-9708.\n    #       If the test harness (mocks) are hardcoding 9701-9708, then an\n    #       indy_dev_pool on different ports will not work.\n\n    print(\"Starting indy_pool ...\")\n\n    # Check if indy_pool is running\n    if containerIsRunning(\"indy_pool\"):\n        print(\"... already running\")\n        exit(0)\n    else:\n        # If the container already exists and isn't running, force remove it and\n        # readd it. This is brute force, but is sufficient and simple.\n        container = getContainer(\"indy_pool\")\n        if container:\n          container.remove(force=True)\n\n    # Build and run indy_pool if it is not already running\n    # Build the indy_pool image from the dockerfile in:\n    #   /vagrant/indy-sdk/ci/indy-pool.dockerfile\n    # \n    # In shell using docker cli:\n    # cd /vagrant/indy-sdk\n    # sudo docker build -f ci/indy-pool.dockerfile -t indy_pool .\n    #\n    # NOTE: https://jira.hyperledger.org/browse/IS-406 prevents indy_pool from\n    #       starting on the `rc` branch. Apply the patch in the Jira issue to\n    #       overcome this problem.\n\n    try:\n      # indy-sdk won't be in /vagrant if the indy-sdk is cloned to a directory outside\n      # the Vagrant project. Regardless of where indy-sdk is cloned, it will be found\n      # in /src/indy-sdk in the Vagrant VM.\n      image = getImage(path=\"/src/indy-sdk\", dockerfile=\"ci/indy-pool.dockerfile\",\n        tag=\"indy_pool\")\n    except TypeError as exc:\n      image = getImage(path=\"/vagrant/indy-sdk\", dockerfile=\"ci/indy-pool.dockerfile\",\n        tag=\"indy_pool\")\n    except:\n      print(\"Failed to find indy-pool.dockerfile in /vagrant/indy-sdk or /src/indy-sdk\")\n\n    # Run a container using the image\n    #\n    # In shell using docker cli:\n    # sudo docker run -itd -p 9701-9708:9701-9708 indy_pool\n    #\n    # NOTE: {'2222/tcp': 3333} is sufficient. A tuple of (address, port) if you\n    #       want to specify the host interface. \n    container = runContainer(image, ports={\n        '9701/tcp': ('0.0.0.0', 9701),\n        '9702/tcp': ('0.0.0.0', 9702),\n        '9703/tcp': ('0.0.0.0', 9703),\n        '9704/tcp': ('0.0.0.0', 9704),\n        '9705/tcp': ('0.0.0.0', 9705),\n        '9706/tcp': ('0.0.0.0', 9706),\n        '9707/tcp': ('0.0.0.0', 9707),\n        '9708/tcp': ('0.0.0.0', 9708)\n      }, detach=True, name=\"indy_pool\"\n    )\n\n    print(\"...started\")"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef restartIndyPool(**kwargs):\n    '''Restart the indy_pool docker container. Idempotent. Ensures that the\n    indy_pool container is a new running instance.'''\n    print(\"Restarting...\")\n    try:\n      stopIndyPool()\n      startIndyPool()\n      print(\"...restarted\")\n    except Exception as exc:\n      eprint(\"...failed to restart\")\n      raise exc", "response": "Restart the indy_pool docker container. Idempotent. Ensures that the indy_pool docker container is a new running instance."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef signingPayloadState(self, identifier=None):\n        if self.operation.get(TXN_TYPE) == ATTRIB:\n            d = deepcopy(super().signingPayloadState(identifier=identifier))\n            op = d[OPERATION]\n            keyName = {RAW, ENC, HASH}.intersection(set(op.keys())).pop()\n            op[keyName] = sha256(op[keyName].encode()).hexdigest()\n            return d\n        return super().signingPayloadState(identifier=identifier)", "response": "Special signing state where the data for an attribute is hashed\n        before signing\n       ."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _migrate_ledger(data_directory,\n                    old_ledger_file, new_ledger_file,\n                    serializer: MappingSerializer = None):\n    \"\"\"\n    Test for the directory, open old and new ledger, migrate data, rename directories\n    \"\"\"\n\n    # we should have ChunkedFileStorage implementation of the Ledger\n    if not os.path.isdir(os.path.join(data_directory, old_ledger_file)):\n        msg = 'Could not find directory {} for migration.'.format(\n            old_ledger_file)\n        logger.error(msg)\n        raise Exception(msg)\n\n    # open the old ledger using the specified serializer\n    old_ledger_file_backup = old_ledger_file + \"_new\"\n    old_txn_log_store = ChunkedFileStore(data_directory,\n                                         old_ledger_file_backup,\n                                         isLineNoKey=True,\n                                         storeContentHash=False)\n    old_ledger = Ledger(CompactMerkleTree(),\n                        dataDir=data_directory,\n                        txn_serializer=serializer,\n                        hash_serializer=serializer,\n                        fileName=old_ledger_file_backup,\n                        transactionLogStore=old_txn_log_store)\n\n    # open the new ledger with new serialization\n    new_ledger = Ledger(CompactMerkleTree(),\n                        dataDir=data_directory,\n                        fileName=new_ledger_file)\n    logger.info(\"new size for {}: {}\".format(\n        old_ledger_file_backup, str(new_ledger.size)))\n\n    # add all txns into the old ledger\n    for _, txn in new_ledger.getAllTxn():\n        old_ledger.add(txn)\n    logger.info(\"old size for {}: {}\".format(\n        new_ledger_file, str(old_ledger.size)))\n\n    old_ledger.stop()\n    new_ledger.stop()\n\n    # now that everything succeeded, remove the new files and move the old\n    # files into place\n    shutil.rmtree(\n        os.path.join(data_directory, new_ledger_file))\n    os.rename(\n        os.path.join(data_directory, old_ledger_file_backup),\n        os.path.join(data_directory, old_ledger_file))", "response": "Test for the directory open old and new ledger and migrate data"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_role(self, request: Request):\n        idr = request.identifier\n        return self._get_role(idr)", "response": "Returns the role corresponding to the DID in the request."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef handleConfigTxn(self, txn) -> None:\n        if get_type(txn) == POOL_CONFIG:\n            self.writes = get_payload_data(txn)[WRITES]", "response": "Handles transaction of type POOL_CONFIG"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef processLedger(self) -> None:\n        logger.debug('{} processing config ledger for any POOL_CONFIGs'.format(\n            self), extra={\"tags\": [\"pool-config\"]})\n        for _, txn in self.ledger.getAllTxn():\n            if get_type(txn) == POOL_CONFIG:\n                self.handleConfigTxn(txn)", "response": "Checks ledger config txns and perfomes recent one\n       "}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef handleRestartRequest(self, req: Request) -> None:\n        txn = req.operation\n        if txn[TXN_TYPE] != POOL_RESTART:\n            return\n\n        action = txn[ACTION]\n        if action == START:\n            when = dateutil.parser.parse(txn[DATETIME]) \\\n                if DATETIME in txn.keys() and txn[DATETIME] not in [\"0\", \"\", None] \\\n                else None\n            fail_timeout = txn.get(TIMEOUT, self.defaultActionTimeout)\n            self.requestRestart(when, fail_timeout)\n            return\n\n        if action == CANCEL:\n            if self.scheduledAction:\n                self._cancelScheduledRestart()\n                logger.info(\"Node '{}' cancels restart\".format(\n                    self.nodeName))\n            return\n\n        logger.error(\n            \"Got {} transaction with unsupported action {}\".format(\n                POOL_RESTART, action))", "response": "Handles a POOL_RESTART transaction."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _scheduleRestart(self,\n                         when: Union[datetime, str],\n                         failTimeout) -> None:\n        \"\"\"\n        Schedules node restart to a newer version\n\n        :param version: version to restart to\n        :param when: restart time\n        \"\"\"\n        logger.info(\"{}'s restarter processing restart\"\n                    .format(self))\n        if isinstance(when, str):\n            when = dateutil.parser.parse(when)\n        now = datetime.utcnow().replace(tzinfo=dateutil.tz.tzutc())\n\n        logger.info(\n            \"Restart of node '{}' has been scheduled on {}\".format(\n                self.nodeName, when))\n\n        ev_data = RestartLogData(when)\n        self._actionLog.append_scheduled(ev_data)\n\n        callAgent = partial(self._callRestartAgent, ev_data,\n                            failTimeout)\n        delay = 0\n        if now < when:\n            delay = (when - now).total_seconds()\n        self.scheduledAction = ev_data\n        self._schedule(callAgent, delay)", "response": "Schedules a restart of the node to a newer version."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _cancelScheduledRestart(self, justification=None) -> None:\n\n        if self.scheduledAction:\n            why_prefix = \": \"\n            why = justification\n            if justification is None:\n                why_prefix = \", \"\n                why = \"cancellation reason not specified\"\n\n            ev_data = self.scheduledAction\n            logger.info(\"Cancelling restart\"\n                        \" of node {}\"\n                        \" scheduled on {}\"\n                        \"{}{}\"\n                        .format(self.nodeName,\n                                ev_data.when,\n                                why_prefix,\n                                why))\n\n            self._unscheduleAction()\n            self._actionLog.append_cancelled(ev_data)\n            logger.info(\n                \"Restart of node '{}'\"\n                \"has been cancelled due to {}\".format(\n                    self.nodeName, why))", "response": "Cancels scheduled restart of the current node."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _callRestartAgent(self, ev_data: RestartLogData, failTimeout) -> None:\n\n        logger.info(\"{}'s restart calling agent for restart\".format(self))\n        self._actionLog.append_started(ev_data)\n        self._action_start_callback()\n        self.scheduledAction = None\n        asyncio.ensure_future(\n            self._sendUpdateRequest(ev_data, failTimeout))", "response": "Callback which is called when restart time come."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _declareTimeoutExceeded(self, ev_data: RestartLogData):\n        logger.info(\"Timeout exceeded for {}\".format(ev_data.when))\n        last = self._actionLog.last_event\n        if (last and last.ev_type == RestartLog.Events.failed and\n                last.data == ev_data):\n            return None\n\n        self._action_failed(ev_data,\n                            reason=\"exceeded restart timeout\")\n\n        self._unscheduleAction()\n        self._actionFailedCallback()", "response": "This function is called when time for restart is up."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nprocess the config ledger for any upgrades and schedules upgrade for the most recent one", "response": "def processLedger(self) -> None:\n        \"\"\"\n        Checks ledger for planned but not yet performed upgrades\n        and schedules upgrade for the most recent one\n\n        Assumption: Only version is enough to identify a release, no hash\n        checking is done\n        :return:\n        \"\"\"\n        logger.debug(\n            '{} processing config ledger for any upgrades'.format(self))\n        last_pool_upgrade_txn_start = self.get_upgrade_txn(\n            lambda txn: get_type(txn) == POOL_UPGRADE and get_payload_data(txn)[ACTION] == START, reverse=True)\n        if last_pool_upgrade_txn_start:\n            logger.info('{} found upgrade START txn {}'.format(\n                self, last_pool_upgrade_txn_start))\n            last_pool_upgrade_txn_seq_no = get_seq_no(last_pool_upgrade_txn_start)\n\n            # searching for CANCEL for this upgrade submitted after START txn\n            last_pool_upgrade_txn_cancel = self.get_upgrade_txn(\n                lambda txn:\n                get_type(txn) == POOL_UPGRADE and get_payload_data(txn)[ACTION] == CANCEL and\n                get_payload_data(txn)[VERSION] == get_payload_data(last_pool_upgrade_txn_start)[VERSION],\n                start_no=last_pool_upgrade_txn_seq_no + 1)\n            if last_pool_upgrade_txn_cancel:\n                logger.info('{} found upgrade CANCEL txn {}'.format(\n                    self, last_pool_upgrade_txn_cancel))\n                return\n\n            self.handleUpgradeTxn(last_pool_upgrade_txn_start)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef didLastExecutedUpgradeSucceeded(self) -> bool:\n        lastEventInfo = self.lastActionEventInfo\n        if lastEventInfo:\n            ev_data = lastEventInfo.data\n            currentPkgVersion = NodeControlUtil.curr_pkg_info(ev_data.pkg_name)[0]\n            if currentPkgVersion:\n                return currentPkgVersion.upstream == ev_data.version\n            else:\n                logger.warning(\n                    \"{} failed to get information about package {} \"\n                    \"scheduled for last upgrade\"\n                    .format(self, ev_data.pkg_name)\n                )\n        return False", "response": "Checks if the last record in upgrade log is about scheduling upgrade. If so checks whether current version is equal to the one in that record. If so checks whether current version is equal to the one in that record. If so returns True."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef handleUpgradeTxn(self, txn) -> None:\n        FINALIZING_EVENT_TYPES = [UpgradeLog.Events.succeeded, UpgradeLog.Events.failed]\n\n        if get_type(txn) != POOL_UPGRADE:\n            return\n\n        logger.info(\"Node '{}' handles upgrade txn {}\".format(self.nodeName, txn))\n        txn_data = get_payload_data(txn)\n        action = txn_data[ACTION]\n        version = txn_data[VERSION]\n        justification = txn_data.get(JUSTIFICATION)\n        pkg_name = txn_data.get(PACKAGE, self.config.UPGRADE_ENTRY)\n        upgrade_id = self.get_action_id(txn)\n\n        # TODO test\n        try:\n            version = src_version_cls(pkg_name)(version)\n        except InvalidVersionError as exc:\n            logger.warning(\n                \"{} can't handle upgrade txn with version {} for package {}: {}\"\n                .format(self, version, pkg_name, exc)\n            )\n            return\n\n        if action == START:\n            # forced txn could have partial schedule list\n            if self.nodeId not in txn_data[SCHEDULE]:\n                logger.info(\"Node '{}' disregards upgrade txn {}\".format(\n                    self.nodeName, txn))\n                return\n\n            last_event = self.lastActionEventInfo\n            if last_event:\n                if (last_event.data.upgrade_id == upgrade_id and\n                        last_event.ev_type in FINALIZING_EVENT_TYPES):\n                    logger.info(\n                        \"Node '{}' has already performed an upgrade with upgrade_id {}. \"\n                        \"Last recorded event is {}\"\n                        .format(self.nodeName, upgrade_id, last_event.data))\n                    return\n\n            when = txn_data[SCHEDULE][self.nodeId]\n            failTimeout = txn_data.get(TIMEOUT, self.defaultActionTimeout)\n\n            if isinstance(when, str):\n                when = dateutil.parser.parse(when)\n\n            new_ev_data = UpgradeLogData(when, version, upgrade_id, pkg_name)\n\n            if self.scheduledAction:\n                if self.scheduledAction == new_ev_data:\n                    logger.debug(\n                        \"Node {} already scheduled upgrade to version '{}' \"\n                        .format(self.nodeName, version))\n                    return\n                else:\n                    logger.info(\n                        \"Node '{}' cancels previous upgrade and schedules a new one to {}\"\n                        .format(self.nodeName, version))\n                    self._cancelScheduledUpgrade(justification)\n\n            logger.info(\"Node '{}' schedules upgrade to {}\".format(self.nodeName, version))\n\n            self._scheduleUpgrade(new_ev_data, failTimeout)\n            return\n\n        if action == CANCEL:\n            if (self.scheduledAction and\n                    self.scheduledAction.version == version):\n                self._cancelScheduledUpgrade(justification)\n                logger.info(\"Node '{}' cancels upgrade to {}\".format(\n                    self.nodeName, version))\n            return\n\n        logger.error(\n            \"Got {} transaction with unsupported action {}\".format(\n                POOL_UPGRADE, action))", "response": "Handles the transaction of type POOL_UPGRADE and returns the new version of the upgrade."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _scheduleUpgrade(self,\n                         ev_data: UpgradeLogData,\n                         failTimeout) -> None:\n        \"\"\"\n        Schedules node upgrade to a newer version\n\n        :param ev_data: upgrade event parameters\n        \"\"\"\n        logger.info(\n            \"{}'s upgrader processing upgrade for version {}={}\"\n            .format(self, ev_data.pkg_name, ev_data.version))\n        now = datetime.utcnow().replace(tzinfo=dateutil.tz.tzutc())\n\n        self._notifier.sendMessageUponNodeUpgradeScheduled(\n            \"Upgrade of package {} on node '{}' to version {} \"\n            \"has been scheduled on {}\"\n            .format(ev_data.pkg_name, self.nodeName,\n                    ev_data.version, ev_data.when))\n        self._actionLog.append_scheduled(ev_data)\n\n        callAgent = partial(self._callUpgradeAgent, ev_data, failTimeout)\n        delay = 0\n        if now < ev_data.when:\n            delay = (ev_data.when - now).total_seconds()\n        self.scheduledAction = ev_data\n        self._schedule(callAgent, delay)", "response": "Schedules a node upgrade to a newer version."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _cancelScheduledUpgrade(self, justification=None) -> None:\n\n        if self.scheduledAction:\n            why_prefix = \": \"\n            why = justification\n            if justification is None:\n                why_prefix = \", \"\n                why = \"cancellation reason not specified\"\n\n            ev_data = self.scheduledAction\n            logger.info(\"Cancelling upgrade {}\"\n                        \" of node {}\"\n                        \" of package {}\"\n                        \" to version {}\"\n                        \" scheduled on {}\"\n                        \"{}{}\"\n                        .format(ev_data.upgrade_id,\n                                self.nodeName,\n                                ev_data.pkg_name,\n                                ev_data.version,\n                                ev_data.when,\n                                why_prefix,\n                                why))\n\n            self._unscheduleAction()\n            self._actionLog.append_cancelled(ev_data)\n            self._notifier.sendMessageUponPoolUpgradeCancel(\n                \"Upgrade of package {} on node '{}' to version {} \"\n                \"has been cancelled due to {}\"\n                .format(ev_data.pkg_name, self.nodeName,\n                        ev_data.version, why))", "response": "Cancels scheduled upgrade of the current version of the package."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _callUpgradeAgent(self, ev_data, failTimeout) -> None:\n\n        logger.info(\"{}'s upgrader calling agent for upgrade\".format(self))\n        self._actionLog.append_started(ev_data)\n        self._action_start_callback()\n        self.scheduledAction = None\n        asyncio.ensure_future(\n            self._sendUpgradeRequest(ev_data, failTimeout))", "response": "Callback which is called when upgrade time come."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nchecks if the schedule of planned node upgrades is valid.", "response": "def isScheduleValid(self, schedule, node_srvs, force) -> (bool, str):\n        \"\"\"\n        Validates schedule of planned node upgrades\n\n        :param schedule: dictionary of node ids and upgrade times\n        :param node_srvs: dictionary of node ids and services\n        :return: a 2-tuple of whether schedule valid or not and the reason\n        \"\"\"\n\n        # flag \"force=True\" ignore basic checks! only datetime format is\n        # checked\n        times = []\n        non_demoted_nodes = set([k for k, v in node_srvs.items() if v])\n        if not force and set(schedule.keys()) != non_demoted_nodes:\n            return False, 'Schedule should contain id of all nodes'\n        now = datetime.utcnow().replace(tzinfo=dateutil.tz.tzutc())\n        for dateStr in schedule.values():\n            try:\n                when = dateutil.parser.parse(dateStr)\n                if when <= now and not force:\n                    return False, '{} is less than current time'.format(when)\n                times.append(when)\n            except ValueError:\n                return False, '{} cannot be parsed to a time'.format(dateStr)\n        if force:\n            return True, ''\n        times = sorted(times)\n        for i in range(len(times) - 1):\n            diff = (times[i + 1] - times[i]).total_seconds()\n            if diff < self.config.MinSepBetweenNodeUpgrades:\n                return False, 'time span between upgrades is {} ' \\\n                              'seconds which is less than specified ' \\\n                              'in the config'.format(diff)\n        return True, ''"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_token(self, url):\n        # A hack to avoid url-encoding the url, since the authorization service\n        # doesn't work with correctly encoded urls\n\n        parsed_url = urlparse.urlsplit(url)\n        parsed_url = parsed_url._replace(path='/authorization/api')\n        self.url = urlparse.urlunsplit(parsed_url)\n\n        response = self.request(method='GET', url='/v1/token?url=' + url)\n        return response.result.text", "response": "Retrieves a temporary access token for the given url."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef synthesize_using_websocket(self,\n                                   text,\n                                   synthesize_callback,\n                                   accept=None,\n                                   voice=None,\n                                   timings=None,\n                                   customization_id=None,\n                                   http_proxy_host=None,\n                                   http_proxy_port=None,\n                                   **kwargs):\n        \"\"\"\n        Synthesizes text to spoken audio using web sockets. It supports the use of\n        the SSML <mark> element to identify the location of user-specified markers in the audio.\n        It can also return timing information for all strings of the input text.\n        Note:The service processes one request per connection.\n\n        :param str text: Provides the text that is to be synthesized. The client can pass plain\n        text or text that is annotated with the Speech Synthesis Markup Language (SSML). For more\n        information, see [Specifying input text](https://console.bluemix.net/docs/services/text-to-speech/http.html#input).\n        SSML input can also include the <mark> element;\n        see [Specifying an SSML mark](https://console.bluemix.net/docs/services/text-to-speech/word-timing.html#mark).\n        The client can pass a maximum of 5 KB of text with the request.\n        :param SynthesizeCallback synthesize_callback: The callback method for the websocket.\n        :param str accept: Specifies the requested format (MIME type) of the audio. For more information, see [Specifying\n        an audio format](https://console.bluemix.net/docs/services/text-to-speech/http.html#format). In addition to the\n        supported specifications, you can use */* to specify the default audio format, audio/ogg;codecs=opus.\n        :param str voice: The voice to use for synthesis.\n        :param list[str] timings: Specifies that the service is to return word timing information for all strings of the\n        input text. The service returns the start and end time of each string of the input. Specify words as the lone element\n        of the array to request word timings. Specify an empty array or omit the parameter to receive no word timings. For\n        more information, see [Obtaining word timings](https://console.bluemix.net/docs/services/text-to-speech/word-timing.html#timing).\n        Not supported for Japanese input text.\n        :param str customization_id: Specifies the globally unique identifier (GUID) for a custom voice model that is to be used for the\n        synthesis. A custom voice model is guaranteed to work only if it matches the language of the voice that is used for the synthesis.\n        If you include a customization ID, you must call the method with the service credentials of the custom model's owner. Omit the\n        parameter to use the specified voice with no customization. For more information, see [Understanding customization]\n        (https://console.bluemix.net/docs/services/text-to-speech/custom-intro.html#customIntro).\n        :param str http_proxy_host: http proxy host name.\n        :param str http_proxy_port: http proxy port. If not set, set to 80.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `dict` containing the `SpeechRecognitionResults` response.\n        :rtype: dict\n        \"\"\"\n        if text is None:\n            raise ValueError('text must be provided')\n        if synthesize_callback is None:\n            raise ValueError('synthesize_callback must be provided')\n        if not isinstance(synthesize_callback, SynthesizeCallback):\n            raise Exception(\n                'Callback is not a derived class of SynthesizeCallback')\n\n        headers = {}\n        if self.default_headers is not None:\n            headers = self.default_headers.copy()\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n\n        if self.token_manager:\n            access_token = self.token_manager.get_token()\n            headers['Authorization'] = '{0} {1}'.format(BEARER, access_token)\n        else:\n            authstring = \"{0}:{1}\".format(self.username, self.password)\n            base64_authorization = base64.b64encode(authstring.encode('utf-8')).decode('utf-8')\n            headers['Authorization'] = 'Basic {0}'.format(base64_authorization)\n\n        url = self.url.replace('https:', 'wss:')\n        params = {\n            'voice': voice,\n            'customization_id': customization_id,\n        }\n        params = dict([(k, v) for k, v in params.items() if v is not None])\n        url += '/v1/synthesize?{0}'.format(urlencode(params))\n\n        options = {\n            'text': text,\n            'accept': accept,\n            'timings': timings\n        }\n        options = dict([(k, v) for k, v in options.items() if v is not None])\n\n        SynthesizeListener(options,\n                           synthesize_callback,\n                           url,\n                           headers,\n                           http_proxy_host,\n                           http_proxy_port,\n                           self.verify)", "response": "Synthesizes text using web sockets."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nclassify images. Classify images with built-in or custom classifiers. :param file images_file: An image file (.gif, .jpg, .png, .tif) or .zip file with images. Maximum image size is 10 MB. Include no more than 20 images and limit the .zip file to 100 MB. Encode the image and .zip file names in UTF-8 if they contain non-ASCII characters. The service assumes UTF-8 encoding if it encounters non-ASCII characters. You can also include an image with the **url** parameter. :param str images_filename: The filename for images_file. :param str images_file_content_type: The content type of images_file. :param str url: The URL of an image (.gif, .jpg, .png, .tif) to analyze. The minimum recommended pixel density is 32X32 pixels, but the service tends to perform better with images that are at least 224 x 224 pixels. The maximum image size is 10 MB. You can also include images with the **images_file** parameter. :param float threshold: The minimum score a class must have to be displayed in the response. Set the threshold to `0.0` to return all identified classes. :param list[str] owners: The categories of classifiers to apply. The **classifier_ids** parameter overrides **owners**, so make sure that **classifier_ids** is empty. - Use `IBM` to classify against the `default` general classifier. You get the same result if both **classifier_ids** and **owners** parameters are empty. - Use `me` to classify against all your custom classifiers. However, for better performance use **classifier_ids** to specify the specific custom classifiers to apply. - Use both `IBM` and `me` to analyze the image against both classifier categories. :param list[str] classifier_ids: Which classifiers to apply. Overrides the **owners** parameter. You can specify both custom and built-in classifier IDs. The built-in `default` classifier is used if both **classifier_ids** and **owners** parameters are empty. The following built-in classifier IDs require no training: - `default`: Returns classes from thousands of general tags. - `food`: Enhances specificity and accuracy for images of food items. - `explicit`: Evaluates whether the image might be pornographic. :param str accept_language: The desired language of parts of the response. See the response for details. :param dict headers: A `dict` containing the request headers :return: A `DetailedResponse` containing the result, headers and HTTP status code. :rtype: DetailedResponse", "response": "def classify(self,\n                 images_file=None,\n                 images_filename=None,\n                 images_file_content_type=None,\n                 url=None,\n                 threshold=None,\n                 owners=None,\n                 classifier_ids=None,\n                 accept_language=None,\n                 **kwargs):\n        \"\"\"\n        Classify images.\n\n        Classify images with built-in or custom classifiers.\n\n        :param file images_file: An image file (.gif, .jpg, .png, .tif) or .zip file with\n        images. Maximum image size is 10 MB. Include no more than 20 images and limit the\n        .zip file to 100 MB. Encode the image and .zip file names in UTF-8 if they contain\n        non-ASCII characters. The service assumes UTF-8 encoding if it encounters\n        non-ASCII characters.\n        You can also include an image with the **url** parameter.\n        :param str images_filename: The filename for images_file.\n        :param str images_file_content_type: The content type of images_file.\n        :param str url: The URL of an image (.gif, .jpg, .png, .tif) to analyze. The\n        minimum recommended pixel density is 32X32 pixels, but the service tends to\n        perform better with images that are at least 224 x 224 pixels. The maximum image\n        size is 10 MB.\n        You can also include images with the **images_file** parameter.\n        :param float threshold: The minimum score a class must have to be displayed in the\n        response. Set the threshold to `0.0` to return all identified classes.\n        :param list[str] owners: The categories of classifiers to apply. The\n        **classifier_ids** parameter overrides **owners**, so make sure that\n        **classifier_ids** is empty.\n        - Use `IBM` to classify against the `default` general classifier. You get the same\n        result if both **classifier_ids** and **owners** parameters are empty.\n        - Use `me` to classify against all your custom classifiers. However, for better\n        performance use **classifier_ids** to specify the specific custom classifiers to\n        apply.\n        - Use both `IBM` and `me` to analyze the image against both classifier categories.\n        :param list[str] classifier_ids: Which classifiers to apply. Overrides the\n        **owners** parameter. You can specify both custom and built-in classifier IDs. The\n        built-in `default` classifier is used if both **classifier_ids** and **owners**\n        parameters are empty.\n        The following built-in classifier IDs require no training:\n        - `default`: Returns classes from thousands of general tags.\n        - `food`: Enhances specificity and accuracy for images of food items.\n        - `explicit`: Evaluates whether the image might be pornographic.\n        :param str accept_language: The desired language of parts of the response. See the\n        response for details.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse\n        \"\"\"\n\n        headers = {'Accept-Language': accept_language}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('watson_vision_combined', 'V3',\n                                      'classify')\n        headers.update(sdk_headers)\n\n        params = {'version': self.version}\n\n        form_data = {}\n        if images_file:\n            if not images_filename and hasattr(images_file, 'name'):\n                images_filename = basename(images_file.name)\n            if not images_filename:\n                raise ValueError('images_filename must be provided')\n            form_data['images_file'] = (images_filename, images_file,\n                                        images_file_content_type or\n                                        'application/octet-stream')\n        if url:\n            form_data['url'] = (None, url, 'text/plain')\n        if threshold:\n            form_data['threshold'] = (None, threshold, 'application/json')\n        if owners:\n            owners = self._convert_list(owners)\n            form_data['owners'] = (None, owners, 'application/json')\n        if classifier_ids:\n            classifier_ids = self._convert_list(classifier_ids)\n            form_data['classifier_ids'] = (None, classifier_ids,\n                                           'application/json')\n\n        url = '/v3/classify'\n        response = self.request(\n            method='POST',\n            url=url,\n            headers=headers,\n            params=params,\n            files=form_data,\n            accept_json=True)\n        return response"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef create_classifier(self,\n                          name,\n                          positive_examples,\n                          negative_examples=None,\n                          negative_examples_filename=None,\n                          **kwargs):\n        \"\"\"\n        Create a classifier.\n\n        Train a new multi-faceted classifier on the uploaded image data. Create your\n        custom classifier with positive or negative examples. Include at least two sets of\n        examples, either two positive example files or one positive and one negative file.\n        You can upload a maximum of 256 MB per call.\n        Encode all names in UTF-8 if they contain non-ASCII characters (.zip and image\n        file names, and classifier and class names). The service assumes UTF-8 encoding if\n        it encounters non-ASCII characters.\n\n        :param str name: The name of the new classifier. Encode special characters in\n        UTF-8.\n        :param dict positive_examples: A dictionary that contains the value for each\n        classname. The value is a .zip file of images that depict the visual subject of a\n        class in the new classifier. You can include more than one positive example file\n        in a call.\n        Specify the parameter name by appending `_positive_examples` to the class name.\n        For example, `goldenretriever_positive_examples` creates the class\n        **goldenretriever**.\n        Include at least 10 images in .jpg or .png format. The minimum recommended image\n        resolution is 32X32 pixels. The maximum number of images is 10,000 images or 100\n        MB per .zip file.\n        Encode special characters in the file name in UTF-8.\n        :param file negative_examples: A .zip file of images that do not depict the visual\n        subject of any of the classes of the new classifier. Must contain a minimum of 10\n        images.\n        Encode special characters in the file name in UTF-8.\n        :param str negative_examples_filename: The filename for negative_examples.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse\n        \"\"\"\n\n        if name is None:\n            raise ValueError('name must be provided')\n        if not positive_examples:\n            raise ValueError('positive_examples must be provided')\n\n        headers = {}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('watson_vision_combined', 'V3',\n                                      'create_classifier')\n        headers.update(sdk_headers)\n\n        params = {'version': self.version}\n\n        form_data = {}\n        form_data['name'] = (None, name, 'text/plain')\n        for key in positive_examples.keys():\n            part_name = '%s_positive_examples' % (key)\n            value = positive_examples[key]\n            if hasattr(value, 'name'):\n                filename = basename(value.name)\n            form_data[part_name] = (filename, value, 'application/octet-stream')\n        if negative_examples:\n            if not negative_examples_filename and hasattr(\n                    negative_examples, 'name'):\n                negative_examples_filename = basename(negative_examples.name)\n            if not negative_examples_filename:\n                raise ValueError('negative_examples_filename must be provided')\n            form_data['negative_examples'] = (negative_examples_filename,\n                                              negative_examples,\n                                              'application/octet-stream')\n\n        url = '/v3/classifiers'\n        response = self.request(\n            method='POST',\n            url=url,\n            headers=headers,\n            params=params,\n            files=form_data,\n            accept_json=True)\n        return response", "response": "Create a new multi - faceted classifier."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef delete_classifier(self, classifier_id, **kwargs):\n\n        if classifier_id is None:\n            raise ValueError('classifier_id must be provided')\n\n        headers = {}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('watson_vision_combined', 'V3',\n                                      'delete_classifier')\n        headers.update(sdk_headers)\n\n        params = {'version': self.version}\n\n        url = '/v3/classifiers/{0}'.format(\n            *self._encode_path_vars(classifier_id))\n        response = self.request(\n            method='DELETE',\n            url=url,\n            headers=headers,\n            params=params,\n            accept_json=True)\n        return response", "response": "Delete a classifier.\n\n        :param str classifier_id: The ID of the classifier.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'class_name') and self.class_name is not None:\n            _dict['class'] = self.class_name\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'class_name') and self.class_name is not None:\n            _dict['class'] = self.class_name\n        if hasattr(self, 'score') and self.score is not None:\n            _dict['score'] = self.score\n        if hasattr(self, 'type_hierarchy') and self.type_hierarchy is not None:\n            _dict['type_hierarchy'] = self.type_hierarchy\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'source_url' in _dict:\n            args['source_url'] = _dict.get('source_url')\n        if 'resolved_url' in _dict:\n            args['resolved_url'] = _dict.get('resolved_url')\n        if 'image' in _dict:\n            args['image'] = _dict.get('image')\n        if 'error' in _dict:\n            args['error'] = ErrorInfo._from_dict(_dict.get('error'))\n        if 'classifiers' in _dict:\n            args['classifiers'] = [\n                ClassifierResult._from_dict(x)\n                for x in (_dict.get('classifiers'))\n            ]\n        else:\n            raise ValueError(\n                'Required property \\'classifiers\\' not present in ClassifiedImage JSON'\n            )\n        return cls(**args)", "response": "Initialize a ClassifiedImage object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'source_url') and self.source_url is not None:\n            _dict['source_url'] = self.source_url\n        if hasattr(self, 'resolved_url') and self.resolved_url is not None:\n            _dict['resolved_url'] = self.resolved_url\n        if hasattr(self, 'image') and self.image is not None:\n            _dict['image'] = self.image\n        if hasattr(self, 'error') and self.error is not None:\n            _dict['error'] = self.error._to_dict()\n        if hasattr(self, 'classifiers') and self.classifiers is not None:\n            _dict['classifiers'] = [x._to_dict() for x in self.classifiers]\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'classifier_id' in _dict:\n            args['classifier_id'] = _dict.get('classifier_id')\n        else:\n            raise ValueError(\n                'Required property \\'classifier_id\\' not present in Classifier JSON'\n            )\n        if 'name' in _dict:\n            args['name'] = _dict.get('name')\n        else:\n            raise ValueError(\n                'Required property \\'name\\' not present in Classifier JSON')\n        if 'owner' in _dict:\n            args['owner'] = _dict.get('owner')\n        if 'status' in _dict:\n            args['status'] = _dict.get('status')\n        if 'core_ml_enabled' in _dict:\n            args['core_ml_enabled'] = _dict.get('core_ml_enabled')\n        if 'explanation' in _dict:\n            args['explanation'] = _dict.get('explanation')\n        if 'created' in _dict:\n            args['created'] = string_to_datetime(_dict.get('created'))\n        if 'classes' in _dict:\n            args['classes'] = [\n                Class._from_dict(x) for x in (_dict.get('classes'))\n            ]\n        if 'retrained' in _dict:\n            args['retrained'] = string_to_datetime(_dict.get('retrained'))\n        if 'updated' in _dict:\n            args['updated'] = string_to_datetime(_dict.get('updated'))\n        return cls(**args)", "response": "Initialize a Classifier object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'classifier_id') and self.classifier_id is not None:\n            _dict['classifier_id'] = self.classifier_id\n        if hasattr(self, 'name') and self.name is not None:\n            _dict['name'] = self.name\n        if hasattr(self, 'owner') and self.owner is not None:\n            _dict['owner'] = self.owner\n        if hasattr(self, 'status') and self.status is not None:\n            _dict['status'] = self.status\n        if hasattr(self,\n                   'core_ml_enabled') and self.core_ml_enabled is not None:\n            _dict['core_ml_enabled'] = self.core_ml_enabled\n        if hasattr(self, 'explanation') and self.explanation is not None:\n            _dict['explanation'] = self.explanation\n        if hasattr(self, 'created') and self.created is not None:\n            _dict['created'] = datetime_to_string(self.created)\n        if hasattr(self, 'classes') and self.classes is not None:\n            _dict['classes'] = [x._to_dict() for x in self.classes]\n        if hasattr(self, 'retrained') and self.retrained is not None:\n            _dict['retrained'] = datetime_to_string(self.retrained)\n        if hasattr(self, 'updated') and self.updated is not None:\n            _dict['updated'] = datetime_to_string(self.updated)\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ninitialize a ClassifierResult object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a ClassifierResult object from a json dictionary.\"\"\"\n        args = {}\n        if 'name' in _dict:\n            args['name'] = _dict.get('name')\n        else:\n            raise ValueError(\n                'Required property \\'name\\' not present in ClassifierResult JSON'\n            )\n        if 'classifier_id' in _dict:\n            args['classifier_id'] = _dict.get('classifier_id')\n        else:\n            raise ValueError(\n                'Required property \\'classifier_id\\' not present in ClassifierResult JSON'\n            )\n        if 'classes' in _dict:\n            args['classes'] = [\n                ClassResult._from_dict(x) for x in (_dict.get('classes'))\n            ]\n        else:\n            raise ValueError(\n                'Required property \\'classes\\' not present in ClassifierResult JSON'\n            )\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'name') and self.name is not None:\n            _dict['name'] = self.name\n        if hasattr(self, 'classifier_id') and self.classifier_id is not None:\n            _dict['classifier_id'] = self.classifier_id\n        if hasattr(self, 'classes') and self.classes is not None:\n            _dict['classes'] = [x._to_dict() for x in self.classes]\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'classifiers') and self.classifiers is not None:\n            _dict['classifiers'] = [x._to_dict() for x in self.classifiers]\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ninitializes a DetectedFaces object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a DetectedFaces object from a json dictionary.\"\"\"\n        args = {}\n        if 'images_processed' in _dict:\n            args['images_processed'] = _dict.get('images_processed')\n        else:\n            raise ValueError(\n                'Required property \\'images_processed\\' not present in DetectedFaces JSON'\n            )\n        if 'images' in _dict:\n            args['images'] = [\n                ImageWithFaces._from_dict(x) for x in (_dict.get('images'))\n            ]\n        else:\n            raise ValueError(\n                'Required property \\'images\\' not present in DetectedFaces JSON'\n            )\n        if 'warnings' in _dict:\n            args['warnings'] = [\n                WarningInfo._from_dict(x) for x in (_dict.get('warnings'))\n            ]\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self,\n                   'images_processed') and self.images_processed is not None:\n            _dict['images_processed'] = self.images_processed\n        if hasattr(self, 'images') and self.images is not None:\n            _dict['images'] = [x._to_dict() for x in self.images]\n        if hasattr(self, 'warnings') and self.warnings is not None:\n            _dict['warnings'] = [x._to_dict() for x in self.warnings]\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'code') and self.code is not None:\n            _dict['code'] = self.code\n        if hasattr(self, 'description') and self.description is not None:\n            _dict['description'] = self.description\n        if hasattr(self, 'error_id') and self.error_id is not None:\n            _dict['error_id'] = self.error_id\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ninitializing a Face object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a Face object from a json dictionary.\"\"\"\n        args = {}\n        if 'age' in _dict:\n            args['age'] = FaceAge._from_dict(_dict.get('age'))\n        if 'gender' in _dict:\n            args['gender'] = FaceGender._from_dict(_dict.get('gender'))\n        if 'face_location' in _dict:\n            args['face_location'] = FaceLocation._from_dict(\n                _dict.get('face_location'))\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'age') and self.age is not None:\n            _dict['age'] = self.age._to_dict()\n        if hasattr(self, 'gender') and self.gender is not None:\n            _dict['gender'] = self.gender._to_dict()\n        if hasattr(self, 'face_location') and self.face_location is not None:\n            _dict['face_location'] = self.face_location._to_dict()\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'min') and self.min is not None:\n            _dict['min'] = self.min\n        if hasattr(self, 'max') and self.max is not None:\n            _dict['max'] = self.max\n        if hasattr(self, 'score') and self.score is not None:\n            _dict['score'] = self.score\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'gender') and self.gender is not None:\n            _dict['gender'] = self.gender\n        if hasattr(self, 'gender_label') and self.gender_label is not None:\n            _dict['gender_label'] = self.gender_label\n        if hasattr(self, 'score') and self.score is not None:\n            _dict['score'] = self.score\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'width') and self.width is not None:\n            _dict['width'] = self.width\n        if hasattr(self, 'height') and self.height is not None:\n            _dict['height'] = self.height\n        if hasattr(self, 'left') and self.left is not None:\n            _dict['left'] = self.left\n        if hasattr(self, 'top') and self.top is not None:\n            _dict['top'] = self.top\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'warning_id') and self.warning_id is not None:\n            _dict['warning_id'] = self.warning_id\n        if hasattr(self, 'description') and self.description is not None:\n            _dict['description'] = self.description\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef message(self,\n                assistant_id,\n                session_id,\n                input=None,\n                context=None,\n                **kwargs):\n        \"\"\"\n        Send user input to assistant.\n\n        Send user input to an assistant and receive a response.\n        There is no rate limit for this operation.\n\n        :param str assistant_id: Unique identifier of the assistant. You can find the\n        assistant ID of an assistant on the **Assistants** tab of the Watson Assistant\n        tool. For information about creating assistants, see the\n        [documentation](https://console.bluemix.net/docs/services/assistant/assistant-add.html#assistant-add-task).\n        **Note:** Currently, the v2 API does not support creating assistants.\n        :param str session_id: Unique identifier of the session.\n        :param MessageInput input: An input object that includes the input text.\n        :param MessageContext context: State information for the conversation. The context\n        is stored by the assistant on a per-session basis. You can use this property to\n        set or modify context variables, which can also be accessed by dialog nodes.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse\n        \"\"\"\n\n        if assistant_id is None:\n            raise ValueError('assistant_id must be provided')\n        if session_id is None:\n            raise ValueError('session_id must be provided')\n        if input is not None:\n            input = self._convert_model(input, MessageInput)\n        if context is not None:\n            context = self._convert_model(context, MessageContext)\n\n        headers = {}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('conversation', 'V2', 'message')\n        headers.update(sdk_headers)\n\n        params = {'version': self.version}\n\n        data = {'input': input, 'context': context}\n\n        url = '/v2/assistants/{0}/sessions/{1}/message'.format(\n            *self._encode_path_vars(assistant_id, session_id))\n        response = self.request(\n            method='POST',\n            url=url,\n            headers=headers,\n            params=params,\n            json=data,\n            accept_json=True)\n        return response", "response": "Send user input to assistant.\n\n        Send user input to an assistant and receive a response.\n        There is no rate limit for this operation.\n\n        :param str assistant_id: Unique identifier of the assistant. You can find the\n        assistant ID of an assistant on the **Assistants** tab of the Watson Assistant\n        tool. For information about creating assistants, see the\n        [documentation](https://console.bluemix.net/docs/services/assistant/assistant-add.html#assistant-add-task).\n        **Note:** Currently, the v2 API does not support creating assistants.\n        :param str session_id: Unique identifier of the session.\n        :param MessageInput input: An input object that includes the input text.\n        :param MessageContext context: State information for the conversation. The context\n        is stored by the assistant on a per-session basis. You can use this property to\n        set or modify context variables, which can also be accessed by dialog nodes.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'group') and self.group is not None:\n            _dict['group'] = self.group\n        if hasattr(self, 'location') and self.location is not None:\n            _dict['location'] = self.location\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'level') and self.level is not None:\n            _dict['level'] = self.level\n        if hasattr(self, 'message') and self.message is not None:\n            _dict['message'] = self.message\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'name') and self.name is not None:\n            _dict['name'] = self.name\n        if hasattr(self, 'action_type') and self.action_type is not None:\n            _dict['type'] = self.action_type\n        if hasattr(self, 'parameters') and self.parameters is not None:\n            _dict['parameters'] = self.parameters\n        if hasattr(self,\n                   'result_variable') and self.result_variable is not None:\n            _dict['result_variable'] = self.result_variable\n        if hasattr(self, 'credentials') and self.credentials is not None:\n            _dict['credentials'] = self.credentials\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'input') and self.input is not None:\n            _dict['input'] = self.input._to_dict()\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'dialog_node') and self.dialog_node is not None:\n            _dict['dialog_node'] = self.dialog_node\n        if hasattr(self, 'title') and self.title is not None:\n            _dict['title'] = self.title\n        if hasattr(self, 'conditions') and self.conditions is not None:\n            _dict['conditions'] = self.conditions\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'label') and self.label is not None:\n            _dict['label'] = self.label\n        if hasattr(self, 'value') and self.value is not None:\n            _dict['value'] = self.value._to_dict()\n        if hasattr(self, 'output') and self.output is not None:\n            _dict['output'] = self.output\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'global' in _dict:\n            args['global_'] = MessageContextGlobal._from_dict(\n                _dict.get('global'))\n        if 'skills' in _dict:\n            args['skills'] = MessageContextSkills._from_dict(\n                _dict.get('skills'))\n        return cls(**args)", "response": "Initialize a MessageContext object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'global_') and self.global_ is not None:\n            _dict['global'] = self.global_._to_dict()\n        if hasattr(self, 'skills') and self.skills is not None:\n            _dict['skills'] = self.skills._to_dict()\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'system') and self.system is not None:\n            _dict['system'] = self.system._to_dict()\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'timezone') and self.timezone is not None:\n            _dict['timezone'] = self.timezone\n        if hasattr(self, 'user_id') and self.user_id is not None:\n            _dict['user_id'] = self.user_id\n        if hasattr(self, 'turn_count') and self.turn_count is not None:\n            _dict['turn_count'] = self.turn_count\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ninitializing a MessageInput object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a MessageInput object from a json dictionary.\"\"\"\n        args = {}\n        if 'message_type' in _dict:\n            args['message_type'] = _dict.get('message_type')\n        if 'text' in _dict:\n            args['text'] = _dict.get('text')\n        if 'options' in _dict:\n            args['options'] = MessageInputOptions._from_dict(\n                _dict.get('options'))\n        if 'intents' in _dict:\n            args['intents'] = [\n                RuntimeIntent._from_dict(x) for x in (_dict.get('intents'))\n            ]\n        if 'entities' in _dict:\n            args['entities'] = [\n                RuntimeEntity._from_dict(x) for x in (_dict.get('entities'))\n            ]\n        if 'suggestion_id' in _dict:\n            args['suggestion_id'] = _dict.get('suggestion_id')\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'message_type') and self.message_type is not None:\n            _dict['message_type'] = self.message_type\n        if hasattr(self, 'text') and self.text is not None:\n            _dict['text'] = self.text\n        if hasattr(self, 'options') and self.options is not None:\n            _dict['options'] = self.options._to_dict()\n        if hasattr(self, 'intents') and self.intents is not None:\n            _dict['intents'] = [x._to_dict() for x in self.intents]\n        if hasattr(self, 'entities') and self.entities is not None:\n            _dict['entities'] = [x._to_dict() for x in self.entities]\n        if hasattr(self, 'suggestion_id') and self.suggestion_id is not None:\n            _dict['suggestion_id'] = self.suggestion_id\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'debug') and self.debug is not None:\n            _dict['debug'] = self.debug\n        if hasattr(self, 'restart') and self.restart is not None:\n            _dict['restart'] = self.restart\n        if hasattr(self,\n                   'alternate_intents') and self.alternate_intents is not None:\n            _dict['alternate_intents'] = self.alternate_intents\n        if hasattr(self, 'return_context') and self.return_context is not None:\n            _dict['return_context'] = self.return_context\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ninitializing a MessageOutput object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a MessageOutput object from a json dictionary.\"\"\"\n        args = {}\n        if 'generic' in _dict:\n            args['generic'] = [\n                DialogRuntimeResponseGeneric._from_dict(x)\n                for x in (_dict.get('generic'))\n            ]\n        if 'intents' in _dict:\n            args['intents'] = [\n                RuntimeIntent._from_dict(x) for x in (_dict.get('intents'))\n            ]\n        if 'entities' in _dict:\n            args['entities'] = [\n                RuntimeEntity._from_dict(x) for x in (_dict.get('entities'))\n            ]\n        if 'actions' in _dict:\n            args['actions'] = [\n                DialogNodeAction._from_dict(x) for x in (_dict.get('actions'))\n            ]\n        if 'debug' in _dict:\n            args['debug'] = MessageOutputDebug._from_dict(_dict.get('debug'))\n        if 'user_defined' in _dict:\n            args['user_defined'] = _dict.get('user_defined')\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'generic') and self.generic is not None:\n            _dict['generic'] = [x._to_dict() for x in self.generic]\n        if hasattr(self, 'intents') and self.intents is not None:\n            _dict['intents'] = [x._to_dict() for x in self.intents]\n        if hasattr(self, 'entities') and self.entities is not None:\n            _dict['entities'] = [x._to_dict() for x in self.entities]\n        if hasattr(self, 'actions') and self.actions is not None:\n            _dict['actions'] = [x._to_dict() for x in self.actions]\n        if hasattr(self, 'debug') and self.debug is not None:\n            _dict['debug'] = self.debug._to_dict()\n        if hasattr(self, 'user_defined') and self.user_defined is not None:\n            _dict['user_defined'] = self.user_defined\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ninitialize a MessageOutputDebug object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a MessageOutputDebug object from a json dictionary.\"\"\"\n        args = {}\n        if 'nodes_visited' in _dict:\n            args['nodes_visited'] = [\n                DialogNodesVisited._from_dict(x)\n                for x in (_dict.get('nodes_visited'))\n            ]\n        if 'log_messages' in _dict:\n            args['log_messages'] = [\n                DialogLogMessage._from_dict(x)\n                for x in (_dict.get('log_messages'))\n            ]\n        if 'branch_exited' in _dict:\n            args['branch_exited'] = _dict.get('branch_exited')\n        if 'branch_exited_reason' in _dict:\n            args['branch_exited_reason'] = _dict.get('branch_exited_reason')\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'nodes_visited') and self.nodes_visited is not None:\n            _dict['nodes_visited'] = [x._to_dict() for x in self.nodes_visited]\n        if hasattr(self, 'log_messages') and self.log_messages is not None:\n            _dict['log_messages'] = [x._to_dict() for x in self.log_messages]\n        if hasattr(self, 'branch_exited') and self.branch_exited is not None:\n            _dict['branch_exited'] = self.branch_exited\n        if hasattr(self, 'branch_exited_reason'\n                  ) and self.branch_exited_reason is not None:\n            _dict['branch_exited_reason'] = self.branch_exited_reason\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'output' in _dict:\n            args['output'] = MessageOutput._from_dict(_dict.get('output'))\n        else:\n            raise ValueError(\n                'Required property \\'output\\' not present in MessageResponse JSON'\n            )\n        if 'context' in _dict:\n            args['context'] = MessageContext._from_dict(_dict.get('context'))\n        return cls(**args)", "response": "Initialize a MessageResponse object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'output') and self.output is not None:\n            _dict['output'] = self.output._to_dict()\n        if hasattr(self, 'context') and self.context is not None:\n            _dict['context'] = self.context._to_dict()\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'entity' in _dict:\n            args['entity'] = _dict.get('entity')\n        else:\n            raise ValueError(\n                'Required property \\'entity\\' not present in RuntimeEntity JSON'\n            )\n        if 'location' in _dict:\n            args['location'] = _dict.get('location')\n        else:\n            raise ValueError(\n                'Required property \\'location\\' not present in RuntimeEntity JSON'\n            )\n        if 'value' in _dict:\n            args['value'] = _dict.get('value')\n        else:\n            raise ValueError(\n                'Required property \\'value\\' not present in RuntimeEntity JSON')\n        if 'confidence' in _dict:\n            args['confidence'] = _dict.get('confidence')\n        if 'metadata' in _dict:\n            args['metadata'] = _dict.get('metadata')\n        if 'groups' in _dict:\n            args['groups'] = [\n                CaptureGroup._from_dict(x) for x in (_dict.get('groups'))\n            ]\n        return cls(**args)", "response": "Initialize a RuntimeEntity object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'entity') and self.entity is not None:\n            _dict['entity'] = self.entity\n        if hasattr(self, 'location') and self.location is not None:\n            _dict['location'] = self.location\n        if hasattr(self, 'value') and self.value is not None:\n            _dict['value'] = self.value\n        if hasattr(self, 'confidence') and self.confidence is not None:\n            _dict['confidence'] = self.confidence\n        if hasattr(self, 'metadata') and self.metadata is not None:\n            _dict['metadata'] = self.metadata\n        if hasattr(self, 'groups') and self.groups is not None:\n            _dict['groups'] = [x._to_dict() for x in self.groups]\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'intent') and self.intent is not None:\n            _dict['intent'] = self.intent\n        if hasattr(self, 'confidence') and self.confidence is not None:\n            _dict['confidence'] = self.confidence\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'session_id') and self.session_id is not None:\n            _dict['session_id'] = self.session_id\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef analyze(self,\n                features,\n                text=None,\n                html=None,\n                url=None,\n                clean=None,\n                xpath=None,\n                fallback_to_raw=None,\n                return_analyzed_text=None,\n                language=None,\n                limit_text_characters=None,\n                **kwargs):\n        \"\"\"\n        Analyze text.\n\n        Analyzes text, HTML, or a public webpage for the following features:\n        - Categories\n        - Concepts\n        - Emotion\n        - Entities\n        - Keywords\n        - Metadata\n        - Relations\n        - Semantic roles\n        - Sentiment\n        - Syntax (Experimental).\n\n        :param Features features: Specific features to analyze the document for.\n        :param str text: The plain text to analyze. One of the `text`, `html`, or `url`\n        parameters is required.\n        :param str html: The HTML file to analyze. One of the `text`, `html`, or `url`\n        parameters is required.\n        :param str url: The webpage to analyze. One of the `text`, `html`, or `url`\n        parameters is required.\n        :param bool clean: Set this to `false` to disable webpage cleaning. To learn more\n        about webpage cleaning, see the [Analyzing\n        webpages](https://cloud.ibm.com/docs/services/natural-language-understanding/analyzing-webpages.html)\n        documentation.\n        :param str xpath: An [XPath\n        query](https://cloud.ibm.com/docs/services/natural-language-understanding/analyzing-webpages.html#xpath)\n        to perform on `html` or `url` input. Results of the query will be appended to the\n        cleaned webpage text before it is analyzed. To analyze only the results of the\n        XPath query, set the `clean` parameter to `false`.\n        :param bool fallback_to_raw: Whether to use raw HTML content if text cleaning\n        fails.\n        :param bool return_analyzed_text: Whether or not to return the analyzed text.\n        :param str language: ISO 639-1 code that specifies the language of your text. This\n        overrides automatic language detection. Language support differs depending on the\n        features you include in your analysis. See [Language\n        support](https://www.bluemix.net/docs/services/natural-language-understanding/language-support.html)\n        for more information.\n        :param int limit_text_characters: Sets the maximum number of characters that are\n        processed by the service.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse\n        \"\"\"\n\n        if features is None:\n            raise ValueError('features must be provided')\n        features = self._convert_model(features, Features)\n\n        headers = {}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('natural-language-understanding', 'V1',\n                                      'analyze')\n        headers.update(sdk_headers)\n\n        params = {'version': self.version}\n\n        data = {\n            'features': features,\n            'text': text,\n            'html': html,\n            'url': url,\n            'clean': clean,\n            'xpath': xpath,\n            'fallback_to_raw': fallback_to_raw,\n            'return_analyzed_text': return_analyzed_text,\n            'language': language,\n            'limit_text_characters': limit_text_characters\n        }\n\n        url = '/v1/analyze'\n        response = self.request(\n            method='POST',\n            url=url,\n            headers=headers,\n            params=params,\n            json=data,\n            accept_json=True)\n        return response", "response": "Analyze the text of a specific language."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef list_models(self, **kwargs):\n\n        headers = {}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('natural-language-understanding', 'V1',\n                                      'list_models')\n        headers.update(sdk_headers)\n\n        params = {'version': self.version}\n\n        url = '/v1/models'\n        response = self.request(\n            method='GET',\n            url=url,\n            headers=headers,\n            params=params,\n            accept_json=True)\n        return response", "response": "List the available models."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ninitialize a AnalysisResults object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a AnalysisResults object from a json dictionary.\"\"\"\n        args = {}\n        if 'language' in _dict:\n            args['language'] = _dict.get('language')\n        if 'analyzed_text' in _dict:\n            args['analyzed_text'] = _dict.get('analyzed_text')\n        if 'retrieved_url' in _dict:\n            args['retrieved_url'] = _dict.get('retrieved_url')\n        if 'usage' in _dict:\n            args['usage'] = AnalysisResultsUsage._from_dict(_dict.get('usage'))\n        if 'concepts' in _dict:\n            args['concepts'] = [\n                ConceptsResult._from_dict(x) for x in (_dict.get('concepts'))\n            ]\n        if 'entities' in _dict:\n            args['entities'] = [\n                EntitiesResult._from_dict(x) for x in (_dict.get('entities'))\n            ]\n        if 'keywords' in _dict:\n            args['keywords'] = [\n                KeywordsResult._from_dict(x) for x in (_dict.get('keywords'))\n            ]\n        if 'categories' in _dict:\n            args['categories'] = [\n                CategoriesResult._from_dict(x)\n                for x in (_dict.get('categories'))\n            ]\n        if 'emotion' in _dict:\n            args['emotion'] = EmotionResult._from_dict(_dict.get('emotion'))\n        if 'metadata' in _dict:\n            args['metadata'] = AnalysisResultsMetadata._from_dict(\n                _dict.get('metadata'))\n        if 'relations' in _dict:\n            args['relations'] = [\n                RelationsResult._from_dict(x) for x in (_dict.get('relations'))\n            ]\n        if 'semantic_roles' in _dict:\n            args['semantic_roles'] = [\n                SemanticRolesResult._from_dict(x)\n                for x in (_dict.get('semantic_roles'))\n            ]\n        if 'sentiment' in _dict:\n            args['sentiment'] = SentimentResult._from_dict(\n                _dict.get('sentiment'))\n        if 'syntax' in _dict:\n            args['syntax'] = SyntaxResult._from_dict(_dict.get('syntax'))\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'language') and self.language is not None:\n            _dict['language'] = self.language\n        if hasattr(self, 'analyzed_text') and self.analyzed_text is not None:\n            _dict['analyzed_text'] = self.analyzed_text\n        if hasattr(self, 'retrieved_url') and self.retrieved_url is not None:\n            _dict['retrieved_url'] = self.retrieved_url\n        if hasattr(self, 'usage') and self.usage is not None:\n            _dict['usage'] = self.usage._to_dict()\n        if hasattr(self, 'concepts') and self.concepts is not None:\n            _dict['concepts'] = [x._to_dict() for x in self.concepts]\n        if hasattr(self, 'entities') and self.entities is not None:\n            _dict['entities'] = [x._to_dict() for x in self.entities]\n        if hasattr(self, 'keywords') and self.keywords is not None:\n            _dict['keywords'] = [x._to_dict() for x in self.keywords]\n        if hasattr(self, 'categories') and self.categories is not None:\n            _dict['categories'] = [x._to_dict() for x in self.categories]\n        if hasattr(self, 'emotion') and self.emotion is not None:\n            _dict['emotion'] = self.emotion._to_dict()\n        if hasattr(self, 'metadata') and self.metadata is not None:\n            _dict['metadata'] = self.metadata._to_dict()\n        if hasattr(self, 'relations') and self.relations is not None:\n            _dict['relations'] = [x._to_dict() for x in self.relations]\n        if hasattr(self, 'semantic_roles') and self.semantic_roles is not None:\n            _dict['semantic_roles'] = [\n                x._to_dict() for x in self.semantic_roles\n            ]\n        if hasattr(self, 'sentiment') and self.sentiment is not None:\n            _dict['sentiment'] = self.sentiment._to_dict()\n        if hasattr(self, 'syntax') and self.syntax is not None:\n            _dict['syntax'] = self.syntax._to_dict()\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'authors' in _dict:\n            args['authors'] = [\n                Author._from_dict(x) for x in (_dict.get('authors'))\n            ]\n        if 'publication_date' in _dict:\n            args['publication_date'] = _dict.get('publication_date')\n        if 'title' in _dict:\n            args['title'] = _dict.get('title')\n        if 'image' in _dict:\n            args['image'] = _dict.get('image')\n        if 'feeds' in _dict:\n            args['feeds'] = [Feed._from_dict(x) for x in (_dict.get('feeds'))]\n        return cls(**args)", "response": "Initialize a AnalysisResultsMetadata object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'authors') and self.authors is not None:\n            _dict['authors'] = [x._to_dict() for x in self.authors]\n        if hasattr(self,\n                   'publication_date') and self.publication_date is not None:\n            _dict['publication_date'] = self.publication_date\n        if hasattr(self, 'title') and self.title is not None:\n            _dict['title'] = self.title\n        if hasattr(self, 'image') and self.image is not None:\n            _dict['image'] = self.image\n        if hasattr(self, 'feeds') and self.feeds is not None:\n            _dict['feeds'] = [x._to_dict() for x in self.feeds]\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'features') and self.features is not None:\n            _dict['features'] = self.features\n        if hasattr(self,\n                   'text_characters') and self.text_characters is not None:\n            _dict['text_characters'] = self.text_characters\n        if hasattr(self, 'text_units') and self.text_units is not None:\n            _dict['text_units'] = self.text_units\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'name') and self.name is not None:\n            _dict['name'] = self.name\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'limit') and self.limit is not None:\n            _dict['limit'] = self.limit\n        if hasattr(self, 'model') and self.model is not None:\n            _dict['model'] = self.model\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'label') and self.label is not None:\n            _dict['label'] = self.label\n        if hasattr(self, 'score') and self.score is not None:\n            _dict['score'] = self.score\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'limit') and self.limit is not None:\n            _dict['limit'] = self.limit\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'text') and self.text is not None:\n            _dict['text'] = self.text\n        if hasattr(self, 'relevance') and self.relevance is not None:\n            _dict['relevance'] = self.relevance\n        if hasattr(self,\n                   'dbpedia_resource') and self.dbpedia_resource is not None:\n            _dict['dbpedia_resource'] = self.dbpedia_resource\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'deleted') and self.deleted is not None:\n            _dict['deleted'] = self.deleted\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'name') and self.name is not None:\n            _dict['name'] = self.name\n        if hasattr(self,\n                   'dbpedia_resource') and self.dbpedia_resource is not None:\n            _dict['dbpedia_resource'] = self.dbpedia_resource\n        if hasattr(self, 'subtype') and self.subtype is not None:\n            _dict['subtype'] = self.subtype\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'emotion') and self.emotion is not None:\n            _dict['emotion'] = self.emotion._to_dict()\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'document') and self.document is not None:\n            _dict['document'] = self.document\n        if hasattr(self, 'targets') and self.targets is not None:\n            _dict['targets'] = self.targets\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'document' in _dict:\n            args['document'] = DocumentEmotionResults._from_dict(\n                _dict.get('document'))\n        if 'targets' in _dict:\n            args['targets'] = [\n                TargetedEmotionResults._from_dict(x)\n                for x in (_dict.get('targets'))\n            ]\n        return cls(**args)", "response": "Initialize a EmotionResult object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'anger') and self.anger is not None:\n            _dict['anger'] = self.anger\n        if hasattr(self, 'disgust') and self.disgust is not None:\n            _dict['disgust'] = self.disgust\n        if hasattr(self, 'fear') and self.fear is not None:\n            _dict['fear'] = self.fear\n        if hasattr(self, 'joy') and self.joy is not None:\n            _dict['joy'] = self.joy\n        if hasattr(self, 'sadness') and self.sadness is not None:\n            _dict['sadness'] = self.sadness\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'limit') and self.limit is not None:\n            _dict['limit'] = self.limit\n        if hasattr(self, 'mentions') and self.mentions is not None:\n            _dict['mentions'] = self.mentions\n        if hasattr(self, 'model') and self.model is not None:\n            _dict['model'] = self.model\n        if hasattr(self, 'sentiment') and self.sentiment is not None:\n            _dict['sentiment'] = self.sentiment\n        if hasattr(self, 'emotion') and self.emotion is not None:\n            _dict['emotion'] = self.emotion\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'type' in _dict:\n            args['type'] = _dict.get('type')\n        if 'text' in _dict:\n            args['text'] = _dict.get('text')\n        if 'relevance' in _dict:\n            args['relevance'] = _dict.get('relevance')\n        if 'mentions' in _dict:\n            args['mentions'] = [\n                EntityMention._from_dict(x) for x in (_dict.get('mentions'))\n            ]\n        if 'count' in _dict:\n            args['count'] = _dict.get('count')\n        if 'emotion' in _dict:\n            args['emotion'] = EmotionScores._from_dict(_dict.get('emotion'))\n        if 'sentiment' in _dict:\n            args['sentiment'] = FeatureSentimentResults._from_dict(\n                _dict.get('sentiment'))\n        if 'disambiguation' in _dict:\n            args['disambiguation'] = DisambiguationResult._from_dict(\n                _dict.get('disambiguation'))\n        return cls(**args)", "response": "Initialize a EntitiesResult object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'type') and self.type is not None:\n            _dict['type'] = self.type\n        if hasattr(self, 'text') and self.text is not None:\n            _dict['text'] = self.text\n        if hasattr(self, 'relevance') and self.relevance is not None:\n            _dict['relevance'] = self.relevance\n        if hasattr(self, 'mentions') and self.mentions is not None:\n            _dict['mentions'] = [x._to_dict() for x in self.mentions]\n        if hasattr(self, 'count') and self.count is not None:\n            _dict['count'] = self.count\n        if hasattr(self, 'emotion') and self.emotion is not None:\n            _dict['emotion'] = self.emotion._to_dict()\n        if hasattr(self, 'sentiment') and self.sentiment is not None:\n            _dict['sentiment'] = self.sentiment._to_dict()\n        if hasattr(self, 'disambiguation') and self.disambiguation is not None:\n            _dict['disambiguation'] = self.disambiguation._to_dict()\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'score') and self.score is not None:\n            _dict['score'] = self.score\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ninitialize a Features object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a Features object from a json dictionary.\"\"\"\n        args = {}\n        if 'concepts' in _dict:\n            args['concepts'] = ConceptsOptions._from_dict(_dict.get('concepts'))\n        if 'emotion' in _dict:\n            args['emotion'] = EmotionOptions._from_dict(_dict.get('emotion'))\n        if 'entities' in _dict:\n            args['entities'] = EntitiesOptions._from_dict(_dict.get('entities'))\n        if 'keywords' in _dict:\n            args['keywords'] = KeywordsOptions._from_dict(_dict.get('keywords'))\n        if 'metadata' in _dict:\n            args['metadata'] = MetadataOptions._from_dict(_dict.get('metadata'))\n        if 'relations' in _dict:\n            args['relations'] = RelationsOptions._from_dict(\n                _dict.get('relations'))\n        if 'semantic_roles' in _dict:\n            args['semantic_roles'] = SemanticRolesOptions._from_dict(\n                _dict.get('semantic_roles'))\n        if 'sentiment' in _dict:\n            args['sentiment'] = SentimentOptions._from_dict(\n                _dict.get('sentiment'))\n        if 'categories' in _dict:\n            args['categories'] = CategoriesOptions._from_dict(\n                _dict.get('categories'))\n        if 'syntax' in _dict:\n            args['syntax'] = SyntaxOptions._from_dict(_dict.get('syntax'))\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'concepts') and self.concepts is not None:\n            _dict['concepts'] = self.concepts._to_dict()\n        if hasattr(self, 'emotion') and self.emotion is not None:\n            _dict['emotion'] = self.emotion._to_dict()\n        if hasattr(self, 'entities') and self.entities is not None:\n            _dict['entities'] = self.entities._to_dict()\n        if hasattr(self, 'keywords') and self.keywords is not None:\n            _dict['keywords'] = self.keywords._to_dict()\n        if hasattr(self, 'metadata') and self.metadata is not None:\n            _dict['metadata'] = self.metadata._to_dict()\n        if hasattr(self, 'relations') and self.relations is not None:\n            _dict['relations'] = self.relations._to_dict()\n        if hasattr(self, 'semantic_roles') and self.semantic_roles is not None:\n            _dict['semantic_roles'] = self.semantic_roles._to_dict()\n        if hasattr(self, 'sentiment') and self.sentiment is not None:\n            _dict['sentiment'] = self.sentiment._to_dict()\n        if hasattr(self, 'categories') and self.categories is not None:\n            _dict['categories'] = self.categories._to_dict()\n        if hasattr(self, 'syntax') and self.syntax is not None:\n            _dict['syntax'] = self.syntax._to_dict()\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'link') and self.link is not None:\n            _dict['link'] = self.link\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'count' in _dict:\n            args['count'] = _dict.get('count')\n        if 'relevance' in _dict:\n            args['relevance'] = _dict.get('relevance')\n        if 'text' in _dict:\n            args['text'] = _dict.get('text')\n        if 'emotion' in _dict:\n            args['emotion'] = EmotionScores._from_dict(_dict.get('emotion'))\n        if 'sentiment' in _dict:\n            args['sentiment'] = FeatureSentimentResults._from_dict(\n                _dict.get('sentiment'))\n        return cls(**args)", "response": "Initialize a KeywordsResult object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'count') and self.count is not None:\n            _dict['count'] = self.count\n        if hasattr(self, 'relevance') and self.relevance is not None:\n            _dict['relevance'] = self.relevance\n        if hasattr(self, 'text') and self.text is not None:\n            _dict['text'] = self.text\n        if hasattr(self, 'emotion') and self.emotion is not None:\n            _dict['emotion'] = self.emotion._to_dict()\n        if hasattr(self, 'sentiment') and self.sentiment is not None:\n            _dict['sentiment'] = self.sentiment._to_dict()\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'models' in _dict:\n            args['models'] = [\n                Model._from_dict(x) for x in (_dict.get('models'))\n            ]\n        return cls(**args)", "response": "Initialize a ListModelsResults object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'models') and self.models is not None:\n            _dict['models'] = [x._to_dict() for x in self.models]\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'status') and self.status is not None:\n            _dict['status'] = self.status\n        if hasattr(self, 'model_id') and self.model_id is not None:\n            _dict['model_id'] = self.model_id\n        if hasattr(self, 'language') and self.language is not None:\n            _dict['language'] = self.language\n        if hasattr(self, 'description') and self.description is not None:\n            _dict['description'] = self.description\n        if hasattr(self, 'workspace_id') and self.workspace_id is not None:\n            _dict['workspace_id'] = self.workspace_id\n        if hasattr(self, 'version') and self.version is not None:\n            _dict['version'] = self.version\n        if hasattr(\n                self,\n                'version_description') and self.version_description is not None:\n            _dict['version_description'] = self.version_description\n        if hasattr(self, 'created') and self.created is not None:\n            _dict['created'] = datetime_to_string(self.created)\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ninitialize a RelationArgument object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a RelationArgument object from a json dictionary.\"\"\"\n        args = {}\n        if 'entities' in _dict:\n            args['entities'] = [\n                RelationEntity._from_dict(x) for x in (_dict.get('entities'))\n            ]\n        if 'location' in _dict:\n            args['location'] = _dict.get('location')\n        if 'text' in _dict:\n            args['text'] = _dict.get('text')\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'entities') and self.entities is not None:\n            _dict['entities'] = [x._to_dict() for x in self.entities]\n        if hasattr(self, 'location') and self.location is not None:\n            _dict['location'] = self.location\n        if hasattr(self, 'text') and self.text is not None:\n            _dict['text'] = self.text\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'model') and self.model is not None:\n            _dict['model'] = self.model\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ninitialize a RelationsResult object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a RelationsResult object from a json dictionary.\"\"\"\n        args = {}\n        if 'score' in _dict:\n            args['score'] = _dict.get('score')\n        if 'sentence' in _dict:\n            args['sentence'] = _dict.get('sentence')\n        if 'type' in _dict:\n            args['type'] = _dict.get('type')\n        if 'arguments' in _dict:\n            args['arguments'] = [\n                RelationArgument._from_dict(x) for x in (_dict.get('arguments'))\n            ]\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'score') and self.score is not None:\n            _dict['score'] = self.score\n        if hasattr(self, 'sentence') and self.sentence is not None:\n            _dict['sentence'] = self.sentence\n        if hasattr(self, 'type') and self.type is not None:\n            _dict['type'] = self.type\n        if hasattr(self, 'arguments') and self.arguments is not None:\n            _dict['arguments'] = [x._to_dict() for x in self.arguments]\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'type') and self.type is not None:\n            _dict['type'] = self.type\n        if hasattr(self, 'text') and self.text is not None:\n            _dict['text'] = self.text\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'text') and self.text is not None:\n            _dict['text'] = self.text\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'limit') and self.limit is not None:\n            _dict['limit'] = self.limit\n        if hasattr(self, 'keywords') and self.keywords is not None:\n            _dict['keywords'] = self.keywords\n        if hasattr(self, 'entities') and self.entities is not None:\n            _dict['entities'] = self.entities\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ninitializes a SemanticRolesResult object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a SemanticRolesResult object from a json dictionary.\"\"\"\n        args = {}\n        if 'sentence' in _dict:\n            args['sentence'] = _dict.get('sentence')\n        if 'subject' in _dict:\n            args['subject'] = SemanticRolesResultSubject._from_dict(\n                _dict.get('subject'))\n        if 'action' in _dict:\n            args['action'] = SemanticRolesResultAction._from_dict(\n                _dict.get('action'))\n        if 'object' in _dict:\n            args['object'] = SemanticRolesResultObject._from_dict(\n                _dict.get('object'))\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'sentence') and self.sentence is not None:\n            _dict['sentence'] = self.sentence\n        if hasattr(self, 'subject') and self.subject is not None:\n            _dict['subject'] = self.subject._to_dict()\n        if hasattr(self, 'action') and self.action is not None:\n            _dict['action'] = self.action._to_dict()\n        if hasattr(self, 'object') and self.object is not None:\n            _dict['object'] = self.object._to_dict()\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'text') and self.text is not None:\n            _dict['text'] = self.text\n        if hasattr(self, 'normalized') and self.normalized is not None:\n            _dict['normalized'] = self.normalized\n        if hasattr(self, 'verb') and self.verb is not None:\n            _dict['verb'] = self.verb._to_dict()\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ninitialize a SemanticRolesResultObject object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a SemanticRolesResultObject object from a json dictionary.\"\"\"\n        args = {}\n        if 'text' in _dict:\n            args['text'] = _dict.get('text')\n        if 'keywords' in _dict:\n            args['keywords'] = [\n                SemanticRolesKeyword._from_dict(x)\n                for x in (_dict.get('keywords'))\n            ]\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'text') and self.text is not None:\n            _dict['text'] = self.text\n        if hasattr(self, 'keywords') and self.keywords is not None:\n            _dict['keywords'] = [x._to_dict() for x in self.keywords]\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'text' in _dict:\n            args['text'] = _dict.get('text')\n        if 'entities' in _dict:\n            args['entities'] = [\n                SemanticRolesEntity._from_dict(x)\n                for x in (_dict.get('entities'))\n            ]\n        if 'keywords' in _dict:\n            args['keywords'] = [\n                SemanticRolesKeyword._from_dict(x)\n                for x in (_dict.get('keywords'))\n            ]\n        return cls(**args)", "response": "Initialize a SemanticRolesResultSubject object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'text') and self.text is not None:\n            _dict['text'] = self.text\n        if hasattr(self, 'entities') and self.entities is not None:\n            _dict['entities'] = [x._to_dict() for x in self.entities]\n        if hasattr(self, 'keywords') and self.keywords is not None:\n            _dict['keywords'] = [x._to_dict() for x in self.keywords]\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'text') and self.text is not None:\n            _dict['text'] = self.text\n        if hasattr(self, 'tense') and self.tense is not None:\n            _dict['tense'] = self.tense\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'document' in _dict:\n            args['document'] = DocumentSentimentResults._from_dict(\n                _dict.get('document'))\n        if 'targets' in _dict:\n            args['targets'] = [\n                TargetedSentimentResults._from_dict(x)\n                for x in (_dict.get('targets'))\n            ]\n        return cls(**args)", "response": "Initialize a SentimentResult object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'tokens') and self.tokens is not None:\n            _dict['tokens'] = self.tokens._to_dict()\n        if hasattr(self, 'sentences') and self.sentences is not None:\n            _dict['sentences'] = self.sentences\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'lemma') and self.lemma is not None:\n            _dict['lemma'] = self.lemma\n        if hasattr(self, 'part_of_speech') and self.part_of_speech is not None:\n            _dict['part_of_speech'] = self.part_of_speech\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'tokens' in _dict:\n            args['tokens'] = [\n                TokenResult._from_dict(x) for x in (_dict.get('tokens'))\n            ]\n        if 'sentences' in _dict:\n            args['sentences'] = [\n                SentenceResult._from_dict(x) for x in (_dict.get('sentences'))\n            ]\n        return cls(**args)", "response": "Initialize a SyntaxResult object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'text') and self.text is not None:\n            _dict['text'] = self.text\n        if hasattr(self, 'emotion') and self.emotion is not None:\n            _dict['emotion'] = self.emotion._to_dict()\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'text') and self.text is not None:\n            _dict['text'] = self.text\n        if hasattr(self, 'score') and self.score is not None:\n            _dict['score'] = self.score\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'text') and self.text is not None:\n            _dict['text'] = self.text\n        if hasattr(self, 'part_of_speech') and self.part_of_speech is not None:\n            _dict['part_of_speech'] = self.part_of_speech\n        if hasattr(self, 'location') and self.location is not None:\n            _dict['location'] = self.location\n        if hasattr(self, 'lemma') and self.lemma is not None:\n            _dict['lemma'] = self.lemma\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef update_configuration(self,\n                             environment_id,\n                             configuration_id,\n                             name,\n                             description=None,\n                             conversions=None,\n                             enrichments=None,\n                             normalizations=None,\n                             source=None,\n                             **kwargs):\n        \"\"\"\n        Update a configuration.\n\n        Replaces an existing configuration.\n          * Completely replaces the original configuration.\n          * The **configuration_id**, **updated**, and **created** fields are accepted in\n        the request, but they are ignored, and an error is not generated. It is also\n        acceptable for users to submit an updated configuration with none of the three\n        properties.\n          * Documents are processed with a snapshot of the configuration as it was at the\n        time the document was submitted to be ingested. This means that already submitted\n        documents will not see any updates made to the configuration.\n\n        :param str environment_id: The ID of the environment.\n        :param str configuration_id: The ID of the configuration.\n        :param str name: The name of the configuration.\n        :param str description: The description of the configuration, if available.\n        :param Conversions conversions: Document conversion settings.\n        :param list[Enrichment] enrichments: An array of document enrichment settings for\n        the configuration.\n        :param list[NormalizationOperation] normalizations: Defines operations that can be\n        used to transform the final output JSON into a normalized form. Operations are\n        executed in the order that they appear in the array.\n        :param Source source: Object containing source parameters for the configuration.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse\n        \"\"\"\n\n        if environment_id is None:\n            raise ValueError('environment_id must be provided')\n        if configuration_id is None:\n            raise ValueError('configuration_id must be provided')\n        if name is None:\n            raise ValueError('name must be provided')\n        if conversions is not None:\n            conversions = self._convert_model(conversions, Conversions)\n        if enrichments is not None:\n            enrichments = [\n                self._convert_model(x, Enrichment) for x in enrichments\n            ]\n        if normalizations is not None:\n            normalizations = [\n                self._convert_model(x, NormalizationOperation)\n                for x in normalizations\n            ]\n        if source is not None:\n            source = self._convert_model(source, Source)\n\n        headers = {}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('discovery', 'V1', 'update_configuration')\n        headers.update(sdk_headers)\n\n        params = {'version': self.version}\n\n        data = {\n            'name': name,\n            'description': description,\n            'conversions': conversions,\n            'enrichments': enrichments,\n            'normalizations': normalizations,\n            'source': source\n        }\n\n        url = '/v1/environments/{0}/configurations/{1}'.format(\n            *self._encode_path_vars(environment_id, configuration_id))\n        response = self.request(\n            method='PUT',\n            url=url,\n            headers=headers,\n            params=params,\n            json=data,\n            accept_json=True)\n        return response", "response": "Update a configuration.\n\n        Replaces an existing configuration.\n          * Completely replaces the original configuration.\n          * The **configuration_id**, **updated**, and **created** fields are accepted in\n        the request, but they are ignored, and an error is not generated. It is also\n        acceptable for users to submit an updated configuration with none of the three\n        properties.\n          * Documents are processed with a snapshot of the configuration as it was at the\n        time the document was submitted to be ingested. This means that already submitted\n        documents will not see any updates made to the configuration.\n\n        :param str environment_id: The ID of the environment.\n        :param str configuration_id: The ID of the configuration.\n        :param str name: The name of the configuration.\n        :param str description: The description of the configuration, if available.\n        :param Conversions conversions: Document conversion settings.\n        :param list[Enrichment] enrichments: An array of document enrichment settings for\n        the configuration.\n        :param list[NormalizationOperation] normalizations: Defines operations that can be\n        used to transform the final output JSON into a normalized form. Operations are\n        executed in the order that they appear in the array.\n        :param Source source: Object containing source parameters for the configuration.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef create_expansions(self, environment_id, collection_id, expansions,\n                          **kwargs):\n        \"\"\"\n        Create or update expansion list.\n\n        Create or replace the Expansion list for this collection. The maximum number of\n        expanded terms per collection is `500`.\n        The current expansion list is replaced with the uploaded content.\n\n        :param str environment_id: The ID of the environment.\n        :param str collection_id: The ID of the collection.\n        :param list[Expansion] expansions: An array of query expansion definitions.\n         Each object in the **expansions** array represents a term or set of terms that\n        will be expanded into other terms. Each expansion object can be configured as\n        bidirectional or unidirectional. Bidirectional means that all terms are expanded\n        to all other terms in the object. Unidirectional means that a set list of terms\n        can be expanded into a second list of terms.\n         To create a bi-directional expansion specify an **expanded_terms** array. When\n        found in a query, all items in the **expanded_terms** array are then expanded to\n        the other items in the same array.\n         To create a uni-directional expansion, specify both an array of **input_terms**\n        and an array of **expanded_terms**. When items in the **input_terms** array are\n        present in a query, they are expanded using the items listed in the\n        **expanded_terms** array.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse\n        \"\"\"\n\n        if environment_id is None:\n            raise ValueError('environment_id must be provided')\n        if collection_id is None:\n            raise ValueError('collection_id must be provided')\n        if expansions is None:\n            raise ValueError('expansions must be provided')\n        expansions = [self._convert_model(x, Expansion) for x in expansions]\n\n        headers = {}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('discovery', 'V1', 'create_expansions')\n        headers.update(sdk_headers)\n\n        params = {'version': self.version}\n\n        data = {'expansions': expansions}\n\n        url = '/v1/environments/{0}/collections/{1}/expansions'.format(\n            *self._encode_path_vars(environment_id, collection_id))\n        response = self.request(\n            method='POST',\n            url=url,\n            headers=headers,\n            params=params,\n            json=data,\n            accept_json=True)\n        return response", "response": "Create or update the current expansion list for a specific environment and collection."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef create_stopword_list(self,\n                             environment_id,\n                             collection_id,\n                             stopword_file,\n                             stopword_filename=None,\n                             **kwargs):\n        \"\"\"\n        Create stopword list.\n\n        Upload a custom stopword list to use with the specified collection.\n\n        :param str environment_id: The ID of the environment.\n        :param str collection_id: The ID of the collection.\n        :param file stopword_file: The content of the stopword list to ingest.\n        :param str stopword_filename: The filename for stopword_file.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse\n        \"\"\"\n\n        if environment_id is None:\n            raise ValueError('environment_id must be provided')\n        if collection_id is None:\n            raise ValueError('collection_id must be provided')\n        if stopword_file is None:\n            raise ValueError('stopword_file must be provided')\n\n        headers = {}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('discovery', 'V1', 'create_stopword_list')\n        headers.update(sdk_headers)\n\n        params = {'version': self.version}\n\n        form_data = {}\n        if not stopword_filename and hasattr(stopword_file, 'name'):\n            stopword_filename = basename(stopword_file.name)\n        if not stopword_filename:\n            raise ValueError('stopword_filename must be provided')\n        form_data['stopword_file'] = (stopword_filename, stopword_file,\n                                      'application/octet-stream')\n\n        url = '/v1/environments/{0}/collections/{1}/word_lists/stopwords'.format(\n            *self._encode_path_vars(environment_id, collection_id))\n        response = self.request(\n            method='POST',\n            url=url,\n            headers=headers,\n            params=params,\n            files=form_data,\n            accept_json=True)\n        return response", "response": "Create stopword list.\n\n        Upload a custom stopword list to use with the specified collection.\n\n        :param str environment_id: The ID of the environment.\n        :param str collection_id: The ID of the collection.\n        :param file stopword_file: The content of the stopword list to ingest.\n        :param str stopword_filename: The filename for stopword_file.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef federated_query(self,\n                        environment_id,\n                        filter=None,\n                        query=None,\n                        natural_language_query=None,\n                        passages=None,\n                        aggregation=None,\n                        count=None,\n                        return_fields=None,\n                        offset=None,\n                        sort=None,\n                        highlight=None,\n                        passages_fields=None,\n                        passages_count=None,\n                        passages_characters=None,\n                        deduplicate=None,\n                        deduplicate_field=None,\n                        collection_ids=None,\n                        similar=None,\n                        similar_document_ids=None,\n                        similar_fields=None,\n                        bias=None,\n                        logging_opt_out=None,\n                        **kwargs):\n        \"\"\"\n        Long environment queries.\n\n        Complex queries might be too long for a standard method query. By using this\n        method, you can construct longer queries. However, these queries may take longer\n        to complete than the standard method. For details, see the [Discovery service\n        documentation](https://cloud.ibm.com/docs/services/discovery?topic=discovery-query-concepts#query-concepts).\n\n        :param str environment_id: The ID of the environment.\n        :param str filter: A cacheable query that excludes documents that don't mention\n        the query content. Filter searches are better for metadata-type searches and for\n        assessing the concepts in the data set.\n        :param str query: A query search returns all documents in your data set with full\n        enrichments and full text, but with the most relevant documents listed first. Use\n        a query search when you want to find the most relevant search results. You cannot\n        use **natural_language_query** and **query** at the same time.\n        :param str natural_language_query: A natural language query that returns relevant\n        documents by utilizing training data and natural language understanding. You\n        cannot use **natural_language_query** and **query** at the same time.\n        :param bool passages: A passages query that returns the most relevant passages\n        from the results.\n        :param str aggregation: An aggregation search that returns an exact answer by\n        combining query search with filters. Useful for applications to build lists,\n        tables, and time series. For a full list of possible aggregations, see the Query\n        reference.\n        :param int count: Number of results to return.\n        :param str return_fields: A comma-separated list of the portion of the document\n        hierarchy to return.\n        :param int offset: The number of query results to skip at the beginning. For\n        example, if the total number of results that are returned is 10 and the offset is\n        8, it returns the last two results.\n        :param str sort: A comma-separated list of fields in the document to sort on. You\n        can optionally specify a sort direction by prefixing the field with `-` for\n        descending or `+` for ascending. Ascending is the default sort direction if no\n        prefix is specified. This parameter cannot be used in the same query as the\n        **bias** parameter.\n        :param bool highlight: When true, a highlight field is returned for each result\n        which contains the fields which match the query with `<em></em>` tags around the\n        matching query terms.\n        :param str passages_fields: A comma-separated list of fields that passages are\n        drawn from. If this parameter not specified, then all top-level fields are\n        included.\n        :param int passages_count: The maximum number of passages to return. The search\n        returns fewer passages if the requested total is not found. The default is `10`.\n        The maximum is `100`.\n        :param int passages_characters: The approximate number of characters that any one\n        passage will have.\n        :param bool deduplicate: When `true`, and used with a Watson Discovery News\n        collection, duplicate results (based on the contents of the **title** field) are\n        removed. Duplicate comparison is limited to the current query only; **offset** is\n        not considered. This parameter is currently Beta functionality.\n        :param str deduplicate_field: When specified, duplicate results based on the field\n        specified are removed from the returned results. Duplicate comparison is limited\n        to the current query only, **offset** is not considered. This parameter is\n        currently Beta functionality.\n        :param str collection_ids: A comma-separated list of collection IDs to be queried\n        against. Required when querying multiple collections, invalid when performing a\n        single collection query.\n        :param bool similar: When `true`, results are returned based on their similarity\n        to the document IDs specified in the **similar.document_ids** parameter.\n        :param str similar_document_ids: A comma-separated list of document IDs to find\n        similar documents.\n        **Tip:** Include the **natural_language_query** parameter to expand the scope of\n        the document similarity search with the natural language query. Other query\n        parameters, such as **filter** and **query**, are subsequently applied and reduce\n        the scope.\n        :param str similar_fields: A comma-separated list of field names that are used as\n        a basis for comparison to identify similar documents. If not specified, the entire\n        document is used for comparison.\n        :param str bias: Field which the returned results will be biased against. The\n        specified field must be either a **date** or **number** format. When a **date**\n        type field is specified returned results are biased towards field values closer to\n        the current date. When a **number** type field is specified, returned results are\n        biased towards higher field values. This parameter cannot be used in the same\n        query as the **sort** parameter.\n        :param bool logging_opt_out: If `true`, queries are not stored in the Discovery\n        **Logs** endpoint.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse\n        \"\"\"\n\n        if environment_id is None:\n            raise ValueError('environment_id must be provided')\n\n        headers = {'X-Watson-Logging-Opt-Out': logging_opt_out}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('discovery', 'V1', 'federated_query')\n        headers.update(sdk_headers)\n\n        params = {'version': self.version}\n\n        data = {\n            'filter': filter,\n            'query': query,\n            'natural_language_query': natural_language_query,\n            'passages': passages,\n            'aggregation': aggregation,\n            'count': count,\n            'return': return_fields,\n            'offset': offset,\n            'sort': sort,\n            'highlight': highlight,\n            'passages.fields': passages_fields,\n            'passages.count': passages_count,\n            'passages.characters': passages_characters,\n            'deduplicate': deduplicate,\n            'deduplicate.field': deduplicate_field,\n            'collection_ids': collection_ids,\n            'similar': similar,\n            'similar.document_ids': similar_document_ids,\n            'similar.fields': similar_fields,\n            'bias': bias\n        }\n\n        url = '/v1/environments/{0}/query'.format(\n            *self._encode_path_vars(environment_id))\n        response = self.request(\n            method='POST',\n            url=url,\n            headers=headers,\n            params=params,\n            json=data,\n            accept_json=True)\n        return response", "response": "This method is used to construct a full cacheable query for the most relevant document in a specific environment."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef query_entities(self,\n                       environment_id,\n                       collection_id,\n                       feature=None,\n                       entity=None,\n                       context=None,\n                       count=None,\n                       evidence_count=None,\n                       **kwargs):\n        \"\"\"\n        Knowledge Graph entity query.\n\n        See the [Knowledge Graph\n        documentation](https://cloud.ibm.com/docs/services/discovery?topic=discovery-kg#kg)\n        for more details.\n\n        :param str environment_id: The ID of the environment.\n        :param str collection_id: The ID of the collection.\n        :param str feature: The entity query feature to perform. Supported features are\n        `disambiguate` and `similar_entities`.\n        :param QueryEntitiesEntity entity: A text string that appears within the entity\n        text field.\n        :param QueryEntitiesContext context: Entity text to provide context for the\n        queried entity and rank based on that association. For example, if you wanted to\n        query the city of London in England your query would look for `London` with the\n        context of `England`.\n        :param int count: The number of results to return. The default is `10`. The\n        maximum is `1000`.\n        :param int evidence_count: The number of evidence items to return for each result.\n        The default is `0`. The maximum number of evidence items per query is 10,000.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse\n        \"\"\"\n\n        if environment_id is None:\n            raise ValueError('environment_id must be provided')\n        if collection_id is None:\n            raise ValueError('collection_id must be provided')\n        if entity is not None:\n            entity = self._convert_model(entity, QueryEntitiesEntity)\n        if context is not None:\n            context = self._convert_model(context, QueryEntitiesContext)\n\n        headers = {}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('discovery', 'V1', 'query_entities')\n        headers.update(sdk_headers)\n\n        params = {'version': self.version}\n\n        data = {\n            'feature': feature,\n            'entity': entity,\n            'context': context,\n            'count': count,\n            'evidence_count': evidence_count\n        }\n\n        url = '/v1/environments/{0}/collections/{1}/query_entities'.format(\n            *self._encode_path_vars(environment_id, collection_id))\n        response = self.request(\n            method='POST',\n            url=url,\n            headers=headers,\n            params=params,\n            json=data,\n            accept_json=True)\n        return response", "response": "Query entities for a specific entity."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nquery the relationship of a specific entity in a specific collection.", "response": "def query_relations(self,\n                        environment_id,\n                        collection_id,\n                        entities=None,\n                        context=None,\n                        sort=None,\n                        filter=None,\n                        count=None,\n                        evidence_count=None,\n                        **kwargs):\n        \"\"\"\n        Knowledge Graph relationship query.\n\n        See the [Knowledge Graph\n        documentation](https://cloud.ibm.com/docs/services/discovery?topic=discovery-kg#kg)\n        for more details.\n\n        :param str environment_id: The ID of the environment.\n        :param str collection_id: The ID of the collection.\n        :param list[QueryRelationsEntity] entities: An array of entities to find\n        relationships for.\n        :param QueryEntitiesContext context: Entity text to provide context for the\n        queried entity and rank based on that association. For example, if you wanted to\n        query the city of London in England your query would look for `London` with the\n        context of `England`.\n        :param str sort: The sorting method for the relationships, can be `score` or\n        `frequency`. `frequency` is the number of unique times each entity is identified.\n        The default is `score`. This parameter cannot be used in the same query as the\n        **bias** parameter.\n        :param QueryRelationsFilter filter:\n        :param int count: The number of results to return. The default is `10`. The\n        maximum is `1000`.\n        :param int evidence_count: The number of evidence items to return for each result.\n        The default is `0`. The maximum number of evidence items per query is 10,000.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse\n        \"\"\"\n\n        if environment_id is None:\n            raise ValueError('environment_id must be provided')\n        if collection_id is None:\n            raise ValueError('collection_id must be provided')\n        if entities is not None:\n            entities = [\n                self._convert_model(x, QueryRelationsEntity) for x in entities\n            ]\n        if context is not None:\n            context = self._convert_model(context, QueryEntitiesContext)\n        if filter is not None:\n            filter = self._convert_model(filter, QueryRelationsFilter)\n\n        headers = {}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('discovery', 'V1', 'query_relations')\n        headers.update(sdk_headers)\n\n        params = {'version': self.version}\n\n        data = {\n            'entities': entities,\n            'context': context,\n            'sort': sort,\n            'filter': filter,\n            'count': count,\n            'evidence_count': evidence_count\n        }\n\n        url = '/v1/environments/{0}/collections/{1}/query_relations'.format(\n            *self._encode_path_vars(environment_id, collection_id))\n        response = self.request(\n            method='POST',\n            url=url,\n            headers=headers,\n            params=params,\n            json=data,\n            accept_json=True)\n        return response"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nadd training data to a collection.", "response": "def add_training_data(self,\n                          environment_id,\n                          collection_id,\n                          natural_language_query=None,\n                          filter=None,\n                          examples=None,\n                          **kwargs):\n        \"\"\"\n        Add query to training data.\n\n        Adds a query to the training data for this collection. The query can contain a\n        filter and natural language query.\n\n        :param str environment_id: The ID of the environment.\n        :param str collection_id: The ID of the collection.\n        :param str natural_language_query: The natural text query for the new training\n        query.\n        :param str filter: The filter used on the collection before the\n        **natural_language_query** is applied.\n        :param list[TrainingExample] examples: Array of training examples.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse\n        \"\"\"\n\n        if environment_id is None:\n            raise ValueError('environment_id must be provided')\n        if collection_id is None:\n            raise ValueError('collection_id must be provided')\n        if examples is not None:\n            examples = [\n                self._convert_model(x, TrainingExample) for x in examples\n            ]\n\n        headers = {}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('discovery', 'V1', 'add_training_data')\n        headers.update(sdk_headers)\n\n        params = {'version': self.version}\n\n        data = {\n            'natural_language_query': natural_language_query,\n            'filter': filter,\n            'examples': examples\n        }\n\n        url = '/v1/environments/{0}/collections/{1}/training_data'.format(\n            *self._encode_path_vars(environment_id, collection_id))\n        response = self.request(\n            method='POST',\n            url=url,\n            headers=headers,\n            params=params,\n            json=data,\n            accept_json=True)\n        return response"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef create_training_example(self,\n                                environment_id,\n                                collection_id,\n                                query_id,\n                                document_id=None,\n                                cross_reference=None,\n                                relevance=None,\n                                **kwargs):\n        \"\"\"\n        Add example to training data query.\n\n        Adds a example to this training data query.\n\n        :param str environment_id: The ID of the environment.\n        :param str collection_id: The ID of the collection.\n        :param str query_id: The ID of the query used for training.\n        :param str document_id: The document ID associated with this training example.\n        :param str cross_reference: The cross reference associated with this training\n        example.\n        :param int relevance: The relevance of the training example.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse\n        \"\"\"\n\n        if environment_id is None:\n            raise ValueError('environment_id must be provided')\n        if collection_id is None:\n            raise ValueError('collection_id must be provided')\n        if query_id is None:\n            raise ValueError('query_id must be provided')\n\n        headers = {}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('discovery', 'V1',\n                                      'create_training_example')\n        headers.update(sdk_headers)\n\n        params = {'version': self.version}\n\n        data = {\n            'document_id': document_id,\n            'cross_reference': cross_reference,\n            'relevance': relevance\n        }\n\n        url = '/v1/environments/{0}/collections/{1}/training_data/{2}/examples'.format(\n            *self._encode_path_vars(environment_id, collection_id, query_id))\n        response = self.request(\n            method='POST',\n            url=url,\n            headers=headers,\n            params=params,\n            json=data,\n            accept_json=True)\n        return response", "response": "Add example to training data query.\n\n        Adds a example to this training data query.\n\n        :param str environment_id: The ID of the environment.\n        :param str collection_id: The ID of the collection.\n        :param str query_id: The ID of the query used for training.\n        :param str document_id: The document ID associated with this training example.\n        :param str cross_reference: The cross reference associated with this training\n        example.\n        :param int relevance: The relevance of the training example.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef create_event(self, type, data, **kwargs):\n\n        if type is None:\n            raise ValueError('type must be provided')\n        if data is None:\n            raise ValueError('data must be provided')\n        data = self._convert_model(data, EventData)\n\n        headers = {}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('discovery', 'V1', 'create_event')\n        headers.update(sdk_headers)\n\n        params = {'version': self.version}\n\n        data = {'type': type, 'data': data}\n\n        url = '/v1/events'\n        response = self.request(\n            method='POST',\n            url=url,\n            headers=headers,\n            params=params,\n            json=data,\n            accept_json=True)\n        return response", "response": "Create event.\n\n        The **Events** API can be used to create log entries that are associated with\n        specific queries. For example, you can record which documents in the results set\n        were \\\"clicked\\\" by a user and when that click occured.\n\n        :param str type: The event type to be created.\n        :param EventData data: Query event data object.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nget the event rate for a given event.", "response": "def get_metrics_event_rate(self,\n                               start_time=None,\n                               end_time=None,\n                               result_type=None,\n                               **kwargs):\n        \"\"\"\n        Percentage of queries with an associated event.\n\n        The percentage of queries using the **natural_language_query** parameter that have\n        a corresponding \\\"click\\\" event over a specified time window.  This metric\n        requires having integrated event tracking in your application using the **Events**\n        API.\n\n        :param datetime start_time: Metric is computed from data recorded after this\n        timestamp; must be in `YYYY-MM-DDThh:mm:ssZ` format.\n        :param datetime end_time: Metric is computed from data recorded before this\n        timestamp; must be in `YYYY-MM-DDThh:mm:ssZ` format.\n        :param str result_type: The type of result to consider when calculating the\n        metric.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse\n        \"\"\"\n\n        headers = {}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('discovery', 'V1',\n                                      'get_metrics_event_rate')\n        headers.update(sdk_headers)\n\n        params = {\n            'version': self.version,\n            'start_time': start_time,\n            'end_time': end_time,\n            'result_type': result_type\n        }\n\n        url = '/v1/metrics/event_rate'\n        response = self.request(\n            method='GET',\n            url=url,\n            headers=headers,\n            params=params,\n            accept_json=True)\n        return response"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef query_log(self,\n                  filter=None,\n                  query=None,\n                  count=None,\n                  offset=None,\n                  sort=None,\n                  **kwargs):\n        \"\"\"\n        Search the query and event log.\n\n        Searches the query and event log to find query sessions that match the specified\n        criteria. Searching the **logs** endpoint uses the standard Discovery query syntax\n        for the parameters that are supported.\n\n        :param str filter: A cacheable query that excludes documents that don't mention\n        the query content. Filter searches are better for metadata-type searches and for\n        assessing the concepts in the data set.\n        :param str query: A query search returns all documents in your data set with full\n        enrichments and full text, but with the most relevant documents listed first. Use\n        a query search when you want to find the most relevant search results. You cannot\n        use **natural_language_query** and **query** at the same time.\n        :param int count: Number of results to return. The maximum for the **count** and\n        **offset** values together in any one query is **10000**.\n        :param int offset: The number of query results to skip at the beginning. For\n        example, if the total number of results that are returned is 10 and the offset is\n        8, it returns the last two results. The maximum for the **count** and **offset**\n        values together in any one query is **10000**.\n        :param list[str] sort: A comma-separated list of fields in the document to sort\n        on. You can optionally specify a sort direction by prefixing the field with `-`\n        for descending or `+` for ascending. Ascending is the default sort direction if no\n        prefix is specified.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse\n        \"\"\"\n\n        headers = {}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('discovery', 'V1', 'query_log')\n        headers.update(sdk_headers)\n\n        params = {\n            'version': self.version,\n            'filter': filter,\n            'query': query,\n            'count': count,\n            'offset': offset,\n            'sort': self._convert_list(sort)\n        }\n\n        url = '/v1/logs'\n        response = self.request(\n            method='GET',\n            url=url,\n            headers=headers,\n            params=params,\n            accept_json=True)\n        return response", "response": "Query the event log."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncreates credentials for a local source.", "response": "def create_credentials(self,\n                           environment_id,\n                           source_type=None,\n                           credential_details=None,\n                           **kwargs):\n        \"\"\"\n        Create credentials.\n\n        Creates a set of credentials to connect to a remote source. Created credentials\n        are used in a configuration to associate a collection with the remote source.\n        **Note:** All credentials are sent over an encrypted connection and encrypted at\n        rest.\n\n        :param str environment_id: The ID of the environment.\n        :param str source_type: The source that this credentials object connects to.\n        -  `box` indicates the credentials are used to connect an instance of Enterprise\n        Box.\n        -  `salesforce` indicates the credentials are used to connect to Salesforce.\n        -  `sharepoint` indicates the credentials are used to connect to Microsoft\n        SharePoint Online.\n        -  `web_crawl` indicates the credentials are used to perform a web crawl.\n        =  `cloud_object_storage` indicates the credentials are used to connect to an IBM\n        Cloud Object Store.\n        :param CredentialDetails credential_details: Object containing details of the\n        stored credentials.\n        Obtain credentials for your source from the administrator of the source.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse\n        \"\"\"\n\n        if environment_id is None:\n            raise ValueError('environment_id must be provided')\n        if credential_details is not None:\n            credential_details = self._convert_model(credential_details,\n                                                     CredentialDetails)\n\n        headers = {}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('discovery', 'V1', 'create_credentials')\n        headers.update(sdk_headers)\n\n        params = {'version': self.version}\n\n        data = {\n            'source_type': source_type,\n            'credential_details': credential_details\n        }\n\n        url = '/v1/environments/{0}/credentials'.format(\n            *self._encode_path_vars(environment_id))\n        response = self.request(\n            method='POST',\n            url=url,\n            headers=headers,\n            params=params,\n            json=data,\n            accept_json=True)\n        return response"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'key' in _dict:\n            args['key'] = _dict.get('key')\n        if 'matching_results' in _dict:\n            args['matching_results'] = _dict.get('matching_results')\n        if 'aggregations' in _dict:\n            args['aggregations'] = [\n                QueryAggregation._from_dict(x)\n                for x in (_dict.get('aggregations'))\n            ]\n        return cls(**args)", "response": "Initialize a AggregationResult object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'key') and self.key is not None:\n            _dict['key'] = self.key\n        if hasattr(self,\n                   'matching_results') and self.matching_results is not None:\n            _dict['matching_results'] = self.matching_results\n        if hasattr(self, 'aggregations') and self.aggregations is not None:\n            _dict['aggregations'] = [x._to_dict() for x in self.aggregations]\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'field') and self.field is not None:\n            _dict['field'] = self.field\n        if hasattr(self, 'value') and self.value is not None:\n            _dict['value'] = self.value\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ninitialize a Collection object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a Collection object from a json dictionary.\"\"\"\n        args = {}\n        if 'collection_id' in _dict:\n            args['collection_id'] = _dict.get('collection_id')\n        if 'name' in _dict:\n            args['name'] = _dict.get('name')\n        if 'description' in _dict:\n            args['description'] = _dict.get('description')\n        if 'created' in _dict:\n            args['created'] = string_to_datetime(_dict.get('created'))\n        if 'updated' in _dict:\n            args['updated'] = string_to_datetime(_dict.get('updated'))\n        if 'status' in _dict:\n            args['status'] = _dict.get('status')\n        if 'configuration_id' in _dict:\n            args['configuration_id'] = _dict.get('configuration_id')\n        if 'language' in _dict:\n            args['language'] = _dict.get('language')\n        if 'document_counts' in _dict:\n            args['document_counts'] = DocumentCounts._from_dict(\n                _dict.get('document_counts'))\n        if 'disk_usage' in _dict:\n            args['disk_usage'] = CollectionDiskUsage._from_dict(\n                _dict.get('disk_usage'))\n        if 'training_status' in _dict:\n            args['training_status'] = TrainingStatus._from_dict(\n                _dict.get('training_status'))\n        if 'source_crawl' in _dict:\n            args['source_crawl'] = SourceStatus._from_dict(\n                _dict.get('source_crawl'))\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'collection_id') and self.collection_id is not None:\n            _dict['collection_id'] = self.collection_id\n        if hasattr(self, 'name') and self.name is not None:\n            _dict['name'] = self.name\n        if hasattr(self, 'description') and self.description is not None:\n            _dict['description'] = self.description\n        if hasattr(self, 'created') and self.created is not None:\n            _dict['created'] = datetime_to_string(self.created)\n        if hasattr(self, 'updated') and self.updated is not None:\n            _dict['updated'] = datetime_to_string(self.updated)\n        if hasattr(self, 'status') and self.status is not None:\n            _dict['status'] = self.status\n        if hasattr(self,\n                   'configuration_id') and self.configuration_id is not None:\n            _dict['configuration_id'] = self.configuration_id\n        if hasattr(self, 'language') and self.language is not None:\n            _dict['language'] = self.language\n        if hasattr(self,\n                   'document_counts') and self.document_counts is not None:\n            _dict['document_counts'] = self.document_counts._to_dict()\n        if hasattr(self, 'disk_usage') and self.disk_usage is not None:\n            _dict['disk_usage'] = self.disk_usage._to_dict()\n        if hasattr(self,\n                   'training_status') and self.training_status is not None:\n            _dict['training_status'] = self.training_status._to_dict()\n        if hasattr(self, 'source_crawl') and self.source_crawl is not None:\n            _dict['source_crawl'] = self.source_crawl._to_dict()\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'used_bytes') and self.used_bytes is not None:\n            _dict['used_bytes'] = self.used_bytes\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'available') and self.available is not None:\n            _dict['available'] = self.available\n        if hasattr(self,\n                   'maximum_allowed') and self.maximum_allowed is not None:\n            _dict['maximum_allowed'] = self.maximum_allowed\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'configuration_id' in _dict:\n            args['configuration_id'] = _dict.get('configuration_id')\n        if 'name' in _dict:\n            args['name'] = _dict.get('name')\n        else:\n            raise ValueError(\n                'Required property \\'name\\' not present in Configuration JSON')\n        if 'created' in _dict:\n            args['created'] = string_to_datetime(_dict.get('created'))\n        if 'updated' in _dict:\n            args['updated'] = string_to_datetime(_dict.get('updated'))\n        if 'description' in _dict:\n            args['description'] = _dict.get('description')\n        if 'conversions' in _dict:\n            args['conversions'] = Conversions._from_dict(\n                _dict.get('conversions'))\n        if 'enrichments' in _dict:\n            args['enrichments'] = [\n                Enrichment._from_dict(x) for x in (_dict.get('enrichments'))\n            ]\n        if 'normalizations' in _dict:\n            args['normalizations'] = [\n                NormalizationOperation._from_dict(x)\n                for x in (_dict.get('normalizations'))\n            ]\n        if 'source' in _dict:\n            args['source'] = Source._from_dict(_dict.get('source'))\n        return cls(**args)", "response": "Initialize a Configuration object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self,\n                   'configuration_id') and self.configuration_id is not None:\n            _dict['configuration_id'] = self.configuration_id\n        if hasattr(self, 'name') and self.name is not None:\n            _dict['name'] = self.name\n        if hasattr(self, 'created') and self.created is not None:\n            _dict['created'] = datetime_to_string(self.created)\n        if hasattr(self, 'updated') and self.updated is not None:\n            _dict['updated'] = datetime_to_string(self.updated)\n        if hasattr(self, 'description') and self.description is not None:\n            _dict['description'] = self.description\n        if hasattr(self, 'conversions') and self.conversions is not None:\n            _dict['conversions'] = self.conversions._to_dict()\n        if hasattr(self, 'enrichments') and self.enrichments is not None:\n            _dict['enrichments'] = [x._to_dict() for x in self.enrichments]\n        if hasattr(self, 'normalizations') and self.normalizations is not None:\n            _dict['normalizations'] = [\n                x._to_dict() for x in self.normalizations\n            ]\n        if hasattr(self, 'source') and self.source is not None:\n            _dict['source'] = self.source._to_dict()\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ninitializing a Conversions object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a Conversions object from a json dictionary.\"\"\"\n        args = {}\n        if 'pdf' in _dict:\n            args['pdf'] = PdfSettings._from_dict(_dict.get('pdf'))\n        if 'word' in _dict:\n            args['word'] = WordSettings._from_dict(_dict.get('word'))\n        if 'html' in _dict:\n            args['html'] = HtmlSettings._from_dict(_dict.get('html'))\n        if 'segment' in _dict:\n            args['segment'] = SegmentSettings._from_dict(_dict.get('segment'))\n        if 'json_normalizations' in _dict:\n            args['json_normalizations'] = [\n                NormalizationOperation._from_dict(x)\n                for x in (_dict.get('json_normalizations'))\n            ]\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'pdf') and self.pdf is not None:\n            _dict['pdf'] = self.pdf._to_dict()\n        if hasattr(self, 'word') and self.word is not None:\n            _dict['word'] = self.word._to_dict()\n        if hasattr(self, 'html') and self.html is not None:\n            _dict['html'] = self.html._to_dict()\n        if hasattr(self, 'segment') and self.segment is not None:\n            _dict['segment'] = self.segment._to_dict()\n        if hasattr(\n                self,\n                'json_normalizations') and self.json_normalizations is not None:\n            _dict['json_normalizations'] = [\n                x._to_dict() for x in self.json_normalizations\n            ]\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'type') and self.type is not None:\n            _dict['type'] = self.type\n        if hasattr(self, 'data') and self.data is not None:\n            _dict['data'] = self.data._to_dict()\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ninitialize a CredentialDetails object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a CredentialDetails object from a json dictionary.\"\"\"\n        args = {}\n        if 'credential_type' in _dict:\n            args['credential_type'] = _dict.get('credential_type')\n        if 'client_id' in _dict:\n            args['client_id'] = _dict.get('client_id')\n        if 'enterprise_id' in _dict:\n            args['enterprise_id'] = _dict.get('enterprise_id')\n        if 'url' in _dict:\n            args['url'] = _dict.get('url')\n        if 'username' in _dict:\n            args['username'] = _dict.get('username')\n        if 'organization_url' in _dict:\n            args['organization_url'] = _dict.get('organization_url')\n        if 'site_collection.path' in _dict:\n            args['site_collection_path'] = _dict.get('site_collection.path')\n        if 'client_secret' in _dict:\n            args['client_secret'] = _dict.get('client_secret')\n        if 'public_key_id' in _dict:\n            args['public_key_id'] = _dict.get('public_key_id')\n        if 'private_key' in _dict:\n            args['private_key'] = _dict.get('private_key')\n        if 'passphrase' in _dict:\n            args['passphrase'] = _dict.get('passphrase')\n        if 'password' in _dict:\n            args['password'] = _dict.get('password')\n        if 'gateway_id' in _dict:\n            args['gateway_id'] = _dict.get('gateway_id')\n        if 'source_version' in _dict:\n            args['source_version'] = _dict.get('source_version')\n        if 'web_application_url' in _dict:\n            args['web_application_url'] = _dict.get('web_application_url')\n        if 'domain' in _dict:\n            args['domain'] = _dict.get('domain')\n        if 'endpoint' in _dict:\n            args['endpoint'] = _dict.get('endpoint')\n        if 'access_key_id' in _dict:\n            args['access_key_id'] = _dict.get('access_key_id')\n        if 'secret_access_key' in _dict:\n            args['secret_access_key'] = _dict.get('secret_access_key')\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self,\n                   'credential_type') and self.credential_type is not None:\n            _dict['credential_type'] = self.credential_type\n        if hasattr(self, 'client_id') and self.client_id is not None:\n            _dict['client_id'] = self.client_id\n        if hasattr(self, 'enterprise_id') and self.enterprise_id is not None:\n            _dict['enterprise_id'] = self.enterprise_id\n        if hasattr(self, 'url') and self.url is not None:\n            _dict['url'] = self.url\n        if hasattr(self, 'username') and self.username is not None:\n            _dict['username'] = self.username\n        if hasattr(self,\n                   'organization_url') and self.organization_url is not None:\n            _dict['organization_url'] = self.organization_url\n        if hasattr(self, 'site_collection_path'\n                  ) and self.site_collection_path is not None:\n            _dict['site_collection.path'] = self.site_collection_path\n        if hasattr(self, 'client_secret') and self.client_secret is not None:\n            _dict['client_secret'] = self.client_secret\n        if hasattr(self, 'public_key_id') and self.public_key_id is not None:\n            _dict['public_key_id'] = self.public_key_id\n        if hasattr(self, 'private_key') and self.private_key is not None:\n            _dict['private_key'] = self.private_key\n        if hasattr(self, 'passphrase') and self.passphrase is not None:\n            _dict['passphrase'] = self.passphrase\n        if hasattr(self, 'password') and self.password is not None:\n            _dict['password'] = self.password\n        if hasattr(self, 'gateway_id') and self.gateway_id is not None:\n            _dict['gateway_id'] = self.gateway_id\n        if hasattr(self, 'source_version') and self.source_version is not None:\n            _dict['source_version'] = self.source_version\n        if hasattr(\n                self,\n                'web_application_url') and self.web_application_url is not None:\n            _dict['web_application_url'] = self.web_application_url\n        if hasattr(self, 'domain') and self.domain is not None:\n            _dict['domain'] = self.domain\n        if hasattr(self, 'endpoint') and self.endpoint is not None:\n            _dict['endpoint'] = self.endpoint\n        if hasattr(self, 'access_key_id') and self.access_key_id is not None:\n            _dict['access_key_id'] = self.access_key_id\n        if hasattr(self,\n                   'secret_access_key') and self.secret_access_key is not None:\n            _dict['secret_access_key'] = self.secret_access_key\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'credential_id') and self.credential_id is not None:\n            _dict['credential_id'] = self.credential_id\n        if hasattr(self, 'source_type') and self.source_type is not None:\n            _dict['source_type'] = self.source_type\n        if hasattr(\n                self,\n                'credential_details') and self.credential_details is not None:\n            _dict['credential_details'] = self.credential_details._to_dict()\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ninitialize a CredentialsList object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a CredentialsList object from a json dictionary.\"\"\"\n        args = {}\n        if 'credentials' in _dict:\n            args['credentials'] = [\n                Credentials._from_dict(x) for x in (_dict.get('credentials'))\n            ]\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'credentials') and self.credentials is not None:\n            _dict['credentials'] = [x._to_dict() for x in self.credentials]\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'collection_id') and self.collection_id is not None:\n            _dict['collection_id'] = self.collection_id\n        if hasattr(self, 'status') and self.status is not None:\n            _dict['status'] = self.status\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self,\n                   'configuration_id') and self.configuration_id is not None:\n            _dict['configuration_id'] = self.configuration_id\n        if hasattr(self, 'status') and self.status is not None:\n            _dict['status'] = self.status\n        if hasattr(self, 'notices') and self.notices is not None:\n            _dict['notices'] = [x._to_dict() for x in self.notices]\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'credential_id') and self.credential_id is not None:\n            _dict['credential_id'] = self.credential_id\n        if hasattr(self, 'status') and self.status is not None:\n            _dict['status'] = self.status\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'document_id') and self.document_id is not None:\n            _dict['document_id'] = self.document_id\n        if hasattr(self, 'status') and self.status is not None:\n            _dict['status'] = self.status\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'environment_id') and self.environment_id is not None:\n            _dict['environment_id'] = self.environment_id\n        if hasattr(self, 'status') and self.status is not None:\n            _dict['status'] = self.status\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'used_bytes') and self.used_bytes is not None:\n            _dict['used_bytes'] = self.used_bytes\n        if hasattr(self, 'maximum_allowed_bytes'\n                  ) and self.maximum_allowed_bytes is not None:\n            _dict['maximum_allowed_bytes'] = self.maximum_allowed_bytes\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ninitialize a DocumentAccepted object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a DocumentAccepted object from a json dictionary.\"\"\"\n        args = {}\n        if 'document_id' in _dict:\n            args['document_id'] = _dict.get('document_id')\n        if 'status' in _dict:\n            args['status'] = _dict.get('status')\n        if 'notices' in _dict:\n            args['notices'] = [\n                Notice._from_dict(x) for x in (_dict.get('notices'))\n            ]\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'document_id') and self.document_id is not None:\n            _dict['document_id'] = self.document_id\n        if hasattr(self, 'status') and self.status is not None:\n            _dict['status'] = self.status\n        if hasattr(self, 'notices') and self.notices is not None:\n            _dict['notices'] = [x._to_dict() for x in self.notices]\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'available') and self.available is not None:\n            _dict['available'] = self.available\n        if hasattr(self, 'processing') and self.processing is not None:\n            _dict['processing'] = self.processing\n        if hasattr(self, 'failed') and self.failed is not None:\n            _dict['failed'] = self.failed\n        if hasattr(self, 'pending') and self.pending is not None:\n            _dict['pending'] = self.pending\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'step') and self.step is not None:\n            _dict['step'] = self.step\n        if hasattr(self, 'snapshot') and self.snapshot is not None:\n            _dict['snapshot'] = self.snapshot\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'document_id' in _dict:\n            args['document_id'] = _dict.get('document_id')\n        else:\n            raise ValueError(\n                'Required property \\'document_id\\' not present in DocumentStatus JSON'\n            )\n        if 'configuration_id' in _dict:\n            args['configuration_id'] = _dict.get('configuration_id')\n        if 'status' in _dict:\n            args['status'] = _dict.get('status')\n        else:\n            raise ValueError(\n                'Required property \\'status\\' not present in DocumentStatus JSON'\n            )\n        if 'status_description' in _dict:\n            args['status_description'] = _dict.get('status_description')\n        else:\n            raise ValueError(\n                'Required property \\'status_description\\' not present in DocumentStatus JSON'\n            )\n        if 'filename' in _dict:\n            args['filename'] = _dict.get('filename')\n        if 'file_type' in _dict:\n            args['file_type'] = _dict.get('file_type')\n        if 'sha1' in _dict:\n            args['sha1'] = _dict.get('sha1')\n        if 'notices' in _dict:\n            args['notices'] = [\n                Notice._from_dict(x) for x in (_dict.get('notices'))\n            ]\n        else:\n            raise ValueError(\n                'Required property \\'notices\\' not present in DocumentStatus JSON'\n            )\n        return cls(**args)", "response": "Initialize a DocumentStatus object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'document_id') and self.document_id is not None:\n            _dict['document_id'] = self.document_id\n        if hasattr(self,\n                   'configuration_id') and self.configuration_id is not None:\n            _dict['configuration_id'] = self.configuration_id\n        if hasattr(self, 'status') and self.status is not None:\n            _dict['status'] = self.status\n        if hasattr(\n                self,\n                'status_description') and self.status_description is not None:\n            _dict['status_description'] = self.status_description\n        if hasattr(self, 'filename') and self.filename is not None:\n            _dict['filename'] = self.filename\n        if hasattr(self, 'file_type') and self.file_type is not None:\n            _dict['file_type'] = self.file_type\n        if hasattr(self, 'sha1') and self.sha1 is not None:\n            _dict['sha1'] = self.sha1\n        if hasattr(self, 'notices') and self.notices is not None:\n            _dict['notices'] = [x._to_dict() for x in self.notices]\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'description') and self.description is not None:\n            _dict['description'] = self.description\n        if hasattr(self,\n                   'destination_field') and self.destination_field is not None:\n            _dict['destination_field'] = self.destination_field\n        if hasattr(self, 'source_field') and self.source_field is not None:\n            _dict['source_field'] = self.source_field\n        if hasattr(self, 'overwrite') and self.overwrite is not None:\n            _dict['overwrite'] = self.overwrite\n        if hasattr(self,\n                   'enrichment_name') and self.enrichment_name is not None:\n            _dict['enrichment'] = self.enrichment_name\n        if hasattr(self, 'ignore_downstream_errors'\n                  ) and self.ignore_downstream_errors is not None:\n            _dict['ignore_downstream_errors'] = self.ignore_downstream_errors\n        if hasattr(self, 'options') and self.options is not None:\n            _dict['options'] = self.options._to_dict()\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'features') and self.features is not None:\n            _dict['features'] = self.features._to_dict()\n        if hasattr(self, 'language') and self.language is not None:\n            _dict['language'] = self.language\n        if hasattr(self, 'model') and self.model is not None:\n            _dict['model'] = self.model\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'environment_id' in _dict:\n            args['environment_id'] = _dict.get('environment_id')\n        if 'name' in _dict:\n            args['name'] = _dict.get('name')\n        if 'description' in _dict:\n            args['description'] = _dict.get('description')\n        if 'created' in _dict:\n            args['created'] = string_to_datetime(_dict.get('created'))\n        if 'updated' in _dict:\n            args['updated'] = string_to_datetime(_dict.get('updated'))\n        if 'status' in _dict:\n            args['status'] = _dict.get('status')\n        if 'read_only' in _dict:\n            args['read_only'] = _dict.get('read_only')\n        if 'size' in _dict:\n            args['size'] = _dict.get('size')\n        if 'requested_size' in _dict:\n            args['requested_size'] = _dict.get('requested_size')\n        if 'index_capacity' in _dict:\n            args['index_capacity'] = IndexCapacity._from_dict(\n                _dict.get('index_capacity'))\n        if 'search_status' in _dict:\n            args['search_status'] = SearchStatus._from_dict(\n                _dict.get('search_status'))\n        return cls(**args)", "response": "Initialize a Environment object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a json dictionary representing this object.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'environment_id') and self.environment_id is not None:\n            _dict['environment_id'] = self.environment_id\n        if hasattr(self, 'name') and self.name is not None:\n            _dict['name'] = self.name\n        if hasattr(self, 'description') and self.description is not None:\n            _dict['description'] = self.description\n        if hasattr(self, 'created') and self.created is not None:\n            _dict['created'] = datetime_to_string(self.created)\n        if hasattr(self, 'updated') and self.updated is not None:\n            _dict['updated'] = datetime_to_string(self.updated)\n        if hasattr(self, 'status') and self.status is not None:\n            _dict['status'] = self.status\n        if hasattr(self, 'read_only') and self.read_only is not None:\n            _dict['read_only'] = self.read_only\n        if hasattr(self, 'size') and self.size is not None:\n            _dict['size'] = self.size\n        if hasattr(self, 'requested_size') and self.requested_size is not None:\n            _dict['requested_size'] = self.requested_size\n        if hasattr(self, 'index_capacity') and self.index_capacity is not None:\n            _dict['index_capacity'] = self.index_capacity._to_dict()\n        if hasattr(self, 'search_status') and self.search_status is not None:\n            _dict['search_status'] = self.search_status._to_dict()\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'indexed') and self.indexed is not None:\n            _dict['indexed'] = self.indexed\n        if hasattr(self,\n                   'maximum_allowed') and self.maximum_allowed is not None:\n            _dict['maximum_allowed'] = self.maximum_allowed\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'environment_id') and self.environment_id is not None:\n            _dict['environment_id'] = self.environment_id\n        if hasattr(self, 'session_token') and self.session_token is not None:\n            _dict['session_token'] = self.session_token\n        if hasattr(self,\n                   'client_timestamp') and self.client_timestamp is not None:\n            _dict['client_timestamp'] = datetime_to_string(\n                self.client_timestamp)\n        if hasattr(self, 'display_rank') and self.display_rank is not None:\n            _dict['display_rank'] = self.display_rank\n        if hasattr(self, 'collection_id') and self.collection_id is not None:\n            _dict['collection_id'] = self.collection_id\n        if hasattr(self, 'document_id') and self.document_id is not None:\n            _dict['document_id'] = self.document_id\n        if hasattr(self, 'query_id') and self.query_id is not None:\n            _dict['query_id'] = self.query_id\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'input_terms') and self.input_terms is not None:\n            _dict['input_terms'] = self.input_terms\n        if hasattr(self, 'expanded_terms') and self.expanded_terms is not None:\n            _dict['expanded_terms'] = self.expanded_terms\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ninitializing a Expansions object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a Expansions object from a json dictionary.\"\"\"\n        args = {}\n        if 'expansions' in _dict:\n            args['expansions'] = [\n                Expansion._from_dict(x) for x in (_dict.get('expansions'))\n            ]\n        else:\n            raise ValueError(\n                'Required property \\'expansions\\' not present in Expansions JSON'\n            )\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'expansions') and self.expansions is not None:\n            _dict['expansions'] = [x._to_dict() for x in self.expansions]\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'field_name') and self.field_name is not None:\n            _dict['field'] = self.field_name\n        if hasattr(self, 'field_type') and self.field_type is not None:\n            _dict['type'] = self.field_type\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'match') and self.match is not None:\n            _dict['match'] = self.match\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'level') and self.level is not None:\n            _dict['level'] = self.level\n        if hasattr(self, 'min_size') and self.min_size is not None:\n            _dict['min_size'] = self.min_size\n        if hasattr(self, 'max_size') and self.max_size is not None:\n            _dict['max_size'] = self.max_size\n        if hasattr(self, 'bold') and self.bold is not None:\n            _dict['bold'] = self.bold\n        if hasattr(self, 'italic') and self.italic is not None:\n            _dict['italic'] = self.italic\n        if hasattr(self, 'name') and self.name is not None:\n            _dict['name'] = self.name\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'gateway_id') and self.gateway_id is not None:\n            _dict['gateway_id'] = self.gateway_id\n        if hasattr(self, 'name') and self.name is not None:\n            _dict['name'] = self.name\n        if hasattr(self, 'status') and self.status is not None:\n            _dict['status'] = self.status\n        if hasattr(self, 'token') and self.token is not None:\n            _dict['token'] = self.token\n        if hasattr(self, 'token_id') and self.token_id is not None:\n            _dict['token_id'] = self.token_id\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'gateway_id') and self.gateway_id is not None:\n            _dict['gateway_id'] = self.gateway_id\n        if hasattr(self, 'status') and self.status is not None:\n            _dict['status'] = self.status\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ninitializing a GatewayList object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a GatewayList object from a json dictionary.\"\"\"\n        args = {}\n        if 'gateways' in _dict:\n            args['gateways'] = [\n                Gateway._from_dict(x) for x in (_dict.get('gateways'))\n            ]\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'gateways') and self.gateways is not None:\n            _dict['gateways'] = [x._to_dict() for x in self.gateways]\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'exclude_tags_completely'\n                  ) and self.exclude_tags_completely is not None:\n            _dict['exclude_tags_completely'] = self.exclude_tags_completely\n        if hasattr(self, 'exclude_tags_keep_content'\n                  ) and self.exclude_tags_keep_content is not None:\n            _dict['exclude_tags_keep_content'] = self.exclude_tags_keep_content\n        if hasattr(self, 'keep_content') and self.keep_content is not None:\n            _dict['keep_content'] = self.keep_content._to_dict()\n        if hasattr(self,\n                   'exclude_content') and self.exclude_content is not None:\n            _dict['exclude_content'] = self.exclude_content._to_dict()\n        if hasattr(\n                self,\n                'keep_tag_attributes') and self.keep_tag_attributes is not None:\n            _dict['keep_tag_attributes'] = self.keep_tag_attributes\n        if hasattr(self, 'exclude_tag_attributes'\n                  ) and self.exclude_tag_attributes is not None:\n            _dict['exclude_tag_attributes'] = self.exclude_tag_attributes\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ninitializes a IndexCapacity object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a IndexCapacity object from a json dictionary.\"\"\"\n        args = {}\n        if 'documents' in _dict:\n            args['documents'] = EnvironmentDocuments._from_dict(\n                _dict.get('documents'))\n        if 'disk_usage' in _dict:\n            args['disk_usage'] = DiskUsage._from_dict(_dict.get('disk_usage'))\n        if 'collections' in _dict:\n            args['collections'] = CollectionUsage._from_dict(\n                _dict.get('collections'))\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'documents') and self.documents is not None:\n            _dict['documents'] = self.documents._to_dict()\n        if hasattr(self, 'disk_usage') and self.disk_usage is not None:\n            _dict['disk_usage'] = self.disk_usage._to_dict()\n        if hasattr(self, 'collections') and self.collections is not None:\n            _dict['collections'] = self.collections._to_dict()\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ninitialize a ListCollectionFieldsResponse object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a ListCollectionFieldsResponse object from a json dictionary.\"\"\"\n        args = {}\n        if 'fields' in _dict:\n            args['fields'] = [\n                Field._from_dict(x) for x in (_dict.get('fields'))\n            ]\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'fields') and self.fields is not None:\n            _dict['fields'] = [x._to_dict() for x in self.fields]\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'collections' in _dict:\n            args['collections'] = [\n                Collection._from_dict(x) for x in (_dict.get('collections'))\n            ]\n        return cls(**args)", "response": "Initialize a ListCollectionsResponse object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'collections') and self.collections is not None:\n            _dict['collections'] = [x._to_dict() for x in self.collections]\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ninitializing a ListConfigurationsResponse object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a ListConfigurationsResponse object from a json dictionary.\"\"\"\n        args = {}\n        if 'configurations' in _dict:\n            args['configurations'] = [\n                Configuration._from_dict(x)\n                for x in (_dict.get('configurations'))\n            ]\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'configurations') and self.configurations is not None:\n            _dict['configurations'] = [\n                x._to_dict() for x in self.configurations\n            ]\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'environments' in _dict:\n            args['environments'] = [\n                Environment._from_dict(x) for x in (_dict.get('environments'))\n            ]\n        return cls(**args)", "response": "Initialize a ListEnvironmentsResponse object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'environments') and self.environments is not None:\n            _dict['environments'] = [x._to_dict() for x in self.environments]\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ninitializes a LogQueryResponse object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a LogQueryResponse object from a json dictionary.\"\"\"\n        args = {}\n        if 'matching_results' in _dict:\n            args['matching_results'] = _dict.get('matching_results')\n        if 'results' in _dict:\n            args['results'] = [\n                LogQueryResponseResult._from_dict(x)\n                for x in (_dict.get('results'))\n            ]\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self,\n                   'matching_results') and self.matching_results is not None:\n            _dict['matching_results'] = self.matching_results\n        if hasattr(self, 'results') and self.results is not None:\n            _dict['results'] = [x._to_dict() for x in self.results]\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'environment_id' in _dict:\n            args['environment_id'] = _dict.get('environment_id')\n        if 'customer_id' in _dict:\n            args['customer_id'] = _dict.get('customer_id')\n        if 'document_type' in _dict:\n            args['document_type'] = _dict.get('document_type')\n        if 'natural_language_query' in _dict:\n            args['natural_language_query'] = _dict.get('natural_language_query')\n        if 'document_results' in _dict:\n            args[\n                'document_results'] = LogQueryResponseResultDocuments._from_dict(\n                    _dict.get('document_results'))\n        if 'created_timestamp' in _dict:\n            args['created_timestamp'] = string_to_datetime(\n                _dict.get('created_timestamp'))\n        if 'client_timestamp' in _dict:\n            args['client_timestamp'] = string_to_datetime(\n                _dict.get('client_timestamp'))\n        if 'query_id' in _dict:\n            args['query_id'] = _dict.get('query_id')\n        if 'session_token' in _dict:\n            args['session_token'] = _dict.get('session_token')\n        if 'collection_id' in _dict:\n            args['collection_id'] = _dict.get('collection_id')\n        if 'display_rank' in _dict:\n            args['display_rank'] = _dict.get('display_rank')\n        if 'document_id' in _dict:\n            args['document_id'] = _dict.get('document_id')\n        if 'event_type' in _dict:\n            args['event_type'] = _dict.get('event_type')\n        if 'result_type' in _dict:\n            args['result_type'] = _dict.get('result_type')\n        return cls(**args)", "response": "Initialize a LogQueryResponseResult object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'environment_id') and self.environment_id is not None:\n            _dict['environment_id'] = self.environment_id\n        if hasattr(self, 'customer_id') and self.customer_id is not None:\n            _dict['customer_id'] = self.customer_id\n        if hasattr(self, 'document_type') and self.document_type is not None:\n            _dict['document_type'] = self.document_type\n        if hasattr(self, 'natural_language_query'\n                  ) and self.natural_language_query is not None:\n            _dict['natural_language_query'] = self.natural_language_query\n        if hasattr(self,\n                   'document_results') and self.document_results is not None:\n            _dict['document_results'] = self.document_results._to_dict()\n        if hasattr(self,\n                   'created_timestamp') and self.created_timestamp is not None:\n            _dict['created_timestamp'] = datetime_to_string(\n                self.created_timestamp)\n        if hasattr(self,\n                   'client_timestamp') and self.client_timestamp is not None:\n            _dict['client_timestamp'] = datetime_to_string(\n                self.client_timestamp)\n        if hasattr(self, 'query_id') and self.query_id is not None:\n            _dict['query_id'] = self.query_id\n        if hasattr(self, 'session_token') and self.session_token is not None:\n            _dict['session_token'] = self.session_token\n        if hasattr(self, 'collection_id') and self.collection_id is not None:\n            _dict['collection_id'] = self.collection_id\n        if hasattr(self, 'display_rank') and self.display_rank is not None:\n            _dict['display_rank'] = self.display_rank\n        if hasattr(self, 'document_id') and self.document_id is not None:\n            _dict['document_id'] = self.document_id\n        if hasattr(self, 'event_type') and self.event_type is not None:\n            _dict['event_type'] = self.event_type\n        if hasattr(self, 'result_type') and self.result_type is not None:\n            _dict['result_type'] = self.result_type\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'results' in _dict:\n            args['results'] = [\n                LogQueryResponseResultDocumentsResult._from_dict(x)\n                for x in (_dict.get('results'))\n            ]\n        if 'count' in _dict:\n            args['count'] = _dict.get('count')\n        return cls(**args)", "response": "Initialize a LogQueryResponseResultDocuments object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'results') and self.results is not None:\n            _dict['results'] = [x._to_dict() for x in self.results]\n        if hasattr(self, 'count') and self.count is not None:\n            _dict['count'] = self.count\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'position') and self.position is not None:\n            _dict['position'] = self.position\n        if hasattr(self, 'document_id') and self.document_id is not None:\n            _dict['document_id'] = self.document_id\n        if hasattr(self, 'score') and self.score is not None:\n            _dict['score'] = self.score\n        if hasattr(self, 'confidence') and self.confidence is not None:\n            _dict['confidence'] = self.confidence\n        if hasattr(self, 'collection_id') and self.collection_id is not None:\n            _dict['collection_id'] = self.collection_id\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'interval' in _dict:\n            args['interval'] = _dict.get('interval')\n        if 'event_type' in _dict:\n            args['event_type'] = _dict.get('event_type')\n        if 'results' in _dict:\n            args['results'] = [\n                MetricAggregationResult._from_dict(x)\n                for x in (_dict.get('results'))\n            ]\n        return cls(**args)", "response": "Initialize a MetricAggregation object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'interval') and self.interval is not None:\n            _dict['interval'] = self.interval\n        if hasattr(self, 'event_type') and self.event_type is not None:\n            _dict['event_type'] = self.event_type\n        if hasattr(self, 'results') and self.results is not None:\n            _dict['results'] = [x._to_dict() for x in self.results]\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'key_as_string') and self.key_as_string is not None:\n            _dict['key_as_string'] = datetime_to_string(self.key_as_string)\n        if hasattr(self, 'key') and self.key is not None:\n            _dict['key'] = self.key\n        if hasattr(self,\n                   'matching_results') and self.matching_results is not None:\n            _dict['matching_results'] = self.matching_results\n        if hasattr(self, 'event_rate') and self.event_rate is not None:\n            _dict['event_rate'] = self.event_rate\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ninitialize a MetricResponse object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a MetricResponse object from a json dictionary.\"\"\"\n        args = {}\n        if 'aggregations' in _dict:\n            args['aggregations'] = [\n                MetricAggregation._from_dict(x)\n                for x in (_dict.get('aggregations'))\n            ]\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'aggregations') and self.aggregations is not None:\n            _dict['aggregations'] = [x._to_dict() for x in self.aggregations]\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ninitializes a MetricTokenAggregation object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a MetricTokenAggregation object from a json dictionary.\"\"\"\n        args = {}\n        if 'event_type' in _dict:\n            args['event_type'] = _dict.get('event_type')\n        if 'results' in _dict:\n            args['results'] = [\n                MetricTokenAggregationResult._from_dict(x)\n                for x in (_dict.get('results'))\n            ]\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'key') and self.key is not None:\n            _dict['key'] = self.key\n        if hasattr(self,\n                   'matching_results') and self.matching_results is not None:\n            _dict['matching_results'] = self.matching_results\n        if hasattr(self, 'event_rate') and self.event_rate is not None:\n            _dict['event_rate'] = self.event_rate\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ninitializes a MetricTokenResponse object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a MetricTokenResponse object from a json dictionary.\"\"\"\n        args = {}\n        if 'aggregations' in _dict:\n            args['aggregations'] = [\n                MetricTokenAggregation._from_dict(x)\n                for x in (_dict.get('aggregations'))\n            ]\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'path') and self.path is not None:\n            _dict['path'] = self.path\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ninitialize a NluEnrichmentFeatures object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a NluEnrichmentFeatures object from a json dictionary.\"\"\"\n        args = {}\n        if 'keywords' in _dict:\n            args['keywords'] = NluEnrichmentKeywords._from_dict(\n                _dict.get('keywords'))\n        if 'entities' in _dict:\n            args['entities'] = NluEnrichmentEntities._from_dict(\n                _dict.get('entities'))\n        if 'sentiment' in _dict:\n            args['sentiment'] = NluEnrichmentSentiment._from_dict(\n                _dict.get('sentiment'))\n        if 'emotion' in _dict:\n            args['emotion'] = NluEnrichmentEmotion._from_dict(\n                _dict.get('emotion'))\n        if 'categories' in _dict:\n            args['categories'] = NluEnrichmentCategories._from_dict(\n                _dict.get('categories'))\n        if 'semantic_roles' in _dict:\n            args['semantic_roles'] = NluEnrichmentSemanticRoles._from_dict(\n                _dict.get('semantic_roles'))\n        if 'relations' in _dict:\n            args['relations'] = NluEnrichmentRelations._from_dict(\n                _dict.get('relations'))\n        if 'concepts' in _dict:\n            args['concepts'] = NluEnrichmentConcepts._from_dict(\n                _dict.get('concepts'))\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'sentiment') and self.sentiment is not None:\n            _dict['sentiment'] = self.sentiment\n        if hasattr(self, 'emotion') and self.emotion is not None:\n            _dict['emotion'] = self.emotion\n        if hasattr(self, 'limit') and self.limit is not None:\n            _dict['limit'] = self.limit\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'operation') and self.operation is not None:\n            _dict['operation'] = self.operation\n        if hasattr(self, 'source_field') and self.source_field is not None:\n            _dict['source_field'] = self.source_field\n        if hasattr(self,\n                   'destination_field') and self.destination_field is not None:\n            _dict['destination_field'] = self.destination_field\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'notice_id') and self.notice_id is not None:\n            _dict['notice_id'] = self.notice_id\n        if hasattr(self, 'created') and self.created is not None:\n            _dict['created'] = datetime_to_string(self.created)\n        if hasattr(self, 'document_id') and self.document_id is not None:\n            _dict['document_id'] = self.document_id\n        if hasattr(self, 'query_id') and self.query_id is not None:\n            _dict['query_id'] = self.query_id\n        if hasattr(self, 'severity') and self.severity is not None:\n            _dict['severity'] = self.severity\n        if hasattr(self, 'step') and self.step is not None:\n            _dict['step'] = self.step\n        if hasattr(self, 'description') and self.description is not None:\n            _dict['description'] = self.description\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ninitializing a PdfHeadingDetection object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a PdfHeadingDetection object from a json dictionary.\"\"\"\n        args = {}\n        if 'fonts' in _dict:\n            args['fonts'] = [\n                FontSetting._from_dict(x) for x in (_dict.get('fonts'))\n            ]\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'fonts') and self.fonts is not None:\n            _dict['fonts'] = [x._to_dict() for x in self.fonts]\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'heading') and self.heading is not None:\n            _dict['heading'] = self.heading._to_dict()\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'type') and self.type is not None:\n            _dict['type'] = self.type\n        if hasattr(self, 'results') and self.results is not None:\n            _dict['results'] = [x._to_dict() for x in self.results]\n        if hasattr(self,\n                   'matching_results') and self.matching_results is not None:\n            _dict['matching_results'] = self.matching_results\n        if hasattr(self, 'aggregations') and self.aggregations is not None:\n            _dict['aggregations'] = [x._to_dict() for x in self.aggregations]\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'entities' in _dict:\n            args['entities'] = [\n                QueryEntitiesResponseItem._from_dict(x)\n                for x in (_dict.get('entities'))\n            ]\n        return cls(**args)", "response": "Initialize a QueryEntitiesResponse object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'entities') and self.entities is not None:\n            _dict['entities'] = [x._to_dict() for x in self.entities]\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'text') and self.text is not None:\n            _dict['text'] = self.text\n        if hasattr(self, 'type') and self.type is not None:\n            _dict['type'] = self.type\n        if hasattr(self, 'evidence') and self.evidence is not None:\n            _dict['evidence'] = [x._to_dict() for x in self.evidence]\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ninitializes a QueryEvidence object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a QueryEvidence object from a json dictionary.\"\"\"\n        args = {}\n        if 'document_id' in _dict:\n            args['document_id'] = _dict.get('document_id')\n        if 'field' in _dict:\n            args['field'] = _dict.get('field')\n        if 'start_offset' in _dict:\n            args['start_offset'] = _dict.get('start_offset')\n        if 'end_offset' in _dict:\n            args['end_offset'] = _dict.get('end_offset')\n        if 'entities' in _dict:\n            args['entities'] = [\n                QueryEvidenceEntity._from_dict(x)\n                for x in (_dict.get('entities'))\n            ]\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'document_id') and self.document_id is not None:\n            _dict['document_id'] = self.document_id\n        if hasattr(self, 'field') and self.field is not None:\n            _dict['field'] = self.field\n        if hasattr(self, 'start_offset') and self.start_offset is not None:\n            _dict['start_offset'] = self.start_offset\n        if hasattr(self, 'end_offset') and self.end_offset is not None:\n            _dict['end_offset'] = self.end_offset\n        if hasattr(self, 'entities') and self.entities is not None:\n            _dict['entities'] = [x._to_dict() for x in self.entities]\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'type') and self.type is not None:\n            _dict['type'] = self.type\n        if hasattr(self, 'text') and self.text is not None:\n            _dict['text'] = self.text\n        if hasattr(self, 'start_offset') and self.start_offset is not None:\n            _dict['start_offset'] = self.start_offset\n        if hasattr(self, 'end_offset') and self.end_offset is not None:\n            _dict['end_offset'] = self.end_offset\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'exclude') and self.exclude is not None:\n            _dict['exclude'] = self.exclude\n        if hasattr(self, 'include') and self.include is not None:\n            _dict['include'] = self.include\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'matching_results' in _dict:\n            args['matching_results'] = _dict.get('matching_results')\n        if 'results' in _dict:\n            args['results'] = [\n                QueryNoticesResult._from_dict(x) for x in (_dict.get('results'))\n            ]\n        if 'aggregations' in _dict:\n            args['aggregations'] = [\n                QueryAggregation._from_dict(x)\n                for x in (_dict.get('aggregations'))\n            ]\n        if 'passages' in _dict:\n            args['passages'] = [\n                QueryPassages._from_dict(x) for x in (_dict.get('passages'))\n            ]\n        if 'duplicates_removed' in _dict:\n            args['duplicates_removed'] = _dict.get('duplicates_removed')\n        return cls(**args)", "response": "Initialize a QueryNoticesResponse object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self,\n                   'matching_results') and self.matching_results is not None:\n            _dict['matching_results'] = self.matching_results\n        if hasattr(self, 'results') and self.results is not None:\n            _dict['results'] = [x._to_dict() for x in self.results]\n        if hasattr(self, 'aggregations') and self.aggregations is not None:\n            _dict['aggregations'] = [x._to_dict() for x in self.aggregations]\n        if hasattr(self, 'passages') and self.passages is not None:\n            _dict['passages'] = [x._to_dict() for x in self.passages]\n        if hasattr(\n                self,\n                'duplicates_removed') and self.duplicates_removed is not None:\n            _dict['duplicates_removed'] = self.duplicates_removed\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _from_dict(cls, _dict):\n        args = {}\n        xtra = _dict.copy()\n        if 'id' in _dict:\n            args['id'] = _dict.get('id')\n            del xtra['id']\n        if 'metadata' in _dict:\n            args['metadata'] = _dict.get('metadata')\n            del xtra['metadata']\n        if 'collection_id' in _dict:\n            args['collection_id'] = _dict.get('collection_id')\n            del xtra['collection_id']\n        if 'result_metadata' in _dict:\n            args['result_metadata'] = QueryResultMetadata._from_dict(\n                _dict.get('result_metadata'))\n            del xtra['result_metadata']\n        if 'title' in _dict:\n            args['title'] = _dict.get('title')\n            del xtra['title']\n        if 'code' in _dict:\n            args['code'] = _dict.get('code')\n            del xtra['code']\n        if 'filename' in _dict:\n            args['filename'] = _dict.get('filename')\n            del xtra['filename']\n        if 'file_type' in _dict:\n            args['file_type'] = _dict.get('file_type')\n            del xtra['file_type']\n        if 'sha1' in _dict:\n            args['sha1'] = _dict.get('sha1')\n            del xtra['sha1']\n        if 'notices' in _dict:\n            args['notices'] = [\n                Notice._from_dict(x) for x in (_dict.get('notices'))\n            ]\n            del xtra['notices']\n        args.update(xtra)\n        return cls(**args)", "response": "Initialize a QueryNoticesResult object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'id') and self.id is not None:\n            _dict['id'] = self.id\n        if hasattr(self, 'metadata') and self.metadata is not None:\n            _dict['metadata'] = self.metadata\n        if hasattr(self, 'collection_id') and self.collection_id is not None:\n            _dict['collection_id'] = self.collection_id\n        if hasattr(self,\n                   'result_metadata') and self.result_metadata is not None:\n            _dict['result_metadata'] = self.result_metadata._to_dict()\n        if hasattr(self, 'title') and self.title is not None:\n            _dict['title'] = self.title\n        if hasattr(self, 'code') and self.code is not None:\n            _dict['code'] = self.code\n        if hasattr(self, 'filename') and self.filename is not None:\n            _dict['filename'] = self.filename\n        if hasattr(self, 'file_type') and self.file_type is not None:\n            _dict['file_type'] = self.file_type\n        if hasattr(self, 'sha1') and self.sha1 is not None:\n            _dict['sha1'] = self.sha1\n        if hasattr(self, 'notices') and self.notices is not None:\n            _dict['notices'] = [x._to_dict() for x in self.notices]\n        if hasattr(self, '_additionalProperties'):\n            for _key in self._additionalProperties:\n                _value = getattr(self, _key, None)\n                if _value is not None:\n                    _dict[_key] = _value\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'document_id') and self.document_id is not None:\n            _dict['document_id'] = self.document_id\n        if hasattr(self, 'passage_score') and self.passage_score is not None:\n            _dict['passage_score'] = self.passage_score\n        if hasattr(self, 'passage_text') and self.passage_text is not None:\n            _dict['passage_text'] = self.passage_text\n        if hasattr(self, 'start_offset') and self.start_offset is not None:\n            _dict['start_offset'] = self.start_offset\n        if hasattr(self, 'end_offset') and self.end_offset is not None:\n            _dict['end_offset'] = self.end_offset\n        if hasattr(self, 'field') and self.field is not None:\n            _dict['field'] = self.field\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ninitialize a QueryRelationsArgument object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a QueryRelationsArgument object from a json dictionary.\"\"\"\n        args = {}\n        if 'entities' in _dict:\n            args['entities'] = [\n                QueryEntitiesEntity._from_dict(x)\n                for x in (_dict.get('entities'))\n            ]\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'relation_types') and self.relation_types is not None:\n            _dict['relation_types'] = self.relation_types._to_dict()\n        if hasattr(self, 'entity_types') and self.entity_types is not None:\n            _dict['entity_types'] = self.entity_types._to_dict()\n        if hasattr(self, 'document_ids') and self.document_ids is not None:\n            _dict['document_ids'] = self.document_ids\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'type' in _dict:\n            args['type'] = _dict.get('type')\n        if 'frequency' in _dict:\n            args['frequency'] = _dict.get('frequency')\n        if 'arguments' in _dict:\n            args['arguments'] = [\n                QueryRelationsArgument._from_dict(x)\n                for x in (_dict.get('arguments'))\n            ]\n        if 'evidence' in _dict:\n            args['evidence'] = [\n                QueryEvidence._from_dict(x) for x in (_dict.get('evidence'))\n            ]\n        return cls(**args)", "response": "Initialize a QueryRelationsRelationship object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'type') and self.type is not None:\n            _dict['type'] = self.type\n        if hasattr(self, 'frequency') and self.frequency is not None:\n            _dict['frequency'] = self.frequency\n        if hasattr(self, 'arguments') and self.arguments is not None:\n            _dict['arguments'] = [x._to_dict() for x in self.arguments]\n        if hasattr(self, 'evidence') and self.evidence is not None:\n            _dict['evidence'] = [x._to_dict() for x in self.evidence]\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'relations' in _dict:\n            args['relations'] = [\n                QueryRelationsRelationship._from_dict(x)\n                for x in (_dict.get('relations'))\n            ]\n        return cls(**args)", "response": "Initialize a QueryRelationsResponse object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'relations') and self.relations is not None:\n            _dict['relations'] = [x._to_dict() for x in self.relations]\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ninitializing a QueryResponse object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a QueryResponse object from a json dictionary.\"\"\"\n        args = {}\n        if 'matching_results' in _dict:\n            args['matching_results'] = _dict.get('matching_results')\n        if 'results' in _dict:\n            args['results'] = [\n                QueryResult._from_dict(x) for x in (_dict.get('results'))\n            ]\n        if 'aggregations' in _dict:\n            args['aggregations'] = [\n                QueryAggregation._from_dict(x)\n                for x in (_dict.get('aggregations'))\n            ]\n        if 'passages' in _dict:\n            args['passages'] = [\n                QueryPassages._from_dict(x) for x in (_dict.get('passages'))\n            ]\n        if 'duplicates_removed' in _dict:\n            args['duplicates_removed'] = _dict.get('duplicates_removed')\n        if 'session_token' in _dict:\n            args['session_token'] = _dict.get('session_token')\n        if 'retrieval_details' in _dict:\n            args['retrieval_details'] = RetrievalDetails._from_dict(\n                _dict.get('retrieval_details'))\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _from_dict(cls, _dict):\n        args = {}\n        xtra = _dict.copy()\n        if 'id' in _dict:\n            args['id'] = _dict.get('id')\n            del xtra['id']\n        if 'metadata' in _dict:\n            args['metadata'] = _dict.get('metadata')\n            del xtra['metadata']\n        if 'collection_id' in _dict:\n            args['collection_id'] = _dict.get('collection_id')\n            del xtra['collection_id']\n        if 'result_metadata' in _dict:\n            args['result_metadata'] = QueryResultMetadata._from_dict(\n                _dict.get('result_metadata'))\n            del xtra['result_metadata']\n        if 'title' in _dict:\n            args['title'] = _dict.get('title')\n            del xtra['title']\n        args.update(xtra)\n        return cls(**args)", "response": "Initialize a QueryResult object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'id') and self.id is not None:\n            _dict['id'] = self.id\n        if hasattr(self, 'metadata') and self.metadata is not None:\n            _dict['metadata'] = self.metadata\n        if hasattr(self, 'collection_id') and self.collection_id is not None:\n            _dict['collection_id'] = self.collection_id\n        if hasattr(self,\n                   'result_metadata') and self.result_metadata is not None:\n            _dict['result_metadata'] = self.result_metadata._to_dict()\n        if hasattr(self, 'title') and self.title is not None:\n            _dict['title'] = self.title\n        if hasattr(self, '_additionalProperties'):\n            for _key in self._additionalProperties:\n                _value = getattr(self, _key, None)\n                if _value is not None:\n                    _dict[_key] = _value\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'score') and self.score is not None:\n            _dict['score'] = self.score\n        if hasattr(self, 'confidence') and self.confidence is not None:\n            _dict['confidence'] = self.confidence\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'document_retrieval_strategy'\n                  ) and self.document_retrieval_strategy is not None:\n            _dict[\n                'document_retrieval_strategy'] = self.document_retrieval_strategy\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'scope') and self.scope is not None:\n            _dict['scope'] = self.scope\n        if hasattr(self, 'status') and self.status is not None:\n            _dict['status'] = self.status\n        if hasattr(\n                self,\n                'status_description') and self.status_description is not None:\n            _dict['status_description'] = self.status_description\n        if hasattr(self, 'last_trained') and self.last_trained is not None:\n            _dict['last_trained'] = self.last_trained\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'enabled') and self.enabled is not None:\n            _dict['enabled'] = self.enabled\n        if hasattr(self, 'selector_tags') and self.selector_tags is not None:\n            _dict['selector_tags'] = self.selector_tags\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'type' in _dict:\n            args['type'] = _dict.get('type')\n        if 'credential_id' in _dict:\n            args['credential_id'] = _dict.get('credential_id')\n        if 'schedule' in _dict:\n            args['schedule'] = SourceSchedule._from_dict(_dict.get('schedule'))\n        if 'options' in _dict:\n            args['options'] = SourceOptions._from_dict(_dict.get('options'))\n        return cls(**args)", "response": "Initialize a Source object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'type') and self.type is not None:\n            _dict['type'] = self.type\n        if hasattr(self, 'credential_id') and self.credential_id is not None:\n            _dict['credential_id'] = self.credential_id\n        if hasattr(self, 'schedule') and self.schedule is not None:\n            _dict['schedule'] = self.schedule._to_dict()\n        if hasattr(self, 'options') and self.options is not None:\n            _dict['options'] = self.options._to_dict()\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'folders' in _dict:\n            args['folders'] = [\n                SourceOptionsFolder._from_dict(x)\n                for x in (_dict.get('folders'))\n            ]\n        if 'objects' in _dict:\n            args['objects'] = [\n                SourceOptionsObject._from_dict(x)\n                for x in (_dict.get('objects'))\n            ]\n        if 'site_collections' in _dict:\n            args['site_collections'] = [\n                SourceOptionsSiteColl._from_dict(x)\n                for x in (_dict.get('site_collections'))\n            ]\n        if 'urls' in _dict:\n            args['urls'] = [\n                SourceOptionsWebCrawl._from_dict(x) for x in (_dict.get('urls'))\n            ]\n        if 'buckets' in _dict:\n            args['buckets'] = [\n                SourceOptionsBuckets._from_dict(x)\n                for x in (_dict.get('buckets'))\n            ]\n        if 'crawl_all_buckets' in _dict:\n            args['crawl_all_buckets'] = _dict.get('crawl_all_buckets')\n        return cls(**args)", "response": "Initialize a SourceOptions object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'folders') and self.folders is not None:\n            _dict['folders'] = [x._to_dict() for x in self.folders]\n        if hasattr(self, 'objects') and self.objects is not None:\n            _dict['objects'] = [x._to_dict() for x in self.objects]\n        if hasattr(self,\n                   'site_collections') and self.site_collections is not None:\n            _dict['site_collections'] = [\n                x._to_dict() for x in self.site_collections\n            ]\n        if hasattr(self, 'urls') and self.urls is not None:\n            _dict['urls'] = [x._to_dict() for x in self.urls]\n        if hasattr(self, 'buckets') and self.buckets is not None:\n            _dict['buckets'] = [x._to_dict() for x in self.buckets]\n        if hasattr(self,\n                   'crawl_all_buckets') and self.crawl_all_buckets is not None:\n            _dict['crawl_all_buckets'] = self.crawl_all_buckets\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'name') and self.name is not None:\n            _dict['name'] = self.name\n        if hasattr(self, 'limit') and self.limit is not None:\n            _dict['limit'] = self.limit\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'owner_user_id') and self.owner_user_id is not None:\n            _dict['owner_user_id'] = self.owner_user_id\n        if hasattr(self, 'folder_id') and self.folder_id is not None:\n            _dict['folder_id'] = self.folder_id\n        if hasattr(self, 'limit') and self.limit is not None:\n            _dict['limit'] = self.limit\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'site_collection_path'\n                  ) and self.site_collection_path is not None:\n            _dict['site_collection_path'] = self.site_collection_path\n        if hasattr(self, 'limit') and self.limit is not None:\n            _dict['limit'] = self.limit\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'url') and self.url is not None:\n            _dict['url'] = self.url\n        if hasattr(self, 'limit_to_starting_hosts'\n                  ) and self.limit_to_starting_hosts is not None:\n            _dict['limit_to_starting_hosts'] = self.limit_to_starting_hosts\n        if hasattr(self, 'crawl_speed') and self.crawl_speed is not None:\n            _dict['crawl_speed'] = self.crawl_speed\n        if hasattr(self, 'allow_untrusted_certificate'\n                  ) and self.allow_untrusted_certificate is not None:\n            _dict[\n                'allow_untrusted_certificate'] = self.allow_untrusted_certificate\n        if hasattr(self, 'maximum_hops') and self.maximum_hops is not None:\n            _dict['maximum_hops'] = self.maximum_hops\n        if hasattr(self,\n                   'request_timeout') and self.request_timeout is not None:\n            _dict['request_timeout'] = self.request_timeout\n        if hasattr(\n                self,\n                'override_robots_txt') and self.override_robots_txt is not None:\n            _dict['override_robots_txt'] = self.override_robots_txt\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'enabled') and self.enabled is not None:\n            _dict['enabled'] = self.enabled\n        if hasattr(self, 'time_zone') and self.time_zone is not None:\n            _dict['time_zone'] = self.time_zone\n        if hasattr(self, 'frequency') and self.frequency is not None:\n            _dict['frequency'] = self.frequency\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'status') and self.status is not None:\n            _dict['status'] = self.status\n        if hasattr(self, 'last_updated') and self.last_updated is not None:\n            _dict['last_updated'] = datetime_to_string(self.last_updated)\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'field') and self.field is not None:\n            _dict['field'] = self.field\n        if hasattr(self, 'count') and self.count is not None:\n            _dict['count'] = self.count\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'field') and self.field is not None:\n            _dict['field'] = self.field\n        if hasattr(self, 'interval') and self.interval is not None:\n            _dict['interval'] = self.interval\n        if hasattr(self, 'anomaly') and self.anomaly is not None:\n            _dict['anomaly'] = self.anomaly\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'text') and self.text is not None:\n            _dict['text'] = self.text\n        if hasattr(self, 'tokens') and self.tokens is not None:\n            _dict['tokens'] = self.tokens\n        if hasattr(self, 'readings') and self.readings is not None:\n            _dict['readings'] = self.readings\n        if hasattr(self, 'part_of_speech') and self.part_of_speech is not None:\n            _dict['part_of_speech'] = self.part_of_speech\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'status') and self.status is not None:\n            _dict['status'] = self.status\n        if hasattr(self, 'type') and self.type is not None:\n            _dict['type'] = self.type\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'size') and self.size is not None:\n            _dict['size'] = self.size\n        if hasattr(self, 'hits') and self.hits is not None:\n            _dict['hits'] = self.hits._to_dict()\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ninitializes a TopHitsResults object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a TopHitsResults object from a json dictionary.\"\"\"\n        args = {}\n        if 'matching_results' in _dict:\n            args['matching_results'] = _dict.get('matching_results')\n        if 'hits' in _dict:\n            args['hits'] = [\n                QueryResult._from_dict(x) for x in (_dict.get('hits'))\n            ]\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self,\n                   'matching_results') and self.matching_results is not None:\n            _dict['matching_results'] = self.matching_results\n        if hasattr(self, 'hits') and self.hits is not None:\n            _dict['hits'] = [x._to_dict() for x in self.hits]\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'environment_id' in _dict:\n            args['environment_id'] = _dict.get('environment_id')\n        if 'collection_id' in _dict:\n            args['collection_id'] = _dict.get('collection_id')\n        if 'queries' in _dict:\n            args['queries'] = [\n                TrainingQuery._from_dict(x) for x in (_dict.get('queries'))\n            ]\n        return cls(**args)", "response": "Initialize a TrainingDataSet object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'environment_id') and self.environment_id is not None:\n            _dict['environment_id'] = self.environment_id\n        if hasattr(self, 'collection_id') and self.collection_id is not None:\n            _dict['collection_id'] = self.collection_id\n        if hasattr(self, 'queries') and self.queries is not None:\n            _dict['queries'] = [x._to_dict() for x in self.queries]\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'document_id') and self.document_id is not None:\n            _dict['document_id'] = self.document_id\n        if hasattr(self,\n                   'cross_reference') and self.cross_reference is not None:\n            _dict['cross_reference'] = self.cross_reference\n        if hasattr(self, 'relevance') and self.relevance is not None:\n            _dict['relevance'] = self.relevance\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'examples' in _dict:\n            args['examples'] = [\n                TrainingExample._from_dict(x) for x in (_dict.get('examples'))\n            ]\n        return cls(**args)", "response": "Initialize a TrainingExampleList object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'examples') and self.examples is not None:\n            _dict['examples'] = [x._to_dict() for x in self.examples]\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'query_id' in _dict:\n            args['query_id'] = _dict.get('query_id')\n        if 'natural_language_query' in _dict:\n            args['natural_language_query'] = _dict.get('natural_language_query')\n        if 'filter' in _dict:\n            args['filter'] = _dict.get('filter')\n        if 'examples' in _dict:\n            args['examples'] = [\n                TrainingExample._from_dict(x) for x in (_dict.get('examples'))\n            ]\n        return cls(**args)", "response": "Initialize a TrainingQuery object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'query_id') and self.query_id is not None:\n            _dict['query_id'] = self.query_id\n        if hasattr(self, 'natural_language_query'\n                  ) and self.natural_language_query is not None:\n            _dict['natural_language_query'] = self.natural_language_query\n        if hasattr(self, 'filter') and self.filter is not None:\n            _dict['filter'] = self.filter\n        if hasattr(self, 'examples') and self.examples is not None:\n            _dict['examples'] = [x._to_dict() for x in self.examples]\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'total_examples') and self.total_examples is not None:\n            _dict['total_examples'] = self.total_examples\n        if hasattr(self, 'available') and self.available is not None:\n            _dict['available'] = self.available\n        if hasattr(self, 'processing') and self.processing is not None:\n            _dict['processing'] = self.processing\n        if hasattr(self, 'minimum_queries_added'\n                  ) and self.minimum_queries_added is not None:\n            _dict['minimum_queries_added'] = self.minimum_queries_added\n        if hasattr(self, 'minimum_examples_added'\n                  ) and self.minimum_examples_added is not None:\n            _dict['minimum_examples_added'] = self.minimum_examples_added\n        if hasattr(self, 'sufficient_label_diversity'\n                  ) and self.sufficient_label_diversity is not None:\n            _dict[\n                'sufficient_label_diversity'] = self.sufficient_label_diversity\n        if hasattr(self, 'notices') and self.notices is not None:\n            _dict['notices'] = self.notices\n        if hasattr(self, 'successfully_trained'\n                  ) and self.successfully_trained is not None:\n            _dict['successfully_trained'] = datetime_to_string(\n                self.successfully_trained)\n        if hasattr(self, 'data_updated') and self.data_updated is not None:\n            _dict['data_updated'] = datetime_to_string(self.data_updated)\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ninitializes a WordHeadingDetection object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a WordHeadingDetection object from a json dictionary.\"\"\"\n        args = {}\n        if 'fonts' in _dict:\n            args['fonts'] = [\n                FontSetting._from_dict(x) for x in (_dict.get('fonts'))\n            ]\n        if 'styles' in _dict:\n            args['styles'] = [\n                WordStyle._from_dict(x) for x in (_dict.get('styles'))\n            ]\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'fonts') and self.fonts is not None:\n            _dict['fonts'] = [x._to_dict() for x in self.fonts]\n        if hasattr(self, 'styles') and self.styles is not None:\n            _dict['styles'] = [x._to_dict() for x in self.styles]\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'level') and self.level is not None:\n            _dict['level'] = self.level\n        if hasattr(self, 'names') and self.names is not None:\n            _dict['names'] = self.names\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'xpaths') and self.xpaths is not None:\n            _dict['xpaths'] = self.xpaths\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nanalyzes a tone of a full document and return a dictionary of the result.", "response": "def tone(self,\n             tone_input,\n             sentences=None,\n             tones=None,\n             content_language=None,\n             accept_language=None,\n             content_type=None,\n             **kwargs):\n        \"\"\"\n        Analyze general tone.\n\n        Use the general purpose endpoint to analyze the tone of your input content. The\n        service analyzes the content for emotional and language tones. The method always\n        analyzes the tone of the full document; by default, it also analyzes the tone of\n        each individual sentence of the content.\n        You can submit no more than 128 KB of total input content and no more than 1000\n        individual sentences in JSON, plain text, or HTML format. The service analyzes the\n        first 1000 sentences for document-level analysis and only the first 100 sentences\n        for sentence-level analysis.\n        Per the JSON specification, the default character encoding for JSON content is\n        effectively always UTF-8; per the HTTP specification, the default encoding for\n        plain text and HTML is ISO-8859-1 (effectively, the ASCII character set). When\n        specifying a content type of plain text or HTML, include the `charset` parameter\n        to indicate the character encoding of the input text; for example: `Content-Type:\n        text/plain;charset=utf-8`. For `text/html`, the service removes HTML tags and\n        analyzes only the textual content.\n        **See also:** [Using the general-purpose\n        endpoint](https://cloud.ibm.com/docs/services/tone-analyzer/using-tone.html#using-the-general-purpose-endpoint).\n\n        :param ToneInput tone_input: JSON, plain text, or HTML input that contains the\n        content to be analyzed. For JSON input, provide an object of type `ToneInput`.\n        :param bool sentences: Indicates whether the service is to return an analysis of\n        each individual sentence in addition to its analysis of the full document. If\n        `true` (the default), the service returns results for each sentence.\n        :param list[str] tones: **`2017-09-21`:** Deprecated. The service continues to\n        accept the parameter for backward-compatibility, but the parameter no longer\n        affects the response.\n        **`2016-05-19`:** A comma-separated list of tones for which the service is to\n        return its analysis of the input; the indicated tones apply both to the full\n        document and to individual sentences of the document. You can specify one or more\n        of the valid values. Omit the parameter to request results for all three tones.\n        :param str content_language: The language of the input text for the request:\n        English or French. Regional variants are treated as their parent language; for\n        example, `en-US` is interpreted as `en`. The input content must match the\n        specified language. Do not submit content that contains both languages. You can\n        use different languages for **Content-Language** and **Accept-Language**.\n        * **`2017-09-21`:** Accepts `en` or `fr`.\n        * **`2016-05-19`:** Accepts only `en`.\n        :param str accept_language: The desired language of the response. For\n        two-character arguments, regional variants are treated as their parent language;\n        for example, `en-US` is interpreted as `en`. You can use different languages for\n        **Content-Language** and **Accept-Language**.\n        :param str content_type: The type of the input. A character encoding can be\n        specified by including a `charset` parameter. For example,\n        'text/plain;charset=utf-8'.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse\n        \"\"\"\n\n        if tone_input is None:\n            raise ValueError('tone_input must be provided')\n        if isinstance(tone_input, ToneInput):\n            tone_input = self._convert_model(tone_input, ToneInput)\n\n        headers = {\n            'Content-Language': content_language,\n            'Accept-Language': accept_language,\n            'Content-Type': content_type\n        }\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('tone_analyzer', 'V3', 'tone')\n        headers.update(sdk_headers)\n\n        params = {\n            'version': self.version,\n            'sentences': sentences,\n            'tones': self._convert_list(tones)\n        }\n\n        if content_type == 'application/json' and isinstance(tone_input, dict):\n            data = json.dumps(tone_input)\n        else:\n            data = tone_input\n\n        url = '/v3/tone'\n        response = self.request(\n            method='POST',\n            url=url,\n            headers=headers,\n            params=params,\n            data=data,\n            accept_json=True)\n        return response"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nanalyzes the tone of a customer service and customer support conversations and return a dict containing the most prevalent subset of the utterances.", "response": "def tone_chat(self,\n                  utterances,\n                  content_language=None,\n                  accept_language=None,\n                  **kwargs):\n        \"\"\"\n        Analyze customer engagement tone.\n\n        Use the customer engagement endpoint to analyze the tone of customer service and\n        customer support conversations. For each utterance of a conversation, the method\n        reports the most prevalent subset of the following seven tones: sad, frustrated,\n        satisfied, excited, polite, impolite, and sympathetic.\n        If you submit more than 50 utterances, the service returns a warning for the\n        overall content and analyzes only the first 50 utterances. If you submit a single\n        utterance that contains more than 500 characters, the service returns an error for\n        that utterance and does not analyze the utterance. The request fails if all\n        utterances have more than 500 characters. Per the JSON specification, the default\n        character encoding for JSON content is effectively always UTF-8.\n        **See also:** [Using the customer-engagement\n        endpoint](https://cloud.ibm.com/docs/services/tone-analyzer/using-tone-chat.html#using-the-customer-engagement-endpoint).\n\n        :param list[Utterance] utterances: An array of `Utterance` objects that provides\n        the input content that the service is to analyze.\n        :param str content_language: The language of the input text for the request:\n        English or French. Regional variants are treated as their parent language; for\n        example, `en-US` is interpreted as `en`. The input content must match the\n        specified language. Do not submit content that contains both languages. You can\n        use different languages for **Content-Language** and **Accept-Language**.\n        * **`2017-09-21`:** Accepts `en` or `fr`.\n        * **`2016-05-19`:** Accepts only `en`.\n        :param str accept_language: The desired language of the response. For\n        two-character arguments, regional variants are treated as their parent language;\n        for example, `en-US` is interpreted as `en`. You can use different languages for\n        **Content-Language** and **Accept-Language**.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse\n        \"\"\"\n\n        if utterances is None:\n            raise ValueError('utterances must be provided')\n        utterances = [self._convert_model(x, Utterance) for x in utterances]\n\n        headers = {\n            'Content-Language': content_language,\n            'Accept-Language': accept_language\n        }\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('tone_analyzer', 'V3', 'tone_chat')\n        headers.update(sdk_headers)\n\n        params = {'version': self.version}\n\n        data = {'utterances': utterances}\n\n        url = '/v3/tone_chat'\n        response = self.request(\n            method='POST',\n            url=url,\n            headers=headers,\n            params=params,\n            json=data,\n            accept_json=True)\n        return response"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'tones' in _dict:\n            args['tones'] = [\n                ToneScore._from_dict(x) for x in (_dict.get('tones'))\n            ]\n        if 'tone_categories' in _dict:\n            args['tone_categories'] = [\n                ToneCategory._from_dict(x)\n                for x in (_dict.get('tone_categories'))\n            ]\n        if 'warning' in _dict:\n            args['warning'] = _dict.get('warning')\n        return cls(**args)", "response": "Initialize a DocumentAnalysis object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'tones') and self.tones is not None:\n            _dict['tones'] = [x._to_dict() for x in self.tones]\n        if hasattr(self,\n                   'tone_categories') and self.tone_categories is not None:\n            _dict['tone_categories'] = [\n                x._to_dict() for x in self.tone_categories\n            ]\n        if hasattr(self, 'warning') and self.warning is not None:\n            _dict['warning'] = self.warning\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'sentence_id' in _dict:\n            args['sentence_id'] = _dict.get('sentence_id')\n        else:\n            raise ValueError(\n                'Required property \\'sentence_id\\' not present in SentenceAnalysis JSON'\n            )\n        if 'text' in _dict:\n            args['text'] = _dict.get('text')\n        else:\n            raise ValueError(\n                'Required property \\'text\\' not present in SentenceAnalysis JSON'\n            )\n        if 'tones' in _dict:\n            args['tones'] = [\n                ToneScore._from_dict(x) for x in (_dict.get('tones'))\n            ]\n        if 'tone_categories' in _dict:\n            args['tone_categories'] = [\n                ToneCategory._from_dict(x)\n                for x in (_dict.get('tone_categories'))\n            ]\n        if 'input_from' in _dict:\n            args['input_from'] = _dict.get('input_from')\n        if 'input_to' in _dict:\n            args['input_to'] = _dict.get('input_to')\n        return cls(**args)", "response": "Initialize a SentenceAnalysis object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'sentence_id') and self.sentence_id is not None:\n            _dict['sentence_id'] = self.sentence_id\n        if hasattr(self, 'text') and self.text is not None:\n            _dict['text'] = self.text\n        if hasattr(self, 'tones') and self.tones is not None:\n            _dict['tones'] = [x._to_dict() for x in self.tones]\n        if hasattr(self,\n                   'tone_categories') and self.tone_categories is not None:\n            _dict['tone_categories'] = [\n                x._to_dict() for x in self.tone_categories\n            ]\n        if hasattr(self, 'input_from') and self.input_from is not None:\n            _dict['input_from'] = self.input_from\n        if hasattr(self, 'input_to') and self.input_to is not None:\n            _dict['input_to'] = self.input_to\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'document_tone' in _dict:\n            args['document_tone'] = DocumentAnalysis._from_dict(\n                _dict.get('document_tone'))\n        else:\n            raise ValueError(\n                'Required property \\'document_tone\\' not present in ToneAnalysis JSON'\n            )\n        if 'sentences_tone' in _dict:\n            args['sentences_tone'] = [\n                SentenceAnalysis._from_dict(x)\n                for x in (_dict.get('sentences_tone'))\n            ]\n        return cls(**args)", "response": "Initialize a ToneAnalysis object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'document_tone') and self.document_tone is not None:\n            _dict['document_tone'] = self.document_tone._to_dict()\n        if hasattr(self, 'sentences_tone') and self.sentences_tone is not None:\n            _dict['sentences_tone'] = [\n                x._to_dict() for x in self.sentences_tone\n            ]\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'tones' in _dict:\n            args['tones'] = [\n                ToneScore._from_dict(x) for x in (_dict.get('tones'))\n            ]\n        else:\n            raise ValueError(\n                'Required property \\'tones\\' not present in ToneCategory JSON')\n        if 'category_id' in _dict:\n            args['category_id'] = _dict.get('category_id')\n        else:\n            raise ValueError(\n                'Required property \\'category_id\\' not present in ToneCategory JSON'\n            )\n        if 'category_name' in _dict:\n            args['category_name'] = _dict.get('category_name')\n        else:\n            raise ValueError(\n                'Required property \\'category_name\\' not present in ToneCategory JSON'\n            )\n        return cls(**args)", "response": "Initialize a ToneCategory object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'tones') and self.tones is not None:\n            _dict['tones'] = [x._to_dict() for x in self.tones]\n        if hasattr(self, 'category_id') and self.category_id is not None:\n            _dict['category_id'] = self.category_id\n        if hasattr(self, 'category_name') and self.category_name is not None:\n            _dict['category_name'] = self.category_name\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'score') and self.score is not None:\n            _dict['score'] = self.score\n        if hasattr(self, 'tone_id') and self.tone_id is not None:\n            _dict['tone_id'] = self.tone_id\n        if hasattr(self, 'tone_name') and self.tone_name is not None:\n            _dict['tone_name'] = self.tone_name\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'text') and self.text is not None:\n            _dict['text'] = self.text\n        if hasattr(self, 'user') and self.user is not None:\n            _dict['user'] = self.user\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ninitializing a UtteranceAnalyses object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a UtteranceAnalyses object from a json dictionary.\"\"\"\n        args = {}\n        if 'utterances_tone' in _dict:\n            args['utterances_tone'] = [\n                UtteranceAnalysis._from_dict(x)\n                for x in (_dict.get('utterances_tone'))\n            ]\n        else:\n            raise ValueError(\n                'Required property \\'utterances_tone\\' not present in UtteranceAnalyses JSON'\n            )\n        if 'warning' in _dict:\n            args['warning'] = _dict.get('warning')\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self,\n                   'utterances_tone') and self.utterances_tone is not None:\n            _dict['utterances_tone'] = [\n                x._to_dict() for x in self.utterances_tone\n            ]\n        if hasattr(self, 'warning') and self.warning is not None:\n            _dict['warning'] = self.warning\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ninitialize a UtteranceAnalysis object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a UtteranceAnalysis object from a json dictionary.\"\"\"\n        args = {}\n        if 'utterance_id' in _dict:\n            args['utterance_id'] = _dict.get('utterance_id')\n        else:\n            raise ValueError(\n                'Required property \\'utterance_id\\' not present in UtteranceAnalysis JSON'\n            )\n        if 'utterance_text' in _dict:\n            args['utterance_text'] = _dict.get('utterance_text')\n        else:\n            raise ValueError(\n                'Required property \\'utterance_text\\' not present in UtteranceAnalysis JSON'\n            )\n        if 'tones' in _dict:\n            args['tones'] = [\n                ToneChatScore._from_dict(x) for x in (_dict.get('tones'))\n            ]\n        else:\n            raise ValueError(\n                'Required property \\'tones\\' not present in UtteranceAnalysis JSON'\n            )\n        if 'error' in _dict:\n            args['error'] = _dict.get('error')\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'utterance_id') and self.utterance_id is not None:\n            _dict['utterance_id'] = self.utterance_id\n        if hasattr(self, 'utterance_text') and self.utterance_text is not None:\n            _dict['utterance_text'] = self.utterance_text\n        if hasattr(self, 'tones') and self.tones is not None:\n            _dict['tones'] = [x._to_dict() for x in self.tones]\n        if hasattr(self, 'error') and self.error is not None:\n            _dict['error'] = self.error\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef message(self,\n                workspace_id,\n                input=None,\n                intents=None,\n                entities=None,\n                alternate_intents=None,\n                context=None,\n                output=None,\n                nodes_visited_details=None,\n                **kwargs):\n        \"\"\"\n        Get response to user input.\n\n        Send user input to a workspace and receive a response.\n        There is no rate limit for this operation.\n\n        :param str workspace_id: Unique identifier of the workspace.\n        :param MessageInput input: An input object that includes the input text.\n        :param list[RuntimeIntent] intents: Intents to use when evaluating the user input.\n        Include intents from the previous response to continue using those intents rather\n        than trying to recognize intents in the new input.\n        :param list[RuntimeEntity] entities: Entities to use when evaluating the message.\n        Include entities from the previous response to continue using those entities\n        rather than detecting entities in the new input.\n        :param bool alternate_intents: Whether to return more than one intent. A value of\n        `true` indicates that all matching intents are returned.\n        :param Context context: State information for the conversation. To maintain state,\n        include the context from the previous response.\n        :param OutputData output: An output object that includes the response to the user,\n        the dialog nodes that were triggered, and messages from the log.\n        :param bool nodes_visited_details: Whether to include additional diagnostic\n        information about the dialog nodes that were visited during processing of the\n        message.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse\n        \"\"\"\n\n        if workspace_id is None:\n            raise ValueError('workspace_id must be provided')\n        if input is not None:\n            input = self._convert_model(input, MessageInput)\n        if intents is not None:\n            intents = [self._convert_model(x, RuntimeIntent) for x in intents]\n        if entities is not None:\n            entities = [self._convert_model(x, RuntimeEntity) for x in entities]\n        if context is not None:\n            context = self._convert_model(context, Context)\n        if output is not None:\n            output = self._convert_model(output, OutputData)\n\n        headers = {}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('conversation', 'V1', 'message')\n        headers.update(sdk_headers)\n\n        params = {\n            'version': self.version,\n            'nodes_visited_details': nodes_visited_details\n        }\n\n        data = {\n            'input': input,\n            'intents': intents,\n            'entities': entities,\n            'alternate_intents': alternate_intents,\n            'context': context,\n            'output': output\n        }\n\n        url = '/v1/workspaces/{0}/message'.format(\n            *self._encode_path_vars(workspace_id))\n        response = self.request(\n            method='POST',\n            url=url,\n            headers=headers,\n            params=params,\n            json=data,\n            accept_json=True)\n        return response", "response": "This method returns a response to a user input from a workspace and receive a response."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nlist the workspaces associated with a Watson Assistant service instance.", "response": "def list_workspaces(self,\n                        page_limit=None,\n                        include_count=None,\n                        sort=None,\n                        cursor=None,\n                        include_audit=None,\n                        **kwargs):\n        \"\"\"\n        List workspaces.\n\n        List the workspaces associated with a Watson Assistant service instance.\n        This operation is limited to 500 requests per 30 minutes. For more information,\n        see **Rate limiting**.\n\n        :param int page_limit: The number of records to return in each page of results.\n        :param bool include_count: Whether to include information about the number of\n        records returned.\n        :param str sort: The attribute by which returned workspaces will be sorted. To\n        reverse the sort order, prefix the value with a minus sign (`-`).\n        :param str cursor: A token identifying the page of results to retrieve.\n        :param bool include_audit: Whether to include the audit properties (`created` and\n        `updated` timestamps) in the response.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse\n        \"\"\"\n\n        headers = {}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('conversation', 'V1', 'list_workspaces')\n        headers.update(sdk_headers)\n\n        params = {\n            'version': self.version,\n            'page_limit': page_limit,\n            'include_count': include_count,\n            'sort': sort,\n            'cursor': cursor,\n            'include_audit': include_audit\n        }\n\n        url = '/v1/workspaces'\n        response = self.request(\n            method='GET',\n            url=url,\n            headers=headers,\n            params=params,\n            accept_json=True)\n        return response"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef update_workspace(self,\n                         workspace_id,\n                         name=None,\n                         description=None,\n                         language=None,\n                         metadata=None,\n                         learning_opt_out=None,\n                         system_settings=None,\n                         intents=None,\n                         entities=None,\n                         dialog_nodes=None,\n                         counterexamples=None,\n                         append=None,\n                         **kwargs):\n        \"\"\"\n        Update workspace.\n\n        Update an existing workspace with new or modified data. You must provide component\n        objects defining the content of the updated workspace.\n        This operation is limited to 30 request per 30 minutes. For more information, see\n        **Rate limiting**.\n\n        :param str workspace_id: Unique identifier of the workspace.\n        :param str name: The name of the workspace. This string cannot contain carriage\n        return, newline, or tab characters, and it must be no longer than 64 characters.\n        :param str description: The description of the workspace. This string cannot\n        contain carriage return, newline, or tab characters, and it must be no longer than\n        128 characters.\n        :param str language: The language of the workspace.\n        :param dict metadata: Any metadata related to the workspace.\n        :param bool learning_opt_out: Whether training data from the workspace (including\n        artifacts such as intents and entities) can be used by IBM for general service\n        improvements. `true` indicates that workspace training data is not to be used.\n        :param WorkspaceSystemSettings system_settings: Global settings for the workspace.\n        :param list[CreateIntent] intents: An array of objects defining the intents for\n        the workspace.\n        :param list[CreateEntity] entities: An array of objects describing the entities\n        for the workspace.\n        :param list[DialogNode] dialog_nodes: An array of objects describing the dialog\n        nodes in the workspace.\n        :param list[Counterexample] counterexamples: An array of objects defining input\n        examples that have been marked as irrelevant input.\n        :param bool append: Whether the new data is to be appended to the existing data in\n        the workspace. If **append**=`false`, elements included in the new data completely\n        replace the corresponding existing elements, including all subelements. For\n        example, if the new data includes **entities** and **append**=`false`, all\n        existing entities in the workspace are discarded and replaced with the new\n        entities.\n        If **append**=`true`, existing elements are preserved, and the new elements are\n        added. If any elements in the new data collide with existing elements, the update\n        request fails.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse\n        \"\"\"\n\n        if workspace_id is None:\n            raise ValueError('workspace_id must be provided')\n        if system_settings is not None:\n            system_settings = self._convert_model(system_settings,\n                                                  WorkspaceSystemSettings)\n        if intents is not None:\n            intents = [self._convert_model(x, CreateIntent) for x in intents]\n        if entities is not None:\n            entities = [self._convert_model(x, CreateEntity) for x in entities]\n        if dialog_nodes is not None:\n            dialog_nodes = [\n                self._convert_model(x, DialogNode) for x in dialog_nodes\n            ]\n        if counterexamples is not None:\n            counterexamples = [\n                self._convert_model(x, Counterexample) for x in counterexamples\n            ]\n\n        headers = {}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('conversation', 'V1', 'update_workspace')\n        headers.update(sdk_headers)\n\n        params = {'version': self.version, 'append': append}\n\n        data = {\n            'name': name,\n            'description': description,\n            'language': language,\n            'metadata': metadata,\n            'learning_opt_out': learning_opt_out,\n            'system_settings': system_settings,\n            'intents': intents,\n            'entities': entities,\n            'dialog_nodes': dialog_nodes,\n            'counterexamples': counterexamples\n        }\n\n        url = '/v1/workspaces/{0}'.format(*self._encode_path_vars(workspace_id))\n        response = self.request(\n            method='POST',\n            url=url,\n            headers=headers,\n            params=params,\n            json=data,\n            accept_json=True)\n        return response", "response": "Update an existing workspace with new or modified data."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef create_intent(self,\n                      workspace_id,\n                      intent,\n                      description=None,\n                      examples=None,\n                      **kwargs):\n        \"\"\"\n        Create intent.\n\n        Create a new intent.\n        This operation is limited to 2000 requests per 30 minutes. For more information,\n        see **Rate limiting**.\n\n        :param str workspace_id: Unique identifier of the workspace.\n        :param str intent: The name of the intent. This string must conform to the\n        following restrictions:\n        - It can contain only Unicode alphanumeric, underscore, hyphen, and dot\n        characters.\n        - It cannot begin with the reserved prefix `sys-`.\n        - It must be no longer than 128 characters.\n        :param str description: The description of the intent. This string cannot contain\n        carriage return, newline, or tab characters, and it must be no longer than 128\n        characters.\n        :param list[Example] examples: An array of user input examples for the intent.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse\n        \"\"\"\n\n        if workspace_id is None:\n            raise ValueError('workspace_id must be provided')\n        if intent is None:\n            raise ValueError('intent must be provided')\n        if examples is not None:\n            examples = [self._convert_model(x, Example) for x in examples]\n\n        headers = {}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('conversation', 'V1', 'create_intent')\n        headers.update(sdk_headers)\n\n        params = {'version': self.version}\n\n        data = {\n            'intent': intent,\n            'description': description,\n            'examples': examples\n        }\n\n        url = '/v1/workspaces/{0}/intents'.format(\n            *self._encode_path_vars(workspace_id))\n        response = self.request(\n            method='POST',\n            url=url,\n            headers=headers,\n            params=params,\n            json=data,\n            accept_json=True)\n        return response", "response": "Create a new intent."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nupdates an existing intent.", "response": "def update_intent(self,\n                      workspace_id,\n                      intent,\n                      new_intent=None,\n                      new_description=None,\n                      new_examples=None,\n                      **kwargs):\n        \"\"\"\n        Update intent.\n\n        Update an existing intent with new or modified data. You must provide component\n        objects defining the content of the updated intent.\n        This operation is limited to 2000 requests per 30 minutes. For more information,\n        see **Rate limiting**.\n\n        :param str workspace_id: Unique identifier of the workspace.\n        :param str intent: The intent name.\n        :param str new_intent: The name of the intent. This string must conform to the\n        following restrictions:\n        - It can contain only Unicode alphanumeric, underscore, hyphen, and dot\n        characters.\n        - It cannot begin with the reserved prefix `sys-`.\n        - It must be no longer than 128 characters.\n        :param str new_description: The description of the intent. This string cannot\n        contain carriage return, newline, or tab characters, and it must be no longer than\n        128 characters.\n        :param list[Example] new_examples: An array of user input examples for the intent.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse\n        \"\"\"\n\n        if workspace_id is None:\n            raise ValueError('workspace_id must be provided')\n        if intent is None:\n            raise ValueError('intent must be provided')\n        if new_examples is not None:\n            new_examples = [\n                self._convert_model(x, Example) for x in new_examples\n            ]\n\n        headers = {}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('conversation', 'V1', 'update_intent')\n        headers.update(sdk_headers)\n\n        params = {'version': self.version}\n\n        data = {\n            'intent': new_intent,\n            'description': new_description,\n            'examples': new_examples\n        }\n\n        url = '/v1/workspaces/{0}/intents/{1}'.format(\n            *self._encode_path_vars(workspace_id, intent))\n        response = self.request(\n            method='POST',\n            url=url,\n            headers=headers,\n            params=params,\n            json=data,\n            accept_json=True)\n        return response"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef create_example(self,\n                       workspace_id,\n                       intent,\n                       text,\n                       mentions=None,\n                       **kwargs):\n        \"\"\"\n        Create user input example.\n\n        Add a new user input example to an intent.\n        This operation is limited to 1000 requests per 30 minutes. For more information,\n        see **Rate limiting**.\n\n        :param str workspace_id: Unique identifier of the workspace.\n        :param str intent: The intent name.\n        :param str text: The text of a user input example. This string must conform to the\n        following restrictions:\n        - It cannot contain carriage return, newline, or tab characters.\n        - It cannot consist of only whitespace characters.\n        - It must be no longer than 1024 characters.\n        :param list[Mention] mentions: An array of contextual entity mentions.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse\n        \"\"\"\n\n        if workspace_id is None:\n            raise ValueError('workspace_id must be provided')\n        if intent is None:\n            raise ValueError('intent must be provided')\n        if text is None:\n            raise ValueError('text must be provided')\n        if mentions is not None:\n            mentions = [self._convert_model(x, Mention) for x in mentions]\n\n        headers = {}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('conversation', 'V1', 'create_example')\n        headers.update(sdk_headers)\n\n        params = {'version': self.version}\n\n        data = {'text': text, 'mentions': mentions}\n\n        url = '/v1/workspaces/{0}/intents/{1}/examples'.format(\n            *self._encode_path_vars(workspace_id, intent))\n        response = self.request(\n            method='POST',\n            url=url,\n            headers=headers,\n            params=params,\n            json=data,\n            accept_json=True)\n        return response", "response": "Create a new user input example."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncreate a new entity.", "response": "def create_entity(self,\n                      workspace_id,\n                      entity,\n                      description=None,\n                      metadata=None,\n                      fuzzy_match=None,\n                      values=None,\n                      **kwargs):\n        \"\"\"\n        Create entity.\n\n        Create a new entity, or enable a system entity.\n        This operation is limited to 1000 requests per 30 minutes. For more information,\n        see **Rate limiting**.\n\n        :param str workspace_id: Unique identifier of the workspace.\n        :param str entity: The name of the entity. This string must conform to the\n        following restrictions:\n        - It can contain only Unicode alphanumeric, underscore, and hyphen characters.\n        - It must be no longer than 64 characters.\n        If you specify an entity name beginning with the reserved prefix `sys-`, it must\n        be the name of a system entity that you want to enable. (Any entity content\n        specified with the request is ignored.).\n        :param str description: The description of the entity. This string cannot contain\n        carriage return, newline, or tab characters, and it must be no longer than 128\n        characters.\n        :param dict metadata: Any metadata related to the entity.\n        :param bool fuzzy_match: Whether to use fuzzy matching for the entity.\n        :param list[CreateValue] values: An array of objects describing the entity values.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse\n        \"\"\"\n\n        if workspace_id is None:\n            raise ValueError('workspace_id must be provided')\n        if entity is None:\n            raise ValueError('entity must be provided')\n        if values is not None:\n            values = [self._convert_model(x, CreateValue) for x in values]\n\n        headers = {}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('conversation', 'V1', 'create_entity')\n        headers.update(sdk_headers)\n\n        params = {'version': self.version}\n\n        data = {\n            'entity': entity,\n            'description': description,\n            'metadata': metadata,\n            'fuzzy_match': fuzzy_match,\n            'values': values\n        }\n\n        url = '/v1/workspaces/{0}/entities'.format(\n            *self._encode_path_vars(workspace_id))\n        response = self.request(\n            method='POST',\n            url=url,\n            headers=headers,\n            params=params,\n            json=data,\n            accept_json=True)\n        return response"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef update_entity(self,\n                      workspace_id,\n                      entity,\n                      new_entity=None,\n                      new_description=None,\n                      new_metadata=None,\n                      new_fuzzy_match=None,\n                      new_values=None,\n                      **kwargs):\n        \"\"\"\n        Update entity.\n\n        Update an existing entity with new or modified data. You must provide component\n        objects defining the content of the updated entity.\n        This operation is limited to 1000 requests per 30 minutes. For more information,\n        see **Rate limiting**.\n\n        :param str workspace_id: Unique identifier of the workspace.\n        :param str entity: The name of the entity.\n        :param str new_entity: The name of the entity. This string must conform to the\n        following restrictions:\n        - It can contain only Unicode alphanumeric, underscore, and hyphen characters.\n        - It cannot begin with the reserved prefix `sys-`.\n        - It must be no longer than 64 characters.\n        :param str new_description: The description of the entity. This string cannot\n        contain carriage return, newline, or tab characters, and it must be no longer than\n        128 characters.\n        :param dict new_metadata: Any metadata related to the entity.\n        :param bool new_fuzzy_match: Whether to use fuzzy matching for the entity.\n        :param list[CreateValue] new_values: An array of objects describing the entity\n        values.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse\n        \"\"\"\n\n        if workspace_id is None:\n            raise ValueError('workspace_id must be provided')\n        if entity is None:\n            raise ValueError('entity must be provided')\n        if new_values is not None:\n            new_values = [\n                self._convert_model(x, CreateValue) for x in new_values\n            ]\n\n        headers = {}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('conversation', 'V1', 'update_entity')\n        headers.update(sdk_headers)\n\n        params = {'version': self.version}\n\n        data = {\n            'entity': new_entity,\n            'description': new_description,\n            'metadata': new_metadata,\n            'fuzzy_match': new_fuzzy_match,\n            'values': new_values\n        }\n\n        url = '/v1/workspaces/{0}/entities/{1}'.format(\n            *self._encode_path_vars(workspace_id, entity))\n        response = self.request(\n            method='POST',\n            url=url,\n            headers=headers,\n            params=params,\n            json=data,\n            accept_json=True)\n        return response", "response": "Update an existing entity."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef create_value(self,\n                     workspace_id,\n                     entity,\n                     value,\n                     metadata=None,\n                     value_type=None,\n                     synonyms=None,\n                     patterns=None,\n                     **kwargs):\n        \"\"\"\n        Create entity value.\n\n        Create a new value for an entity.\n        This operation is limited to 1000 requests per 30 minutes. For more information,\n        see **Rate limiting**.\n\n        :param str workspace_id: Unique identifier of the workspace.\n        :param str entity: The name of the entity.\n        :param str value: The text of the entity value. This string must conform to the\n        following restrictions:\n        - It cannot contain carriage return, newline, or tab characters.\n        - It cannot consist of only whitespace characters.\n        - It must be no longer than 64 characters.\n        :param dict metadata: Any metadata related to the entity value.\n        :param str value_type: Specifies the type of entity value.\n        :param list[str] synonyms: An array of synonyms for the entity value. A value can\n        specify either synonyms or patterns (depending on the value type), but not both. A\n        synonym must conform to the following resrictions:\n        - It cannot contain carriage return, newline, or tab characters.\n        - It cannot consist of only whitespace characters.\n        - It must be no longer than 64 characters.\n        :param list[str] patterns: An array of patterns for the entity value. A value can\n        specify either synonyms or patterns (depending on the value type), but not both. A\n        pattern is a regular expression no longer than 512 characters. For more\n        information about how to specify a pattern, see the\n        [documentation](https://cloud.ibm.com/docs/services/assistant/entities.html#entities-create-dictionary-based).\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse\n        \"\"\"\n\n        if workspace_id is None:\n            raise ValueError('workspace_id must be provided')\n        if entity is None:\n            raise ValueError('entity must be provided')\n        if value is None:\n            raise ValueError('value must be provided')\n\n        headers = {}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('conversation', 'V1', 'create_value')\n        headers.update(sdk_headers)\n\n        params = {'version': self.version}\n\n        data = {\n            'value': value,\n            'metadata': metadata,\n            'type': value_type,\n            'synonyms': synonyms,\n            'patterns': patterns\n        }\n\n        url = '/v1/workspaces/{0}/entities/{1}/values'.format(\n            *self._encode_path_vars(workspace_id, entity))\n        response = self.request(\n            method='POST',\n            url=url,\n            headers=headers,\n            params=params,\n            json=data,\n            accept_json=True)\n        return response", "response": "Creates a new value for an entity."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nupdating entity value. Update an existing entity value with new or modified data. You must provide component objects defining the content of the updated entity value. This operation is limited to 1000 requests per 30 minutes. For more information, see **Rate limiting**. :param str workspace_id: Unique identifier of the workspace. :param str entity: The name of the entity. :param str value: The text of the entity value. :param str new_value: The text of the entity value. This string must conform to the following restrictions: - It cannot contain carriage return, newline, or tab characters. - It cannot consist of only whitespace characters. - It must be no longer than 64 characters. :param dict new_metadata: Any metadata related to the entity value. :param str new_value_type: Specifies the type of entity value. :param list[str] new_synonyms: An array of synonyms for the entity value. A value can specify either synonyms or patterns (depending on the value type), but not both. A synonym must conform to the following resrictions: - It cannot contain carriage return, newline, or tab characters. - It cannot consist of only whitespace characters. - It must be no longer than 64 characters. :param list[str] new_patterns: An array of patterns for the entity value. A value can specify either synonyms or patterns (depending on the value type), but not both. A pattern is a regular expression no longer than 512 characters. For more information about how to specify a pattern, see the [documentation](https://cloud.ibm.com/docs/services/assistant/entities.html#entities-create-dictionary-based). :param dict headers: A `dict` containing the request headers :return: A `DetailedResponse` containing the result, headers and HTTP status code. :rtype: DetailedResponse", "response": "def update_value(self,\n                     workspace_id,\n                     entity,\n                     value,\n                     new_value=None,\n                     new_metadata=None,\n                     new_value_type=None,\n                     new_synonyms=None,\n                     new_patterns=None,\n                     **kwargs):\n        \"\"\"\n        Update entity value.\n\n        Update an existing entity value with new or modified data. You must provide\n        component objects defining the content of the updated entity value.\n        This operation is limited to 1000 requests per 30 minutes. For more information,\n        see **Rate limiting**.\n\n        :param str workspace_id: Unique identifier of the workspace.\n        :param str entity: The name of the entity.\n        :param str value: The text of the entity value.\n        :param str new_value: The text of the entity value. This string must conform to\n        the following restrictions:\n        - It cannot contain carriage return, newline, or tab characters.\n        - It cannot consist of only whitespace characters.\n        - It must be no longer than 64 characters.\n        :param dict new_metadata: Any metadata related to the entity value.\n        :param str new_value_type: Specifies the type of entity value.\n        :param list[str] new_synonyms: An array of synonyms for the entity value. A value\n        can specify either synonyms or patterns (depending on the value type), but not\n        both. A synonym must conform to the following resrictions:\n        - It cannot contain carriage return, newline, or tab characters.\n        - It cannot consist of only whitespace characters.\n        - It must be no longer than 64 characters.\n        :param list[str] new_patterns: An array of patterns for the entity value. A value\n        can specify either synonyms or patterns (depending on the value type), but not\n        both. A pattern is a regular expression no longer than 512 characters. For more\n        information about how to specify a pattern, see the\n        [documentation](https://cloud.ibm.com/docs/services/assistant/entities.html#entities-create-dictionary-based).\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse\n        \"\"\"\n\n        if workspace_id is None:\n            raise ValueError('workspace_id must be provided')\n        if entity is None:\n            raise ValueError('entity must be provided')\n        if value is None:\n            raise ValueError('value must be provided')\n\n        headers = {}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('conversation', 'V1', 'update_value')\n        headers.update(sdk_headers)\n\n        params = {'version': self.version}\n\n        data = {\n            'value': new_value,\n            'metadata': new_metadata,\n            'type': new_value_type,\n            'synonyms': new_synonyms,\n            'patterns': new_patterns\n        }\n\n        url = '/v1/workspaces/{0}/entities/{1}/values/{2}'.format(\n            *self._encode_path_vars(workspace_id, entity, value))\n        response = self.request(\n            method='POST',\n            url=url,\n            headers=headers,\n            params=params,\n            json=data,\n            accept_json=True)\n        return response"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef create_dialog_node(self,\n                           workspace_id,\n                           dialog_node,\n                           description=None,\n                           conditions=None,\n                           parent=None,\n                           previous_sibling=None,\n                           output=None,\n                           context=None,\n                           metadata=None,\n                           next_step=None,\n                           title=None,\n                           node_type=None,\n                           event_name=None,\n                           variable=None,\n                           actions=None,\n                           digress_in=None,\n                           digress_out=None,\n                           digress_out_slots=None,\n                           user_label=None,\n                           **kwargs):\n        \"\"\"\n        Create dialog node.\n\n        Create a new dialog node.\n        This operation is limited to 500 requests per 30 minutes. For more information,\n        see **Rate limiting**.\n\n        :param str workspace_id: Unique identifier of the workspace.\n        :param str dialog_node: The dialog node ID. This string must conform to the\n        following restrictions:\n        - It can contain only Unicode alphanumeric, space, underscore, hyphen, and dot\n        characters.\n        - It must be no longer than 1024 characters.\n        :param str description: The description of the dialog node. This string cannot\n        contain carriage return, newline, or tab characters, and it must be no longer than\n        128 characters.\n        :param str conditions: The condition that will trigger the dialog node. This\n        string cannot contain carriage return, newline, or tab characters, and it must be\n        no longer than 2048 characters.\n        :param str parent: The ID of the parent dialog node. This property is omitted if\n        the dialog node has no parent.\n        :param str previous_sibling: The ID of the previous sibling dialog node. This\n        property is omitted if the dialog node has no previous sibling.\n        :param DialogNodeOutput output: The output of the dialog node. For more\n        information about how to specify dialog node output, see the\n        [documentation](https://cloud.ibm.com/docs/services/assistant/dialog-overview.html#dialog-overview-responses).\n        :param dict context: The context for the dialog node.\n        :param dict metadata: The metadata for the dialog node.\n        :param DialogNodeNextStep next_step: The next step to execute following this\n        dialog node.\n        :param str title: The alias used to identify the dialog node. This string must\n        conform to the following restrictions:\n        - It can contain only Unicode alphanumeric, space, underscore, hyphen, and dot\n        characters.\n        - It must be no longer than 64 characters.\n        :param str node_type: How the dialog node is processed.\n        :param str event_name: How an `event_handler` node is processed.\n        :param str variable: The location in the dialog context where output is stored.\n        :param list[DialogNodeAction] actions: An array of objects describing any actions\n        to be invoked by the dialog node.\n        :param str digress_in: Whether this top-level dialog node can be digressed into.\n        :param str digress_out: Whether this dialog node can be returned to after a\n        digression.\n        :param str digress_out_slots: Whether the user can digress to top-level nodes\n        while filling out slots.\n        :param str user_label: A label that can be displayed externally to describe the\n        purpose of the node to users. This string must be no longer than 512 characters.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse\n        \"\"\"\n\n        if workspace_id is None:\n            raise ValueError('workspace_id must be provided')\n        if dialog_node is None:\n            raise ValueError('dialog_node must be provided')\n        if output is not None:\n            output = self._convert_model(output, DialogNodeOutput)\n        if next_step is not None:\n            next_step = self._convert_model(next_step, DialogNodeNextStep)\n        if actions is not None:\n            actions = [\n                self._convert_model(x, DialogNodeAction) for x in actions\n            ]\n\n        headers = {}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('conversation', 'V1',\n                                      'create_dialog_node')\n        headers.update(sdk_headers)\n\n        params = {'version': self.version}\n\n        data = {\n            'dialog_node': dialog_node,\n            'description': description,\n            'conditions': conditions,\n            'parent': parent,\n            'previous_sibling': previous_sibling,\n            'output': output,\n            'context': context,\n            'metadata': metadata,\n            'next_step': next_step,\n            'title': title,\n            'type': node_type,\n            'event_name': event_name,\n            'variable': variable,\n            'actions': actions,\n            'digress_in': digress_in,\n            'digress_out': digress_out,\n            'digress_out_slots': digress_out_slots,\n            'user_label': user_label\n        }\n\n        url = '/v1/workspaces/{0}/dialog_nodes'.format(\n            *self._encode_path_vars(workspace_id))\n        response = self.request(\n            method='POST',\n            url=url,\n            headers=headers,\n            params=params,\n            json=data,\n            accept_json=True)\n        return response", "response": "This method creates a dialog node."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nupdating dialog node. Update an existing dialog node with new or modified data. This operation is limited to 500 requests per 30 minutes. For more information, see **Rate limiting**. :param str workspace_id: Unique identifier of the workspace. :param str dialog_node: The dialog node ID (for example, `get_order`). :param str new_dialog_node: The dialog node ID. This string must conform to the following restrictions: - It can contain only Unicode alphanumeric, space, underscore, hyphen, and dot characters. - It must be no longer than 1024 characters. :param str new_description: The description of the dialog node. This string cannot contain carriage return, newline, or tab characters, and it must be no longer than 128 characters. :param str new_conditions: The condition that will trigger the dialog node. This string cannot contain carriage return, newline, or tab characters, and it must be no longer than 2048 characters. :param str new_parent: The ID of the parent dialog node. This property is omitted if the dialog node has no parent. :param str new_previous_sibling: The ID of the previous sibling dialog node. This property is omitted if the dialog node has no previous sibling. :param DialogNodeOutput new_output: The output of the dialog node. For more information about how to specify dialog node output, see the [documentation](https://cloud.ibm.com/docs/services/assistant/dialog-overview.html#dialog-overview-responses). :param dict new_context: The context for the dialog node. :param dict new_metadata: The metadata for the dialog node. :param DialogNodeNextStep new_next_step: The next step to execute following this dialog node. :param str new_title: The alias used to identify the dialog node. This string must conform to the following restrictions: - It can contain only Unicode alphanumeric, space, underscore, hyphen, and dot characters. - It must be no longer than 64 characters. :param str new_node_type: How the dialog node is processed. :param str new_event_name: How an `event_handler` node is processed. :param str new_variable: The location in the dialog context where output is stored. :param list[DialogNodeAction] new_actions: An array of objects describing any actions to be invoked by the dialog node. :param str new_digress_in: Whether this top-level dialog node can be digressed into. :param str new_digress_out: Whether this dialog node can be returned to after a digression. :param str new_digress_out_slots: Whether the user can digress to top-level nodes while filling out slots. :param str new_user_label: A label that can be displayed externally to describe the purpose of the node to users. This string must be no longer than 512 characters. :param dict headers: A `dict` containing the request headers :return: A `DetailedResponse` containing the result, headers and HTTP status code. :rtype: DetailedResponse", "response": "def update_dialog_node(self,\n                           workspace_id,\n                           dialog_node,\n                           new_dialog_node=None,\n                           new_description=None,\n                           new_conditions=None,\n                           new_parent=None,\n                           new_previous_sibling=None,\n                           new_output=None,\n                           new_context=None,\n                           new_metadata=None,\n                           new_next_step=None,\n                           new_title=None,\n                           new_node_type=None,\n                           new_event_name=None,\n                           new_variable=None,\n                           new_actions=None,\n                           new_digress_in=None,\n                           new_digress_out=None,\n                           new_digress_out_slots=None,\n                           new_user_label=None,\n                           **kwargs):\n        \"\"\"\n        Update dialog node.\n\n        Update an existing dialog node with new or modified data.\n        This operation is limited to 500 requests per 30 minutes. For more information,\n        see **Rate limiting**.\n\n        :param str workspace_id: Unique identifier of the workspace.\n        :param str dialog_node: The dialog node ID (for example, `get_order`).\n        :param str new_dialog_node: The dialog node ID. This string must conform to the\n        following restrictions:\n        - It can contain only Unicode alphanumeric, space, underscore, hyphen, and dot\n        characters.\n        - It must be no longer than 1024 characters.\n        :param str new_description: The description of the dialog node. This string cannot\n        contain carriage return, newline, or tab characters, and it must be no longer than\n        128 characters.\n        :param str new_conditions: The condition that will trigger the dialog node. This\n        string cannot contain carriage return, newline, or tab characters, and it must be\n        no longer than 2048 characters.\n        :param str new_parent: The ID of the parent dialog node. This property is omitted\n        if the dialog node has no parent.\n        :param str new_previous_sibling: The ID of the previous sibling dialog node. This\n        property is omitted if the dialog node has no previous sibling.\n        :param DialogNodeOutput new_output: The output of the dialog node. For more\n        information about how to specify dialog node output, see the\n        [documentation](https://cloud.ibm.com/docs/services/assistant/dialog-overview.html#dialog-overview-responses).\n        :param dict new_context: The context for the dialog node.\n        :param dict new_metadata: The metadata for the dialog node.\n        :param DialogNodeNextStep new_next_step: The next step to execute following this\n        dialog node.\n        :param str new_title: The alias used to identify the dialog node. This string must\n        conform to the following restrictions:\n        - It can contain only Unicode alphanumeric, space, underscore, hyphen, and dot\n        characters.\n        - It must be no longer than 64 characters.\n        :param str new_node_type: How the dialog node is processed.\n        :param str new_event_name: How an `event_handler` node is processed.\n        :param str new_variable: The location in the dialog context where output is\n        stored.\n        :param list[DialogNodeAction] new_actions: An array of objects describing any\n        actions to be invoked by the dialog node.\n        :param str new_digress_in: Whether this top-level dialog node can be digressed\n        into.\n        :param str new_digress_out: Whether this dialog node can be returned to after a\n        digression.\n        :param str new_digress_out_slots: Whether the user can digress to top-level nodes\n        while filling out slots.\n        :param str new_user_label: A label that can be displayed externally to describe\n        the purpose of the node to users. This string must be no longer than 512\n        characters.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse\n        \"\"\"\n\n        if workspace_id is None:\n            raise ValueError('workspace_id must be provided')\n        if dialog_node is None:\n            raise ValueError('dialog_node must be provided')\n        if new_output is not None:\n            new_output = self._convert_model(new_output, DialogNodeOutput)\n        if new_next_step is not None:\n            new_next_step = self._convert_model(new_next_step,\n                                                DialogNodeNextStep)\n        if new_actions is not None:\n            new_actions = [\n                self._convert_model(x, DialogNodeAction) for x in new_actions\n            ]\n\n        headers = {}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('conversation', 'V1',\n                                      'update_dialog_node')\n        headers.update(sdk_headers)\n\n        params = {'version': self.version}\n\n        data = {\n            'dialog_node': new_dialog_node,\n            'description': new_description,\n            'conditions': new_conditions,\n            'parent': new_parent,\n            'previous_sibling': new_previous_sibling,\n            'output': new_output,\n            'context': new_context,\n            'metadata': new_metadata,\n            'next_step': new_next_step,\n            'title': new_title,\n            'type': new_node_type,\n            'event_name': new_event_name,\n            'variable': new_variable,\n            'actions': new_actions,\n            'digress_in': new_digress_in,\n            'digress_out': new_digress_out,\n            'digress_out_slots': new_digress_out_slots,\n            'user_label': new_user_label\n        }\n\n        url = '/v1/workspaces/{0}/dialog_nodes/{1}'.format(\n            *self._encode_path_vars(workspace_id, dialog_node))\n        response = self.request(\n            method='POST',\n            url=url,\n            headers=headers,\n            params=params,\n            json=data,\n            accept_json=True)\n        return response"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _from_dict(cls, _dict):\n        args = {}\n        xtra = _dict.copy()\n        if 'conversation_id' in _dict:\n            args['conversation_id'] = _dict.get('conversation_id')\n            del xtra['conversation_id']\n        if 'system' in _dict:\n            args['system'] = SystemResponse._from_dict(_dict.get('system'))\n            del xtra['system']\n        if 'metadata' in _dict:\n            args['metadata'] = MessageContextMetadata._from_dict(\n                _dict.get('metadata'))\n            del xtra['metadata']\n        args.update(xtra)\n        return cls(**args)", "response": "Initialize a Context object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self,\n                   'conversation_id') and self.conversation_id is not None:\n            _dict['conversation_id'] = self.conversation_id\n        if hasattr(self, 'system') and self.system is not None:\n            _dict['system'] = self.system._to_dict()\n        if hasattr(self, 'metadata') and self.metadata is not None:\n            _dict['metadata'] = self.metadata._to_dict()\n        if hasattr(self, '_additionalProperties'):\n            for _key in self._additionalProperties:\n                _value = getattr(self, _key, None)\n                if _value is not None:\n                    _dict[_key] = _value\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'text') and self.text is not None:\n            _dict['text'] = self.text\n        if hasattr(self, 'created') and self.created is not None:\n            _dict['created'] = datetime_to_string(self.created)\n        if hasattr(self, 'updated') and self.updated is not None:\n            _dict['updated'] = datetime_to_string(self.updated)\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self,\n                   'counterexamples') and self.counterexamples is not None:\n            _dict['counterexamples'] = [\n                x._to_dict() for x in self.counterexamples\n            ]\n        if hasattr(self, 'pagination') and self.pagination is not None:\n            _dict['pagination'] = self.pagination._to_dict()\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'entity') and self.entity is not None:\n            _dict['entity'] = self.entity\n        if hasattr(self, 'description') and self.description is not None:\n            _dict['description'] = self.description\n        if hasattr(self, 'metadata') and self.metadata is not None:\n            _dict['metadata'] = self.metadata\n        if hasattr(self, 'fuzzy_match') and self.fuzzy_match is not None:\n            _dict['fuzzy_match'] = self.fuzzy_match\n        if hasattr(self, 'created') and self.created is not None:\n            _dict['created'] = datetime_to_string(self.created)\n        if hasattr(self, 'updated') and self.updated is not None:\n            _dict['updated'] = datetime_to_string(self.updated)\n        if hasattr(self, 'values') and self.values is not None:\n            _dict['values'] = [x._to_dict() for x in self.values]\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'intent') and self.intent is not None:\n            _dict['intent'] = self.intent\n        if hasattr(self, 'description') and self.description is not None:\n            _dict['description'] = self.description\n        if hasattr(self, 'created') and self.created is not None:\n            _dict['created'] = datetime_to_string(self.created)\n        if hasattr(self, 'updated') and self.updated is not None:\n            _dict['updated'] = datetime_to_string(self.updated)\n        if hasattr(self, 'examples') and self.examples is not None:\n            _dict['examples'] = [x._to_dict() for x in self.examples]\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'value') and self.value is not None:\n            _dict['value'] = self.value\n        if hasattr(self, 'metadata') and self.metadata is not None:\n            _dict['metadata'] = self.metadata\n        if hasattr(self, 'value_type') and self.value_type is not None:\n            _dict['type'] = self.value_type\n        if hasattr(self, 'synonyms') and self.synonyms is not None:\n            _dict['synonyms'] = self.synonyms\n        if hasattr(self, 'patterns') and self.patterns is not None:\n            _dict['patterns'] = self.patterns\n        if hasattr(self, 'created') and self.created is not None:\n            _dict['created'] = datetime_to_string(self.created)\n        if hasattr(self, 'updated') and self.updated is not None:\n            _dict['updated'] = datetime_to_string(self.updated)\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ninitializing a DialogNode object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a DialogNode object from a json dictionary.\"\"\"\n        args = {}\n        if 'dialog_node' in _dict:\n            args['dialog_node'] = _dict.get('dialog_node')\n        else:\n            raise ValueError(\n                'Required property \\'dialog_node\\' not present in DialogNode JSON'\n            )\n        if 'description' in _dict:\n            args['description'] = _dict.get('description')\n        if 'conditions' in _dict:\n            args['conditions'] = _dict.get('conditions')\n        if 'parent' in _dict:\n            args['parent'] = _dict.get('parent')\n        if 'previous_sibling' in _dict:\n            args['previous_sibling'] = _dict.get('previous_sibling')\n        if 'output' in _dict:\n            args['output'] = DialogNodeOutput._from_dict(_dict.get('output'))\n        if 'context' in _dict:\n            args['context'] = _dict.get('context')\n        if 'metadata' in _dict:\n            args['metadata'] = _dict.get('metadata')\n        if 'next_step' in _dict:\n            args['next_step'] = DialogNodeNextStep._from_dict(\n                _dict.get('next_step'))\n        if 'title' in _dict:\n            args['title'] = _dict.get('title')\n        if 'type' in _dict or 'node_type' in _dict:\n            args['node_type'] = _dict.get('type') or _dict.get('node_type')\n        if 'event_name' in _dict:\n            args['event_name'] = _dict.get('event_name')\n        if 'variable' in _dict:\n            args['variable'] = _dict.get('variable')\n        if 'actions' in _dict:\n            args['actions'] = [\n                DialogNodeAction._from_dict(x) for x in (_dict.get('actions'))\n            ]\n        if 'digress_in' in _dict:\n            args['digress_in'] = _dict.get('digress_in')\n        if 'digress_out' in _dict:\n            args['digress_out'] = _dict.get('digress_out')\n        if 'digress_out_slots' in _dict:\n            args['digress_out_slots'] = _dict.get('digress_out_slots')\n        if 'user_label' in _dict:\n            args['user_label'] = _dict.get('user_label')\n        if 'disabled' in _dict:\n            args['disabled'] = _dict.get('disabled')\n        if 'created' in _dict:\n            args['created'] = string_to_datetime(_dict.get('created'))\n        if 'updated' in _dict:\n            args['updated'] = string_to_datetime(_dict.get('updated'))\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'dialog_node') and self.dialog_node is not None:\n            _dict['dialog_node'] = self.dialog_node\n        if hasattr(self, 'description') and self.description is not None:\n            _dict['description'] = self.description\n        if hasattr(self, 'conditions') and self.conditions is not None:\n            _dict['conditions'] = self.conditions\n        if hasattr(self, 'parent') and self.parent is not None:\n            _dict['parent'] = self.parent\n        if hasattr(self,\n                   'previous_sibling') and self.previous_sibling is not None:\n            _dict['previous_sibling'] = self.previous_sibling\n        if hasattr(self, 'output') and self.output is not None:\n            _dict['output'] = self.output._to_dict()\n        if hasattr(self, 'context') and self.context is not None:\n            _dict['context'] = self.context\n        if hasattr(self, 'metadata') and self.metadata is not None:\n            _dict['metadata'] = self.metadata\n        if hasattr(self, 'next_step') and self.next_step is not None:\n            _dict['next_step'] = self.next_step._to_dict()\n        if hasattr(self, 'title') and self.title is not None:\n            _dict['title'] = self.title\n        if hasattr(self, 'node_type') and self.node_type is not None:\n            _dict['type'] = self.node_type\n        if hasattr(self, 'event_name') and self.event_name is not None:\n            _dict['event_name'] = self.event_name\n        if hasattr(self, 'variable') and self.variable is not None:\n            _dict['variable'] = self.variable\n        if hasattr(self, 'actions') and self.actions is not None:\n            _dict['actions'] = [x._to_dict() for x in self.actions]\n        if hasattr(self, 'digress_in') and self.digress_in is not None:\n            _dict['digress_in'] = self.digress_in\n        if hasattr(self, 'digress_out') and self.digress_out is not None:\n            _dict['digress_out'] = self.digress_out\n        if hasattr(self,\n                   'digress_out_slots') and self.digress_out_slots is not None:\n            _dict['digress_out_slots'] = self.digress_out_slots\n        if hasattr(self, 'user_label') and self.user_label is not None:\n            _dict['user_label'] = self.user_label\n        if hasattr(self, 'disabled') and self.disabled is not None:\n            _dict['disabled'] = self.disabled\n        if hasattr(self, 'created') and self.created is not None:\n            _dict['created'] = datetime_to_string(self.created)\n        if hasattr(self, 'updated') and self.updated is not None:\n            _dict['updated'] = datetime_to_string(self.updated)\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'dialog_nodes') and self.dialog_nodes is not None:\n            _dict['dialog_nodes'] = [x._to_dict() for x in self.dialog_nodes]\n        if hasattr(self, 'pagination') and self.pagination is not None:\n            _dict['pagination'] = self.pagination._to_dict()\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'behavior') and self.behavior is not None:\n            _dict['behavior'] = self.behavior\n        if hasattr(self, 'dialog_node') and self.dialog_node is not None:\n            _dict['dialog_node'] = self.dialog_node\n        if hasattr(self, 'selector') and self.selector is not None:\n            _dict['selector'] = self.selector\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _from_dict(cls, _dict):\n        args = {}\n        xtra = _dict.copy()\n        if 'generic' in _dict:\n            args['generic'] = [\n                DialogNodeOutputGeneric._from_dict(x)\n                for x in (_dict.get('generic'))\n            ]\n            del xtra['generic']\n        if 'modifiers' in _dict:\n            args['modifiers'] = DialogNodeOutputModifiers._from_dict(\n                _dict.get('modifiers'))\n            del xtra['modifiers']\n        args.update(xtra)\n        return cls(**args)", "response": "Initialize a DialogNodeOutput object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'generic') and self.generic is not None:\n            _dict['generic'] = [x._to_dict() for x in self.generic]\n        if hasattr(self, 'modifiers') and self.modifiers is not None:\n            _dict['modifiers'] = self.modifiers._to_dict()\n        if hasattr(self, '_additionalProperties'):\n            for _key in self._additionalProperties:\n                _value = getattr(self, _key, None)\n                if _value is not None:\n                    _dict[_key] = _value\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'response_type') and self.response_type is not None:\n            _dict['response_type'] = self.response_type\n        if hasattr(self, 'values') and self.values is not None:\n            _dict['values'] = [x._to_dict() for x in self.values]\n        if hasattr(self,\n                   'selection_policy') and self.selection_policy is not None:\n            _dict['selection_policy'] = self.selection_policy\n        if hasattr(self, 'delimiter') and self.delimiter is not None:\n            _dict['delimiter'] = self.delimiter\n        if hasattr(self, 'time') and self.time is not None:\n            _dict['time'] = self.time\n        if hasattr(self, 'typing') and self.typing is not None:\n            _dict['typing'] = self.typing\n        if hasattr(self, 'source') and self.source is not None:\n            _dict['source'] = self.source\n        if hasattr(self, 'title') and self.title is not None:\n            _dict['title'] = self.title\n        if hasattr(self, 'description') and self.description is not None:\n            _dict['description'] = self.description\n        if hasattr(self, 'preference') and self.preference is not None:\n            _dict['preference'] = self.preference\n        if hasattr(self, 'options') and self.options is not None:\n            _dict['options'] = [x._to_dict() for x in self.options]\n        if hasattr(self, 'message_to_human_agent'\n                  ) and self.message_to_human_agent is not None:\n            _dict['message_to_human_agent'] = self.message_to_human_agent\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'overwrite') and self.overwrite is not None:\n            _dict['overwrite'] = self.overwrite\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'response_type' in _dict:\n            args['response_type'] = _dict.get('response_type')\n        else:\n            raise ValueError(\n                'Required property \\'response_type\\' not present in DialogRuntimeResponseGeneric JSON'\n            )\n        if 'text' in _dict:\n            args['text'] = _dict.get('text')\n        if 'time' in _dict:\n            args['time'] = _dict.get('time')\n        if 'typing' in _dict:\n            args['typing'] = _dict.get('typing')\n        if 'source' in _dict:\n            args['source'] = _dict.get('source')\n        if 'title' in _dict:\n            args['title'] = _dict.get('title')\n        if 'description' in _dict:\n            args['description'] = _dict.get('description')\n        if 'preference' in _dict:\n            args['preference'] = _dict.get('preference')\n        if 'options' in _dict:\n            args['options'] = [\n                DialogNodeOutputOptionsElement._from_dict(x)\n                for x in (_dict.get('options'))\n            ]\n        if 'message_to_human_agent' in _dict:\n            args['message_to_human_agent'] = _dict.get('message_to_human_agent')\n        if 'topic' in _dict:\n            args['topic'] = _dict.get('topic')\n        if 'dialog_node' in _dict:\n            args['dialog_node'] = _dict.get('dialog_node')\n        if 'suggestions' in _dict:\n            args['suggestions'] = [\n                DialogSuggestion._from_dict(x)\n                for x in (_dict.get('suggestions'))\n            ]\n        return cls(**args)", "response": "Initialize a DialogRuntimeResponseGeneric object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'response_type') and self.response_type is not None:\n            _dict['response_type'] = self.response_type\n        if hasattr(self, 'text') and self.text is not None:\n            _dict['text'] = self.text\n        if hasattr(self, 'time') and self.time is not None:\n            _dict['time'] = self.time\n        if hasattr(self, 'typing') and self.typing is not None:\n            _dict['typing'] = self.typing\n        if hasattr(self, 'source') and self.source is not None:\n            _dict['source'] = self.source\n        if hasattr(self, 'title') and self.title is not None:\n            _dict['title'] = self.title\n        if hasattr(self, 'description') and self.description is not None:\n            _dict['description'] = self.description\n        if hasattr(self, 'preference') and self.preference is not None:\n            _dict['preference'] = self.preference\n        if hasattr(self, 'options') and self.options is not None:\n            _dict['options'] = [x._to_dict() for x in self.options]\n        if hasattr(self, 'message_to_human_agent'\n                  ) and self.message_to_human_agent is not None:\n            _dict['message_to_human_agent'] = self.message_to_human_agent\n        if hasattr(self, 'topic') and self.topic is not None:\n            _dict['topic'] = self.topic\n        if hasattr(self, 'dialog_node') and self.dialog_node is not None:\n            _dict['dialog_node'] = self.dialog_node\n        if hasattr(self, 'suggestions') and self.suggestions is not None:\n            _dict['suggestions'] = [x._to_dict() for x in self.suggestions]\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'input' in _dict:\n            args['input'] = MessageInput._from_dict(_dict.get('input'))\n        if 'intents' in _dict:\n            args['intents'] = [\n                RuntimeIntent._from_dict(x) for x in (_dict.get('intents'))\n            ]\n        if 'entities' in _dict:\n            args['entities'] = [\n                RuntimeEntity._from_dict(x) for x in (_dict.get('entities'))\n            ]\n        return cls(**args)", "response": "Initialize a DialogSuggestionValue object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'input') and self.input is not None:\n            _dict['input'] = self.input._to_dict()\n        if hasattr(self, 'intents') and self.intents is not None:\n            _dict['intents'] = [x._to_dict() for x in self.intents]\n        if hasattr(self, 'entities') and self.entities is not None:\n            _dict['entities'] = [x._to_dict() for x in self.entities]\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'entities') and self.entities is not None:\n            _dict['entities'] = [x._to_dict() for x in self.entities]\n        if hasattr(self, 'pagination') and self.pagination is not None:\n            _dict['pagination'] = self.pagination._to_dict()\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'examples') and self.examples is not None:\n            _dict['examples'] = [x._to_dict() for x in self.examples]\n        if hasattr(self, 'pagination') and self.pagination is not None:\n            _dict['pagination'] = self.pagination._to_dict()\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'intents') and self.intents is not None:\n            _dict['intents'] = [x._to_dict() for x in self.intents]\n        if hasattr(self, 'pagination') and self.pagination is not None:\n            _dict['pagination'] = self.pagination._to_dict()\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'request' in _dict:\n            args['request'] = MessageRequest._from_dict(_dict.get('request'))\n        else:\n            raise ValueError(\n                'Required property \\'request\\' not present in Log JSON')\n        if 'response' in _dict:\n            args['response'] = MessageResponse._from_dict(_dict.get('response'))\n        else:\n            raise ValueError(\n                'Required property \\'response\\' not present in Log JSON')\n        if 'log_id' in _dict:\n            args['log_id'] = _dict.get('log_id')\n        else:\n            raise ValueError(\n                'Required property \\'log_id\\' not present in Log JSON')\n        if 'request_timestamp' in _dict:\n            args['request_timestamp'] = _dict.get('request_timestamp')\n        else:\n            raise ValueError(\n                'Required property \\'request_timestamp\\' not present in Log JSON'\n            )\n        if 'response_timestamp' in _dict:\n            args['response_timestamp'] = _dict.get('response_timestamp')\n        else:\n            raise ValueError(\n                'Required property \\'response_timestamp\\' not present in Log JSON'\n            )\n        if 'workspace_id' in _dict:\n            args['workspace_id'] = _dict.get('workspace_id')\n        else:\n            raise ValueError(\n                'Required property \\'workspace_id\\' not present in Log JSON')\n        if 'language' in _dict:\n            args['language'] = _dict.get('language')\n        else:\n            raise ValueError(\n                'Required property \\'language\\' not present in Log JSON')\n        return cls(**args)", "response": "Initialize a Log object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'request') and self.request is not None:\n            _dict['request'] = self.request._to_dict()\n        if hasattr(self, 'response') and self.response is not None:\n            _dict['response'] = self.response._to_dict()\n        if hasattr(self, 'log_id') and self.log_id is not None:\n            _dict['log_id'] = self.log_id\n        if hasattr(self,\n                   'request_timestamp') and self.request_timestamp is not None:\n            _dict['request_timestamp'] = self.request_timestamp\n        if hasattr(\n                self,\n                'response_timestamp') and self.response_timestamp is not None:\n            _dict['response_timestamp'] = self.response_timestamp\n        if hasattr(self, 'workspace_id') and self.workspace_id is not None:\n            _dict['workspace_id'] = self.workspace_id\n        if hasattr(self, 'language') and self.language is not None:\n            _dict['language'] = self.language\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ninitializing a LogCollection object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a LogCollection object from a json dictionary.\"\"\"\n        args = {}\n        if 'logs' in _dict:\n            args['logs'] = [Log._from_dict(x) for x in (_dict.get('logs'))]\n        else:\n            raise ValueError(\n                'Required property \\'logs\\' not present in LogCollection JSON')\n        if 'pagination' in _dict:\n            args['pagination'] = LogPagination._from_dict(\n                _dict.get('pagination'))\n        else:\n            raise ValueError(\n                'Required property \\'pagination\\' not present in LogCollection JSON'\n            )\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'logs') and self.logs is not None:\n            _dict['logs'] = [x._to_dict() for x in self.logs]\n        if hasattr(self, 'pagination') and self.pagination is not None:\n            _dict['pagination'] = self.pagination._to_dict()\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'level') and self.level is not None:\n            _dict['level'] = self.level\n        if hasattr(self, 'msg') and self.msg is not None:\n            _dict['msg'] = self.msg\n        if hasattr(self, '_additionalProperties'):\n            for _key in self._additionalProperties:\n                _value = getattr(self, _key, None)\n                if _value is not None:\n                    _dict[_key] = _value\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'next_url') and self.next_url is not None:\n            _dict['next_url'] = self.next_url\n        if hasattr(self, 'matched') and self.matched is not None:\n            _dict['matched'] = self.matched\n        if hasattr(self, 'next_cursor') and self.next_cursor is not None:\n            _dict['next_cursor'] = self.next_cursor\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'entity') and self.entity is not None:\n            _dict['entity'] = self.entity\n        if hasattr(self, 'location') and self.location is not None:\n            _dict['location'] = self.location\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'deployment') and self.deployment is not None:\n            _dict['deployment'] = self.deployment\n        if hasattr(self, 'user_id') and self.user_id is not None:\n            _dict['user_id'] = self.user_id\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'text') and self.text is not None:\n            _dict['text'] = self.text\n        if hasattr(self, '_additionalProperties'):\n            for _key in self._additionalProperties:\n                _value = getattr(self, _key, None)\n                if _value is not None:\n                    _dict[_key] = _value\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ninitializing a MessageRequest object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a MessageRequest object from a json dictionary.\"\"\"\n        args = {}\n        if 'input' in _dict:\n            args['input'] = MessageInput._from_dict(_dict.get('input'))\n        if 'intents' in _dict:\n            args['intents'] = [\n                RuntimeIntent._from_dict(x) for x in (_dict.get('intents'))\n            ]\n        if 'entities' in _dict:\n            args['entities'] = [\n                RuntimeEntity._from_dict(x) for x in (_dict.get('entities'))\n            ]\n        if 'alternate_intents' in _dict:\n            args['alternate_intents'] = _dict.get('alternate_intents')\n        if 'context' in _dict:\n            args['context'] = Context._from_dict(_dict.get('context'))\n        if 'output' in _dict:\n            args['output'] = OutputData._from_dict(_dict.get('output'))\n        if 'actions' in _dict:\n            args['actions'] = [\n                DialogNodeAction._from_dict(x) for x in (_dict.get('actions'))\n            ]\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'input') and self.input is not None:\n            _dict['input'] = self.input._to_dict()\n        if hasattr(self, 'intents') and self.intents is not None:\n            _dict['intents'] = [x._to_dict() for x in self.intents]\n        if hasattr(self, 'entities') and self.entities is not None:\n            _dict['entities'] = [x._to_dict() for x in self.entities]\n        if hasattr(self,\n                   'alternate_intents') and self.alternate_intents is not None:\n            _dict['alternate_intents'] = self.alternate_intents\n        if hasattr(self, 'context') and self.context is not None:\n            _dict['context'] = self.context._to_dict()\n        if hasattr(self, 'output') and self.output is not None:\n            _dict['output'] = self.output._to_dict()\n        if hasattr(self, 'actions') and self.actions is not None:\n            _dict['actions'] = [x._to_dict() for x in self.actions]\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _from_dict(cls, _dict):\n        args = {}\n        xtra = _dict.copy()\n        if 'log_messages' in _dict:\n            args['log_messages'] = [\n                LogMessage._from_dict(x) for x in (_dict.get('log_messages'))\n            ]\n            del xtra['log_messages']\n        else:\n            raise ValueError(\n                'Required property \\'log_messages\\' not present in OutputData JSON'\n            )\n        if 'text' in _dict:\n            args['text'] = _dict.get('text')\n            del xtra['text']\n        else:\n            raise ValueError(\n                'Required property \\'text\\' not present in OutputData JSON')\n        if 'generic' in _dict:\n            args['generic'] = [\n                DialogRuntimeResponseGeneric._from_dict(x)\n                for x in (_dict.get('generic'))\n            ]\n            del xtra['generic']\n        if 'nodes_visited' in _dict:\n            args['nodes_visited'] = _dict.get('nodes_visited')\n            del xtra['nodes_visited']\n        if 'nodes_visited_details' in _dict:\n            args['nodes_visited_details'] = [\n                DialogNodeVisitedDetails._from_dict(x)\n                for x in (_dict.get('nodes_visited_details'))\n            ]\n            del xtra['nodes_visited_details']\n        args.update(xtra)\n        return cls(**args)", "response": "Initialize a OutputData object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'log_messages') and self.log_messages is not None:\n            _dict['log_messages'] = [x._to_dict() for x in self.log_messages]\n        if hasattr(self, 'text') and self.text is not None:\n            _dict['text'] = self.text\n        if hasattr(self, 'generic') and self.generic is not None:\n            _dict['generic'] = [x._to_dict() for x in self.generic]\n        if hasattr(self, 'nodes_visited') and self.nodes_visited is not None:\n            _dict['nodes_visited'] = self.nodes_visited\n        if hasattr(self, 'nodes_visited_details'\n                  ) and self.nodes_visited_details is not None:\n            _dict['nodes_visited_details'] = [\n                x._to_dict() for x in self.nodes_visited_details\n            ]\n        if hasattr(self, '_additionalProperties'):\n            for _key in self._additionalProperties:\n                _value = getattr(self, _key, None)\n                if _value is not None:\n                    _dict[_key] = _value\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _from_dict(cls, _dict):\n        args = {}\n        xtra = _dict.copy()\n        if 'entity' in _dict:\n            args['entity'] = _dict.get('entity')\n            del xtra['entity']\n        else:\n            raise ValueError(\n                'Required property \\'entity\\' not present in RuntimeEntity JSON'\n            )\n        if 'location' in _dict:\n            args['location'] = _dict.get('location')\n            del xtra['location']\n        else:\n            raise ValueError(\n                'Required property \\'location\\' not present in RuntimeEntity JSON'\n            )\n        if 'value' in _dict:\n            args['value'] = _dict.get('value')\n            del xtra['value']\n        else:\n            raise ValueError(\n                'Required property \\'value\\' not present in RuntimeEntity JSON')\n        if 'confidence' in _dict:\n            args['confidence'] = _dict.get('confidence')\n            del xtra['confidence']\n        if 'metadata' in _dict:\n            args['metadata'] = _dict.get('metadata')\n            del xtra['metadata']\n        if 'groups' in _dict:\n            args['groups'] = [\n                CaptureGroup._from_dict(x) for x in (_dict.get('groups'))\n            ]\n            del xtra['groups']\n        args.update(xtra)\n        return cls(**args)", "response": "Initialize a RuntimeEntity object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'entity') and self.entity is not None:\n            _dict['entity'] = self.entity\n        if hasattr(self, 'location') and self.location is not None:\n            _dict['location'] = self.location\n        if hasattr(self, 'value') and self.value is not None:\n            _dict['value'] = self.value\n        if hasattr(self, 'confidence') and self.confidence is not None:\n            _dict['confidence'] = self.confidence\n        if hasattr(self, 'metadata') and self.metadata is not None:\n            _dict['metadata'] = self.metadata\n        if hasattr(self, 'groups') and self.groups is not None:\n            _dict['groups'] = [x._to_dict() for x in self.groups]\n        if hasattr(self, '_additionalProperties'):\n            for _key in self._additionalProperties:\n                _value = getattr(self, _key, None)\n                if _value is not None:\n                    _dict[_key] = _value\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ninitializes a RuntimeIntent object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a RuntimeIntent object from a json dictionary.\"\"\"\n        args = {}\n        xtra = _dict.copy()\n        if 'intent' in _dict:\n            args['intent'] = _dict.get('intent')\n            del xtra['intent']\n        else:\n            raise ValueError(\n                'Required property \\'intent\\' not present in RuntimeIntent JSON'\n            )\n        if 'confidence' in _dict:\n            args['confidence'] = _dict.get('confidence')\n            del xtra['confidence']\n        else:\n            raise ValueError(\n                'Required property \\'confidence\\' not present in RuntimeIntent JSON'\n            )\n        args.update(xtra)\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'intent') and self.intent is not None:\n            _dict['intent'] = self.intent\n        if hasattr(self, 'confidence') and self.confidence is not None:\n            _dict['confidence'] = self.confidence\n        if hasattr(self, '_additionalProperties'):\n            for _key in self._additionalProperties:\n                _value = getattr(self, _key, None)\n                if _value is not None:\n                    _dict[_key] = _value\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'synonym') and self.synonym is not None:\n            _dict['synonym'] = self.synonym\n        if hasattr(self, 'created') and self.created is not None:\n            _dict['created'] = datetime_to_string(self.created)\n        if hasattr(self, 'updated') and self.updated is not None:\n            _dict['updated'] = datetime_to_string(self.updated)\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'synonyms') and self.synonyms is not None:\n            _dict['synonyms'] = [x._to_dict() for x in self.synonyms]\n        if hasattr(self, 'pagination') and self.pagination is not None:\n            _dict['pagination'] = self.pagination._to_dict()\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'values' in _dict:\n            args['values'] = [\n                Value._from_dict(x) for x in (_dict.get('values'))\n            ]\n        else:\n            raise ValueError(\n                'Required property \\'values\\' not present in ValueCollection JSON'\n            )\n        if 'pagination' in _dict:\n            args['pagination'] = Pagination._from_dict(_dict.get('pagination'))\n        else:\n            raise ValueError(\n                'Required property \\'pagination\\' not present in ValueCollection JSON'\n            )\n        return cls(**args)", "response": "Initialize a ValueCollection object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'values') and self.values is not None:\n            _dict['values'] = [x._to_dict() for x in self.values]\n        if hasattr(self, 'pagination') and self.pagination is not None:\n            _dict['pagination'] = self.pagination._to_dict()\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'name' in _dict:\n            args['name'] = _dict.get('name')\n        else:\n            raise ValueError(\n                'Required property \\'name\\' not present in Workspace JSON')\n        if 'description' in _dict:\n            args['description'] = _dict.get('description')\n        if 'language' in _dict:\n            args['language'] = _dict.get('language')\n        else:\n            raise ValueError(\n                'Required property \\'language\\' not present in Workspace JSON')\n        if 'metadata' in _dict:\n            args['metadata'] = _dict.get('metadata')\n        if 'learning_opt_out' in _dict:\n            args['learning_opt_out'] = _dict.get('learning_opt_out')\n        else:\n            raise ValueError(\n                'Required property \\'learning_opt_out\\' not present in Workspace JSON'\n            )\n        if 'system_settings' in _dict:\n            args['system_settings'] = WorkspaceSystemSettings._from_dict(\n                _dict.get('system_settings'))\n        if 'workspace_id' in _dict:\n            args['workspace_id'] = _dict.get('workspace_id')\n        else:\n            raise ValueError(\n                'Required property \\'workspace_id\\' not present in Workspace JSON'\n            )\n        if 'status' in _dict:\n            args['status'] = _dict.get('status')\n        if 'created' in _dict:\n            args['created'] = string_to_datetime(_dict.get('created'))\n        if 'updated' in _dict:\n            args['updated'] = string_to_datetime(_dict.get('updated'))\n        if 'intents' in _dict:\n            args['intents'] = [\n                Intent._from_dict(x) for x in (_dict.get('intents'))\n            ]\n        if 'entities' in _dict:\n            args['entities'] = [\n                Entity._from_dict(x) for x in (_dict.get('entities'))\n            ]\n        if 'dialog_nodes' in _dict:\n            args['dialog_nodes'] = [\n                DialogNode._from_dict(x) for x in (_dict.get('dialog_nodes'))\n            ]\n        if 'counterexamples' in _dict:\n            args['counterexamples'] = [\n                Counterexample._from_dict(x)\n                for x in (_dict.get('counterexamples'))\n            ]\n        return cls(**args)", "response": "Initialize a Workspace object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'name') and self.name is not None:\n            _dict['name'] = self.name\n        if hasattr(self, 'description') and self.description is not None:\n            _dict['description'] = self.description\n        if hasattr(self, 'language') and self.language is not None:\n            _dict['language'] = self.language\n        if hasattr(self, 'metadata') and self.metadata is not None:\n            _dict['metadata'] = self.metadata\n        if hasattr(self,\n                   'learning_opt_out') and self.learning_opt_out is not None:\n            _dict['learning_opt_out'] = self.learning_opt_out\n        if hasattr(self,\n                   'system_settings') and self.system_settings is not None:\n            _dict['system_settings'] = self.system_settings._to_dict()\n        if hasattr(self, 'workspace_id') and self.workspace_id is not None:\n            _dict['workspace_id'] = self.workspace_id\n        if hasattr(self, 'status') and self.status is not None:\n            _dict['status'] = self.status\n        if hasattr(self, 'created') and self.created is not None:\n            _dict['created'] = datetime_to_string(self.created)\n        if hasattr(self, 'updated') and self.updated is not None:\n            _dict['updated'] = datetime_to_string(self.updated)\n        if hasattr(self, 'intents') and self.intents is not None:\n            _dict['intents'] = [x._to_dict() for x in self.intents]\n        if hasattr(self, 'entities') and self.entities is not None:\n            _dict['entities'] = [x._to_dict() for x in self.entities]\n        if hasattr(self, 'dialog_nodes') and self.dialog_nodes is not None:\n            _dict['dialog_nodes'] = [x._to_dict() for x in self.dialog_nodes]\n        if hasattr(self,\n                   'counterexamples') and self.counterexamples is not None:\n            _dict['counterexamples'] = [\n                x._to_dict() for x in self.counterexamples\n            ]\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'workspaces') and self.workspaces is not None:\n            _dict['workspaces'] = [x._to_dict() for x in self.workspaces]\n        if hasattr(self, 'pagination') and self.pagination is not None:\n            _dict['pagination'] = self.pagination._to_dict()\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ninitialize a WorkspaceSystemSettings object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a WorkspaceSystemSettings object from a json dictionary.\"\"\"\n        args = {}\n        if 'tooling' in _dict:\n            args['tooling'] = WorkspaceSystemSettingsTooling._from_dict(\n                _dict.get('tooling'))\n        if 'disambiguation' in _dict:\n            args[\n                'disambiguation'] = WorkspaceSystemSettingsDisambiguation._from_dict(\n                    _dict.get('disambiguation'))\n        if 'human_agent_assist' in _dict:\n            args['human_agent_assist'] = _dict.get('human_agent_assist')\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'tooling') and self.tooling is not None:\n            _dict['tooling'] = self.tooling._to_dict()\n        if hasattr(self, 'disambiguation') and self.disambiguation is not None:\n            _dict['disambiguation'] = self.disambiguation._to_dict()\n        if hasattr(\n                self,\n                'human_agent_assist') and self.human_agent_assist is not None:\n            _dict['human_agent_assist'] = self.human_agent_assist\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'prompt') and self.prompt is not None:\n            _dict['prompt'] = self.prompt\n        if hasattr(self, 'none_of_the_above_prompt'\n                  ) and self.none_of_the_above_prompt is not None:\n            _dict['none_of_the_above_prompt'] = self.none_of_the_above_prompt\n        if hasattr(self, 'enabled') and self.enabled is not None:\n            _dict['enabled'] = self.enabled\n        if hasattr(self, 'sensitivity') and self.sensitivity is not None:\n            _dict['sensitivity'] = self.sensitivity\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'store_generic_responses'\n                  ) and self.store_generic_responses is not None:\n            _dict['store_generic_responses'] = self.store_generic_responses\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef classify(self, classifier_id, text, **kwargs):\n\n        if classifier_id is None:\n            raise ValueError('classifier_id must be provided')\n        if text is None:\n            raise ValueError('text must be provided')\n\n        headers = {}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('natural_language_classifier', 'V1',\n                                      'classify')\n        headers.update(sdk_headers)\n\n        data = {'text': text}\n\n        url = '/v1/classifiers/{0}/classify'.format(\n            *self._encode_path_vars(classifier_id))\n        response = self.request(\n            method='POST',\n            url=url,\n            headers=headers,\n            json=data,\n            accept_json=True)\n        return response", "response": "Classify a phrase.\n\n        Returns label information for the input. The status must be `Available` before you\n        can use the classifier to classify text.\n\n        :param str classifier_id: Classifier ID to use.\n        :param str text: The submitted phrase. The maximum length is 2048 characters.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nclassify multiple phrases. Returns label information for multiple phrases. The status must be `Available` before you can use the classifier to classify text. Note that classifying Japanese texts is a beta feature. :param str classifier_id: Classifier ID to use. :param list[ClassifyInput] collection: The submitted phrases. :param dict headers: A `dict` containing the request headers :return: A `DetailedResponse` containing the result, headers and HTTP status code. :rtype: DetailedResponse", "response": "def classify_collection(self, classifier_id, collection, **kwargs):\n        \"\"\"\n        Classify multiple phrases.\n\n        Returns label information for multiple phrases. The status must be `Available`\n        before you can use the classifier to classify text.\n        Note that classifying Japanese texts is a beta feature.\n\n        :param str classifier_id: Classifier ID to use.\n        :param list[ClassifyInput] collection: The submitted phrases.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse\n        \"\"\"\n\n        if classifier_id is None:\n            raise ValueError('classifier_id must be provided')\n        if collection is None:\n            raise ValueError('collection must be provided')\n        collection = [self._convert_model(x, ClassifyInput) for x in collection]\n\n        headers = {}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('natural_language_classifier', 'V1',\n                                      'classify_collection')\n        headers.update(sdk_headers)\n\n        data = {'collection': collection}\n\n        url = '/v1/classifiers/{0}/classify_collection'.format(\n            *self._encode_path_vars(classifier_id))\n        response = self.request(\n            method='POST',\n            url=url,\n            headers=headers,\n            json=data,\n            accept_json=True)\n        return response"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef create_classifier(self, metadata, training_data, **kwargs):\n\n        if metadata is None:\n            raise ValueError('metadata must be provided')\n        if training_data is None:\n            raise ValueError('training_data must be provided')\n\n        headers = {}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('natural_language_classifier', 'V1',\n                                      'create_classifier')\n        headers.update(sdk_headers)\n\n        form_data = {}\n        form_data['training_metadata'] = (None, metadata, 'application/json')\n        form_data['training_data'] = (None, training_data, 'text/csv')\n\n        url = '/v1/classifiers'\n        response = self.request(\n            method='POST',\n            url=url,\n            headers=headers,\n            files=form_data,\n            accept_json=True)\n        return response", "response": "Create a new classifier."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'classifier_id' in _dict:\n            args['classifier_id'] = _dict.get('classifier_id')\n        if 'url' in _dict:\n            args['url'] = _dict.get('url')\n        if 'text' in _dict:\n            args['text'] = _dict.get('text')\n        if 'top_class' in _dict:\n            args['top_class'] = _dict.get('top_class')\n        if 'classes' in _dict:\n            args['classes'] = [\n                ClassifiedClass._from_dict(x) for x in (_dict.get('classes'))\n            ]\n        return cls(**args)", "response": "Initialize a Classification object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'classifier_id') and self.classifier_id is not None:\n            _dict['classifier_id'] = self.classifier_id\n        if hasattr(self, 'url') and self.url is not None:\n            _dict['url'] = self.url\n        if hasattr(self, 'text') and self.text is not None:\n            _dict['text'] = self.text\n        if hasattr(self, 'top_class') and self.top_class is not None:\n            _dict['top_class'] = self.top_class\n        if hasattr(self, 'classes') and self.classes is not None:\n            _dict['classes'] = [x._to_dict() for x in self.classes]\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'classifier_id' in _dict:\n            args['classifier_id'] = _dict.get('classifier_id')\n        if 'url' in _dict:\n            args['url'] = _dict.get('url')\n        if 'collection' in _dict:\n            args['collection'] = [\n                CollectionItem._from_dict(x) for x in (_dict.get('collection'))\n            ]\n        return cls(**args)", "response": "Initialize a ClassificationCollection object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'classifier_id') and self.classifier_id is not None:\n            _dict['classifier_id'] = self.classifier_id\n        if hasattr(self, 'url') and self.url is not None:\n            _dict['url'] = self.url\n        if hasattr(self, 'collection') and self.collection is not None:\n            _dict['collection'] = [x._to_dict() for x in self.collection]\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'confidence') and self.confidence is not None:\n            _dict['confidence'] = self.confidence\n        if hasattr(self, 'class_name') and self.class_name is not None:\n            _dict['class_name'] = self.class_name\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'name') and self.name is not None:\n            _dict['name'] = self.name\n        if hasattr(self, 'url') and self.url is not None:\n            _dict['url'] = self.url\n        if hasattr(self, 'status') and self.status is not None:\n            _dict['status'] = self.status\n        if hasattr(self, 'classifier_id') and self.classifier_id is not None:\n            _dict['classifier_id'] = self.classifier_id\n        if hasattr(self, 'created') and self.created is not None:\n            _dict['created'] = datetime_to_string(self.created)\n        if hasattr(\n                self,\n                'status_description') and self.status_description is not None:\n            _dict['status_description'] = self.status_description\n        if hasattr(self, 'language') and self.language is not None:\n            _dict['language'] = self.language\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ninitializes a ClassifierList object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a ClassifierList object from a json dictionary.\"\"\"\n        args = {}\n        if 'classifiers' in _dict:\n            args['classifiers'] = [\n                Classifier._from_dict(x) for x in (_dict.get('classifiers'))\n            ]\n        else:\n            raise ValueError(\n                'Required property \\'classifiers\\' not present in ClassifierList JSON'\n            )\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'text') and self.text is not None:\n            _dict['text'] = self.text\n        if hasattr(self, 'top_class') and self.top_class is not None:\n            _dict['top_class'] = self.top_class\n        if hasattr(self, 'classes') and self.classes is not None:\n            _dict['classes'] = [x._to_dict() for x in self.classes]\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef translate(self, text, model_id=None, source=None, target=None,\n                  **kwargs):\n        \"\"\"\n        Translate.\n\n        Translates the input text from the source language to the target language.\n\n        :param list[str] text: Input text in UTF-8 encoding. Multiple entries will result\n        in multiple translations in the response.\n        :param str model_id: A globally unique string that identifies the underlying model\n        that is used for translation.\n        :param str source: Translation source language code.\n        :param str target: Translation target language code.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse\n        \"\"\"\n\n        if text is None:\n            raise ValueError('text must be provided')\n\n        headers = {}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('language_translator', 'V3', 'translate')\n        headers.update(sdk_headers)\n\n        params = {'version': self.version}\n\n        data = {\n            'text': text,\n            'model_id': model_id,\n            'source': source,\n            'target': target\n        }\n\n        url = '/v3/translate'\n        response = self.request(\n            method='POST',\n            url=url,\n            headers=headers,\n            params=params,\n            json=data,\n            accept_json=True)\n        return response", "response": "Translate a text from one language to another."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nidentify language. Identifies the language of the input text. :param str text: Input text in UTF-8 format. :param dict headers: A `dict` containing the request headers :return: A `DetailedResponse` containing the result, headers and HTTP status code. :rtype: DetailedResponse", "response": "def identify(self, text, **kwargs):\n        \"\"\"\n        Identify language.\n\n        Identifies the language of the input text.\n\n        :param str text: Input text in UTF-8 format.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse\n        \"\"\"\n\n        if text is None:\n            raise ValueError('text must be provided')\n\n        headers = {}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('language_translator', 'V3', 'identify')\n        headers.update(sdk_headers)\n\n        params = {'version': self.version}\n\n        data = text\n        headers['content-type'] = 'text/plain'\n\n        url = '/v3/identify'\n        response = self.request(\n            method='POST',\n            url=url,\n            headers=headers,\n            params=params,\n            data=data,\n            accept_json=True)\n        return response"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef create_model(self,\n                     base_model_id,\n                     forced_glossary=None,\n                     parallel_corpus=None,\n                     name=None,\n                     **kwargs):\n        \"\"\"\n        Create model.\n\n        Uploads Translation Memory eXchange (TMX) files to customize a translation model.\n        You can either customize a model with a forced glossary or with a corpus that\n        contains parallel sentences. To create a model that is customized with a parallel\n        corpus <b>and</b> a forced glossary, proceed in two steps: customize with a\n        parallel corpus first and then customize the resulting model with a glossary.\n        Depending on the type of customization and the size of the uploaded corpora,\n        training can range from minutes for a glossary to several hours for a large\n        parallel corpus. You can upload a single forced glossary file and this file must\n        be less than <b>10 MB</b>. You can upload multiple parallel corpora tmx files. The\n        cumulative file size of all uploaded files is limited to <b>250 MB</b>. To\n        successfully train with a parallel corpus you must have at least <b>5,000 parallel\n        sentences</b> in your corpus.\n        You can have a <b>maxium of 10 custom models per language pair</b>.\n\n        :param str base_model_id: The model ID of the model to use as the base for\n        customization. To see available models, use the `List models` method. Usually all\n        IBM provided models are customizable. In addition, all your models that have been\n        created via parallel corpus customization, can be further customized with a forced\n        glossary.\n        :param file forced_glossary: A TMX file with your customizations. The\n        customizations in the file completely overwrite the domain translaton data,\n        including high frequency or high confidence phrase translations. You can upload\n        only one glossary with a file size less than 10 MB per call. A forced glossary\n        should contain single words or short phrases.\n        :param file parallel_corpus: A TMX file with parallel sentences for source and\n        target language. You can upload multiple parallel_corpus files in one request. All\n        uploaded parallel_corpus files combined, your parallel corpus must contain at\n        least 5,000 parallel sentences to train successfully.\n        :param str name: An optional model name that you can use to identify the model.\n        Valid characters are letters, numbers, dashes, underscores, spaces and\n        apostrophes. The maximum length is 32 characters.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse\n        \"\"\"\n\n        if base_model_id is None:\n            raise ValueError('base_model_id must be provided')\n\n        headers = {}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('language_translator', 'V3',\n                                      'create_model')\n        headers.update(sdk_headers)\n\n        params = {\n            'version': self.version,\n            'base_model_id': base_model_id,\n            'name': name\n        }\n\n        form_data = {}\n        if forced_glossary:\n            form_data['forced_glossary'] = (None, forced_glossary,\n                                            'application/octet-stream')\n        if parallel_corpus:\n            form_data['parallel_corpus'] = (None, parallel_corpus,\n                                            'application/octet-stream')\n\n        url = '/v3/models'\n        response = self.request(\n            method='POST',\n            url=url,\n            headers=headers,\n            params=params,\n            files=form_data,\n            accept_json=True)\n        return response", "response": "Create a translation model for the given language pair."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'status') and self.status is not None:\n            _dict['status'] = self.status\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'language') and self.language is not None:\n            _dict['language'] = self.language\n        if hasattr(self, 'name') and self.name is not None:\n            _dict['name'] = self.name\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'languages' in _dict:\n            args['languages'] = [\n                IdentifiableLanguage._from_dict(x)\n                for x in (_dict.get('languages'))\n            ]\n        else:\n            raise ValueError(\n                'Required property \\'languages\\' not present in IdentifiableLanguages JSON'\n            )\n        return cls(**args)", "response": "Initialize a IdentifiableLanguages object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'languages') and self.languages is not None:\n            _dict['languages'] = [x._to_dict() for x in self.languages]\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'language') and self.language is not None:\n            _dict['language'] = self.language\n        if hasattr(self, 'confidence') and self.confidence is not None:\n            _dict['confidence'] = self.confidence\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'languages' in _dict:\n            args['languages'] = [\n                IdentifiedLanguage._from_dict(x)\n                for x in (_dict.get('languages'))\n            ]\n        else:\n            raise ValueError(\n                'Required property \\'languages\\' not present in IdentifiedLanguages JSON'\n            )\n        return cls(**args)", "response": "Initialize a IdentifiedLanguages object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(\n                self,\n                'translation_output') and self.translation_output is not None:\n            _dict['translation'] = self.translation_output\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'model_id') and self.model_id is not None:\n            _dict['model_id'] = self.model_id\n        if hasattr(self, 'name') and self.name is not None:\n            _dict['name'] = self.name\n        if hasattr(self, 'source') and self.source is not None:\n            _dict['source'] = self.source\n        if hasattr(self, 'target') and self.target is not None:\n            _dict['target'] = self.target\n        if hasattr(self, 'base_model_id') and self.base_model_id is not None:\n            _dict['base_model_id'] = self.base_model_id\n        if hasattr(self, 'domain') and self.domain is not None:\n            _dict['domain'] = self.domain\n        if hasattr(self, 'customizable') and self.customizable is not None:\n            _dict['customizable'] = self.customizable\n        if hasattr(self, 'default_model') and self.default_model is not None:\n            _dict['default_model'] = self.default_model\n        if hasattr(self, 'owner') and self.owner is not None:\n            _dict['owner'] = self.owner\n        if hasattr(self, 'status') and self.status is not None:\n            _dict['status'] = self.status\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'models' in _dict:\n            args['models'] = [\n                TranslationModel._from_dict(x) for x in (_dict.get('models'))\n            ]\n        else:\n            raise ValueError(\n                'Required property \\'models\\' not present in TranslationModels JSON'\n            )\n        return cls(**args)", "response": "Initialize a TranslationModels object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'word_count' in _dict:\n            args['word_count'] = _dict.get('word_count')\n        else:\n            raise ValueError(\n                'Required property \\'word_count\\' not present in TranslationResult JSON'\n            )\n        if 'character_count' in _dict:\n            args['character_count'] = _dict.get('character_count')\n        else:\n            raise ValueError(\n                'Required property \\'character_count\\' not present in TranslationResult JSON'\n            )\n        if 'translations' in _dict:\n            args['translations'] = [\n                Translation._from_dict(x) for x in (_dict.get('translations'))\n            ]\n        else:\n            raise ValueError(\n                'Required property \\'translations\\' not present in TranslationResult JSON'\n            )\n        return cls(**args)", "response": "Initialize a TranslationResult object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'word_count') and self.word_count is not None:\n            _dict['word_count'] = self.word_count\n        if hasattr(self,\n                   'character_count') and self.character_count is not None:\n            _dict['character_count'] = self.character_count\n        if hasattr(self, 'translations') and self.translations is not None:\n            _dict['translations'] = [x._to_dict() for x in self.translations]\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nrecognize audio. Sends audio and returns transcription results for a recognition request. You can pass a maximum of 100 MB and a minimum of 100 bytes of audio with a request. The service automatically detects the endianness of the incoming audio and, for audio that includes multiple channels, downmixes the audio to one-channel mono during transcoding. The method returns only final results; to enable interim results, use the WebSocket API. **See also:** [Making a basic HTTP request](https://cloud.ibm.com/docs/services/speech-to-text/http.html#HTTP-basic). ### Streaming mode For requests to transcribe live audio as it becomes available, you must set the `Transfer-Encoding` header to `chunked` to use streaming mode. In streaming mode, the service closes the connection (status code 408) if it does not receive at least 15 seconds of audio (including silence) in any 30-second period. The service also closes the connection (status code 400) if it detects no speech for `inactivity_timeout` seconds of streaming audio; use the `inactivity_timeout` parameter to change the default of 30 seconds. **See also:** * [Audio transmission](https://cloud.ibm.com/docs/services/speech-to-text/input.html#transmission) * [Timeouts](https://cloud.ibm.com/docs/services/speech-to-text/input.html#timeouts) ### Audio formats (content types) The service accepts audio in the following formats (MIME types). * For formats that are labeled **Required**, you must use the `Content-Type` header with the request to specify the format of the audio. * For all other formats, you can omit the `Content-Type` header or specify `application/octet-stream` with the header to have the service automatically detect the format of the audio. (With the `curl` command, you can specify either `\\\"Content-Type:\\\"` or `\\\"Content-Type: application/octet-stream\\\"`.) Where indicated, the format that you specify must include the sampling rate and can optionally include the number of channels and the endianness of the audio. * `audio/alaw` (**Required.** Specify the sampling rate (`rate`) of the audio.) * `audio/basic` (**Required.** Use only with narrowband models.) * `audio/flac` * `audio/g729` (Use only with narrowband models.) * `audio/l16` (**Required.** Specify the sampling rate (`rate`) and optionally the number of channels (`channels`) and endianness (`endianness`) of the audio.) * `audio/mp3` * `audio/mpeg` * `audio/mulaw` (**Required.** Specify the sampling rate (`rate`) of the audio.) * `audio/ogg` (The service automatically detects the codec of the input audio.) * `audio/ogg;codecs=opus` * `audio/ogg;codecs=vorbis` * `audio/wav` (Provide audio with a maximum of nine channels.) * `audio/webm` (The service automatically detects the codec of the input audio.) * `audio/webm;codecs=opus` * `audio/webm;codecs=vorbis` The sampling rate of the audio must match the sampling rate of the model for the recognition request: for broadband models, at least 16 kHz; for narrowband models, at least 8 kHz. If the sampling rate of the audio is higher than the minimum required rate, the service down-samples the audio to the appropriate rate. If the sampling rate of the audio is lower than the minimum required rate, the request fails. **See also:** [Audio formats](https://cloud.ibm.com/docs/services/speech-to-text/audio-formats.html). ### Multipart speech recognition **Note:** The Watson SDKs do not support multipart speech recognition. The HTTP `POST` method of the service also supports multipart speech recognition. With multipart requests, you pass all audio data as multipart form data. You specify some parameters as request headers and query parameters, but you pass JSON metadata as form data to control most aspects of the transcription. The multipart approach is intended for use with browsers for which JavaScript is disabled or when the parameters used with the request are greater than the 8 KB limit imposed by most HTTP servers and proxies. You can encounter this limit, for example, if you want to spot a very large number of keywords. **See also:** [Making a multipart HTTP request](https://cloud.ibm.com/docs/services/speech-to-text/http.html#HTTP-multi). :param file audio: The audio to transcribe. :param str model: The identifier of the model that is to be used for the recognition request. See [Languages and models](https://cloud.ibm.com/docs/services/speech-to-text/models.html). :param str language_customization_id: The customization ID (GUID) of a custom language model that is to be used with the recognition request. The base model of the specified custom language model must match the model specified with the `model` parameter. You must make the request with credentials for the instance of the service that owns the custom model. By default, no custom language model is used. See [Custom models](https://cloud.ibm.com/docs/services/speech-to-text/input.html#custom-input). **Note:** Use this parameter instead of the deprecated `customization_id` parameter. :param str acoustic_customization_id: The customization ID (GUID) of a custom acoustic model that is to be used with the recognition request. The base model of the specified custom acoustic model must match the model specified with the `model` parameter. You must make the request with credentials for the instance of the service that owns the custom model. By default, no custom acoustic model is used. See [Custom models](https://cloud.ibm.com/docs/services/speech-to-text/input.html#custom-input). :param str base_model_version: The version of the specified base model that is to be used with recognition request. Multiple versions of a base model can exist when a model is updated for internal improvements. The parameter is intended primarily for use with custom models that have been upgraded for a new base model. The default value depends on whether the parameter is used with or without a custom model. See [Base model version](https://cloud.ibm.com/docs/services/speech-to-text/input.html#version). :param float customization_weight: If you specify the customization ID (GUID) of a custom language model with the recognition request, the customization weight tells the service how much weight to give to words from the custom language model compared to those from the base model for the current request. Specify a value between 0.0 and 1.0. Unless a different customization weight was specified for the custom model when it was trained, the default value is 0.3. A customization weight that you specify overrides a weight that was specified when the custom model was trained. The default value yields the best performance in general. Assign a higher value if your audio makes frequent use of OOV words from the custom model. Use caution when setting the weight: a higher value can improve the accuracy of phrases from the custom model's domain, but it can negatively affect performance on non-domain phrases. See [Custom models](https://cloud.ibm.com/docs/services/speech-to-text/input.html#custom-input). :param int inactivity_timeout: The time in seconds after which, if only silence (no speech) is detected in streaming audio, the connection is closed with a 400 error. The parameter is useful for stopping audio submission from a live microphone when a user simply walks away. Use `-1` for infinity. See [Inactivity timeout](https://cloud.ibm.com/docs/services/speech-to-text/input.html#timeouts-inactivity). :param list[str] keywords: An array of keyword strings to spot in the audio. Each keyword string can include one or more string tokens. Keywords are spotted only in the final results, not in interim hypotheses. If you specify any keywords, you must also specify a keywords threshold. You can spot a maximum of 1000 keywords. Omit the parameter or specify an empty array if you do not need to spot keywords. See [Keyword spotting](https://cloud.ibm.com/docs/services/speech-to-text/output.html#keyword_spotting). :param float keywords_threshold: A confidence value that is the lower bound for spotting a keyword. A word is considered to match a keyword if its confidence is greater than or equal to the threshold. Specify a probability between 0.0 and 1.0. If you specify a threshold, you must also specify one or more keywords. The service performs no keyword spotting if you omit either parameter. See [Keyword spotting](https://cloud.ibm.com/docs/services/speech-to-text/output.html#keyword_spotting). :param int max_alternatives: The maximum number of alternative transcripts that the service is to return. By default, the service returns a single transcript. If you specify a value of `0`, the service uses the default value, `1`. See [Maximum alternatives](https://cloud.ibm.com/docs/services/speech-to-text/output.html#max_alternatives). :param float word_alternatives_threshold: A confidence value that is the lower bound for identifying a hypothesis as a possible word alternative (also known as \\\"Confusion Networks\\\"). An alternative word is considered if its confidence is greater than or equal to the threshold. Specify a probability between 0.0 and 1.0. By default, the service computes no alternative words. See [Word alternatives](https://cloud.ibm.com/docs/services/speech-to-text/output.html#word_alternatives). :param bool word_confidence: If `true`, the service returns a confidence measure in the range of 0.0 to 1.0 for each word. By default, the service returns no word confidence scores. See [Word confidence](https://cloud.ibm.com/docs/services/speech-to-text/output.html#word_confidence). :param bool timestamps: If `true`, the service returns time alignment for each word. By default, no timestamps are returned. See [Word timestamps](https://cloud.ibm.com/docs/services/speech-to-text/output.html#word_timestamps). :param bool profanity_filter: If `true`, the service filters profanity from all output except for keyword results by replacing inappropriate words with a series of asterisks. Set the parameter to `false` to return results with no censoring. Applies to US English transcription only. See [Profanity filtering](https://cloud.ibm.com/docs/services/speech-to-text/output.html#profanity_filter). :param bool smart_formatting: If `true`, the service converts dates, times, series of digits and numbers, phone numbers, currency values, and internet addresses into more readable, conventional representations in the final transcript of a recognition request. For US English, the service also converts certain keyword strings to punctuation symbols. By default, the service performs no smart formatting. **Note:** Applies to US English, Japanese, and Spanish transcription only. See [Smart formatting](https://cloud.ibm.com/docs/services/speech-to-text/output.html#smart_formatting). :param bool speaker_labels: If `true`, the response includes labels that identify which words were spoken by which participants in a multi-person exchange. By default, the service returns no speaker labels. Setting `speaker_labels` to `true` forces the `timestamps` parameter to be `true`, regardless of whether you specify `false` for the parameter. **Note:** Applies to US English, Japanese, and Spanish transcription only. To determine whether a language model supports speaker labels, you can also use the **Get a model** method and check that the attribute `speaker_labels` is set to `true`. See [Speaker labels](https://cloud.ibm.com/docs/services/speech-to-text/output.html#speaker_labels). :param str customization_id: **Deprecated.** Use the `language_customization_id` parameter to specify the customization ID (GUID) of a custom language model that is to be used with the recognition request. Do not specify both parameters with a request. :param str grammar_name: The name of a grammar that is to be used with the recognition request. If you specify a grammar, you must also use the `language_customization_id` parameter to specify the name of the custom language model for which the grammar is defined. The service recognizes only strings that are recognized by the specified grammar; it does not recognize other custom words from the model's words resource. See [Grammars](https://cloud.ibm.com/docs/services/speech-to-text/input.html#grammars-input). :param bool redaction: If `true`, the service redacts, or masks, numeric data from final transcripts. The feature redacts any number that has three or more consecutive digits by replacing each digit with an `X` character. It is intended to redact sensitive numeric data, such as credit card numbers. By default, the service performs no redaction. When you enable redaction, the service automatically enables smart formatting, regardless of whether you explicitly disable that feature. To ensure maximum security, the service also disables keyword spotting (ignores the `keywords` and `keywords_threshold` parameters) and returns only a single final transcript (forces the `max_alternatives` parameter to be `1`). **Note:** Applies to US English, Japanese, and Korean transcription only. See [Numeric redaction](https://cloud.ibm.com/docs/services/speech-to-text/output.html#redaction). :param str content_type: The format (MIME type) of the audio. For more information about specifying an audio format, see **Audio formats (content types)** in the method description. :param dict headers: A `dict` containing the request headers :return: A `DetailedResponse` containing the result, headers and HTTP status code. :rtype: DetailedResponse", "response": "def recognize(self,\n                  audio,\n                  model=None,\n                  language_customization_id=None,\n                  acoustic_customization_id=None,\n                  base_model_version=None,\n                  customization_weight=None,\n                  inactivity_timeout=None,\n                  keywords=None,\n                  keywords_threshold=None,\n                  max_alternatives=None,\n                  word_alternatives_threshold=None,\n                  word_confidence=None,\n                  timestamps=None,\n                  profanity_filter=None,\n                  smart_formatting=None,\n                  speaker_labels=None,\n                  customization_id=None,\n                  grammar_name=None,\n                  redaction=None,\n                  content_type=None,\n                  **kwargs):\n        \"\"\"\n        Recognize audio.\n\n        Sends audio and returns transcription results for a recognition request. You can\n        pass a maximum of 100 MB and a minimum of 100 bytes of audio with a request. The\n        service automatically detects the endianness of the incoming audio and, for audio\n        that includes multiple channels, downmixes the audio to one-channel mono during\n        transcoding. The method returns only final results; to enable interim results, use\n        the WebSocket API.\n        **See also:** [Making a basic HTTP\n        request](https://cloud.ibm.com/docs/services/speech-to-text/http.html#HTTP-basic).\n        ### Streaming mode\n         For requests to transcribe live audio as it becomes available, you must set the\n        `Transfer-Encoding` header to `chunked` to use streaming mode. In streaming mode,\n        the service closes the connection (status code 408) if it does not receive at\n        least 15 seconds of audio (including silence) in any 30-second period. The service\n        also closes the connection (status code 400) if it detects no speech for\n        `inactivity_timeout` seconds of streaming audio; use the `inactivity_timeout`\n        parameter to change the default of 30 seconds.\n        **See also:**\n        * [Audio\n        transmission](https://cloud.ibm.com/docs/services/speech-to-text/input.html#transmission)\n        *\n        [Timeouts](https://cloud.ibm.com/docs/services/speech-to-text/input.html#timeouts)\n        ### Audio formats (content types)\n         The service accepts audio in the following formats (MIME types).\n        * For formats that are labeled **Required**, you must use the `Content-Type`\n        header with the request to specify the format of the audio.\n        * For all other formats, you can omit the `Content-Type` header or specify\n        `application/octet-stream` with the header to have the service automatically\n        detect the format of the audio. (With the `curl` command, you can specify either\n        `\\\"Content-Type:\\\"` or `\\\"Content-Type: application/octet-stream\\\"`.)\n        Where indicated, the format that you specify must include the sampling rate and\n        can optionally include the number of channels and the endianness of the audio.\n        * `audio/alaw` (**Required.** Specify the sampling rate (`rate`) of the audio.)\n        * `audio/basic` (**Required.** Use only with narrowband models.)\n        * `audio/flac`\n        * `audio/g729` (Use only with narrowband models.)\n        * `audio/l16` (**Required.** Specify the sampling rate (`rate`) and optionally the\n        number of channels (`channels`) and endianness (`endianness`) of the audio.)\n        * `audio/mp3`\n        * `audio/mpeg`\n        * `audio/mulaw` (**Required.** Specify the sampling rate (`rate`) of the audio.)\n        * `audio/ogg` (The service automatically detects the codec of the input audio.)\n        * `audio/ogg;codecs=opus`\n        * `audio/ogg;codecs=vorbis`\n        * `audio/wav` (Provide audio with a maximum of nine channels.)\n        * `audio/webm` (The service automatically detects the codec of the input audio.)\n        * `audio/webm;codecs=opus`\n        * `audio/webm;codecs=vorbis`\n        The sampling rate of the audio must match the sampling rate of the model for the\n        recognition request: for broadband models, at least 16 kHz; for narrowband models,\n        at least 8 kHz. If the sampling rate of the audio is higher than the minimum\n        required rate, the service down-samples the audio to the appropriate rate. If the\n        sampling rate of the audio is lower than the minimum required rate, the request\n        fails.\n         **See also:** [Audio\n        formats](https://cloud.ibm.com/docs/services/speech-to-text/audio-formats.html).\n        ### Multipart speech recognition\n         **Note:** The Watson SDKs do not support multipart speech recognition.\n        The HTTP `POST` method of the service also supports multipart speech recognition.\n        With multipart requests, you pass all audio data as multipart form data. You\n        specify some parameters as request headers and query parameters, but you pass JSON\n        metadata as form data to control most aspects of the transcription.\n        The multipart approach is intended for use with browsers for which JavaScript is\n        disabled or when the parameters used with the request are greater than the 8 KB\n        limit imposed by most HTTP servers and proxies. You can encounter this limit, for\n        example, if you want to spot a very large number of keywords.\n        **See also:** [Making a multipart HTTP\n        request](https://cloud.ibm.com/docs/services/speech-to-text/http.html#HTTP-multi).\n\n        :param file audio: The audio to transcribe.\n        :param str model: The identifier of the model that is to be used for the\n        recognition request. See [Languages and\n        models](https://cloud.ibm.com/docs/services/speech-to-text/models.html).\n        :param str language_customization_id: The customization ID (GUID) of a custom\n        language model that is to be used with the recognition request. The base model of\n        the specified custom language model must match the model specified with the\n        `model` parameter. You must make the request with credentials for the instance of\n        the service that owns the custom model. By default, no custom language model is\n        used. See [Custom\n        models](https://cloud.ibm.com/docs/services/speech-to-text/input.html#custom-input).\n        **Note:** Use this parameter instead of the deprecated `customization_id`\n        parameter.\n        :param str acoustic_customization_id: The customization ID (GUID) of a custom\n        acoustic model that is to be used with the recognition request. The base model of\n        the specified custom acoustic model must match the model specified with the\n        `model` parameter. You must make the request with credentials for the instance of\n        the service that owns the custom model. By default, no custom acoustic model is\n        used. See [Custom\n        models](https://cloud.ibm.com/docs/services/speech-to-text/input.html#custom-input).\n        :param str base_model_version: The version of the specified base model that is to\n        be used with recognition request. Multiple versions of a base model can exist when\n        a model is updated for internal improvements. The parameter is intended primarily\n        for use with custom models that have been upgraded for a new base model. The\n        default value depends on whether the parameter is used with or without a custom\n        model. See [Base model\n        version](https://cloud.ibm.com/docs/services/speech-to-text/input.html#version).\n        :param float customization_weight: If you specify the customization ID (GUID) of a\n        custom language model with the recognition request, the customization weight tells\n        the service how much weight to give to words from the custom language model\n        compared to those from the base model for the current request.\n        Specify a value between 0.0 and 1.0. Unless a different customization weight was\n        specified for the custom model when it was trained, the default value is 0.3. A\n        customization weight that you specify overrides a weight that was specified when\n        the custom model was trained.\n        The default value yields the best performance in general. Assign a higher value if\n        your audio makes frequent use of OOV words from the custom model. Use caution when\n        setting the weight: a higher value can improve the accuracy of phrases from the\n        custom model's domain, but it can negatively affect performance on non-domain\n        phrases.\n        See [Custom\n        models](https://cloud.ibm.com/docs/services/speech-to-text/input.html#custom-input).\n        :param int inactivity_timeout: The time in seconds after which, if only silence\n        (no speech) is detected in streaming audio, the connection is closed with a 400\n        error. The parameter is useful for stopping audio submission from a live\n        microphone when a user simply walks away. Use `-1` for infinity. See [Inactivity\n        timeout](https://cloud.ibm.com/docs/services/speech-to-text/input.html#timeouts-inactivity).\n        :param list[str] keywords: An array of keyword strings to spot in the audio. Each\n        keyword string can include one or more string tokens. Keywords are spotted only in\n        the final results, not in interim hypotheses. If you specify any keywords, you\n        must also specify a keywords threshold. You can spot a maximum of 1000 keywords.\n        Omit the parameter or specify an empty array if you do not need to spot keywords.\n        See [Keyword\n        spotting](https://cloud.ibm.com/docs/services/speech-to-text/output.html#keyword_spotting).\n        :param float keywords_threshold: A confidence value that is the lower bound for\n        spotting a keyword. A word is considered to match a keyword if its confidence is\n        greater than or equal to the threshold. Specify a probability between 0.0 and 1.0.\n        If you specify a threshold, you must also specify one or more keywords. The\n        service performs no keyword spotting if you omit either parameter. See [Keyword\n        spotting](https://cloud.ibm.com/docs/services/speech-to-text/output.html#keyword_spotting).\n        :param int max_alternatives: The maximum number of alternative transcripts that\n        the service is to return. By default, the service returns a single transcript. If\n        you specify a value of `0`, the service uses the default value, `1`. See [Maximum\n        alternatives](https://cloud.ibm.com/docs/services/speech-to-text/output.html#max_alternatives).\n        :param float word_alternatives_threshold: A confidence value that is the lower\n        bound for identifying a hypothesis as a possible word alternative (also known as\n        \\\"Confusion Networks\\\"). An alternative word is considered if its confidence is\n        greater than or equal to the threshold. Specify a probability between 0.0 and 1.0.\n        By default, the service computes no alternative words. See [Word\n        alternatives](https://cloud.ibm.com/docs/services/speech-to-text/output.html#word_alternatives).\n        :param bool word_confidence: If `true`, the service returns a confidence measure\n        in the range of 0.0 to 1.0 for each word. By default, the service returns no word\n        confidence scores. See [Word\n        confidence](https://cloud.ibm.com/docs/services/speech-to-text/output.html#word_confidence).\n        :param bool timestamps: If `true`, the service returns time alignment for each\n        word. By default, no timestamps are returned. See [Word\n        timestamps](https://cloud.ibm.com/docs/services/speech-to-text/output.html#word_timestamps).\n        :param bool profanity_filter: If `true`, the service filters profanity from all\n        output except for keyword results by replacing inappropriate words with a series\n        of asterisks. Set the parameter to `false` to return results with no censoring.\n        Applies to US English transcription only. See [Profanity\n        filtering](https://cloud.ibm.com/docs/services/speech-to-text/output.html#profanity_filter).\n        :param bool smart_formatting: If `true`, the service converts dates, times, series\n        of digits and numbers, phone numbers, currency values, and internet addresses into\n        more readable, conventional representations in the final transcript of a\n        recognition request. For US English, the service also converts certain keyword\n        strings to punctuation symbols. By default, the service performs no smart\n        formatting.\n        **Note:** Applies to US English, Japanese, and Spanish transcription only.\n        See [Smart\n        formatting](https://cloud.ibm.com/docs/services/speech-to-text/output.html#smart_formatting).\n        :param bool speaker_labels: If `true`, the response includes labels that identify\n        which words were spoken by which participants in a multi-person exchange. By\n        default, the service returns no speaker labels. Setting `speaker_labels` to `true`\n        forces the `timestamps` parameter to be `true`, regardless of whether you specify\n        `false` for the parameter.\n        **Note:** Applies to US English, Japanese, and Spanish transcription only. To\n        determine whether a language model supports speaker labels, you can also use the\n        **Get a model** method and check that the attribute `speaker_labels` is set to\n        `true`.\n        See [Speaker\n        labels](https://cloud.ibm.com/docs/services/speech-to-text/output.html#speaker_labels).\n        :param str customization_id: **Deprecated.** Use the `language_customization_id`\n        parameter to specify the customization ID (GUID) of a custom language model that\n        is to be used with the recognition request. Do not specify both parameters with a\n        request.\n        :param str grammar_name: The name of a grammar that is to be used with the\n        recognition request. If you specify a grammar, you must also use the\n        `language_customization_id` parameter to specify the name of the custom language\n        model for which the grammar is defined. The service recognizes only strings that\n        are recognized by the specified grammar; it does not recognize other custom words\n        from the model's words resource. See\n        [Grammars](https://cloud.ibm.com/docs/services/speech-to-text/input.html#grammars-input).\n        :param bool redaction: If `true`, the service redacts, or masks, numeric data from\n        final transcripts. The feature redacts any number that has three or more\n        consecutive digits by replacing each digit with an `X` character. It is intended\n        to redact sensitive numeric data, such as credit card numbers. By default, the\n        service performs no redaction.\n        When you enable redaction, the service automatically enables smart formatting,\n        regardless of whether you explicitly disable that feature. To ensure maximum\n        security, the service also disables keyword spotting (ignores the `keywords` and\n        `keywords_threshold` parameters) and returns only a single final transcript\n        (forces the `max_alternatives` parameter to be `1`).\n        **Note:** Applies to US English, Japanese, and Korean transcription only.\n        See [Numeric\n        redaction](https://cloud.ibm.com/docs/services/speech-to-text/output.html#redaction).\n        :param str content_type: The format (MIME type) of the audio. For more information\n        about specifying an audio format, see **Audio formats (content types)** in the\n        method description.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse\n        \"\"\"\n\n        if audio is None:\n            raise ValueError('audio must be provided')\n\n        headers = {'Content-Type': content_type}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('speech_to_text', 'V1', 'recognize')\n        headers.update(sdk_headers)\n\n        params = {\n            'model': model,\n            'language_customization_id': language_customization_id,\n            'acoustic_customization_id': acoustic_customization_id,\n            'base_model_version': base_model_version,\n            'customization_weight': customization_weight,\n            'inactivity_timeout': inactivity_timeout,\n            'keywords': self._convert_list(keywords),\n            'keywords_threshold': keywords_threshold,\n            'max_alternatives': max_alternatives,\n            'word_alternatives_threshold': word_alternatives_threshold,\n            'word_confidence': word_confidence,\n            'timestamps': timestamps,\n            'profanity_filter': profanity_filter,\n            'smart_formatting': smart_formatting,\n            'speaker_labels': speaker_labels,\n            'customization_id': customization_id,\n            'grammar_name': grammar_name,\n            'redaction': redaction\n        }\n\n        data = audio\n\n        url = '/v1/recognize'\n        response = self.request(\n            method='POST',\n            url=url,\n            headers=headers,\n            params=params,\n            data=data,\n            accept_json=True)\n        return response"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef register_callback(self, callback_url, user_secret=None, **kwargs):\n\n        if callback_url is None:\n            raise ValueError('callback_url must be provided')\n\n        headers = {}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('speech_to_text', 'V1',\n                                      'register_callback')\n        headers.update(sdk_headers)\n\n        params = {'callback_url': callback_url, 'user_secret': user_secret}\n\n        url = '/v1/register_callback'\n        response = self.request(\n            method='POST',\n            url=url,\n            headers=headers,\n            params=params,\n            accept_json=True)\n        return response", "response": "Register a callback URL with the service."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef unregister_callback(self, callback_url, **kwargs):\n\n        if callback_url is None:\n            raise ValueError('callback_url must be provided')\n\n        headers = {}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('speech_to_text', 'V1',\n                                      'unregister_callback')\n        headers.update(sdk_headers)\n\n        params = {'callback_url': callback_url}\n\n        url = '/v1/unregister_callback'\n        response = self.request(\n            method='POST',\n            url=url,\n            headers=headers,\n            params=params,\n            accept_json=False)\n        return response", "response": "Unregisters a callback URL that was previously white - listed with a ** Register a callback** request for use with the asynchronous interface."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncreates a custom language model for a specified base model.", "response": "def create_language_model(self,\n                              name,\n                              base_model_name,\n                              dialect=None,\n                              description=None,\n                              **kwargs):\n        \"\"\"\n        Create a custom language model.\n\n        Creates a new custom language model for a specified base model. The custom\n        language model can be used only with the base model for which it is created. The\n        model is owned by the instance of the service whose credentials are used to create\n        it.\n        **See also:** [Create a custom language\n        model](https://cloud.ibm.com/docs/services/speech-to-text/language-create.html#createModel-language).\n\n        :param str name: A user-defined name for the new custom language model. Use a name\n        that is unique among all custom language models that you own. Use a localized name\n        that matches the language of the custom model. Use a name that describes the\n        domain of the custom model, such as `Medical custom model` or `Legal custom\n        model`.\n        :param str base_model_name: The name of the base language model that is to be\n        customized by the new custom language model. The new custom model can be used only\n        with the base model that it customizes.\n        To determine whether a base model supports language model customization, use the\n        **Get a model** method and check that the attribute `custom_language_model` is set\n        to `true`. You can also refer to [Language support for\n        customization](https://cloud.ibm.com/docs/services/speech-to-text/custom.html#languageSupport).\n        :param str dialect: The dialect of the specified language that is to be used with\n        the custom language model. The parameter is meaningful only for Spanish models,\n        for which the service creates a custom language model that is suited for speech in\n        one of the following dialects:\n        * `es-ES` for Castilian Spanish (the default)\n        * `es-LA` for Latin American Spanish\n        * `es-US` for North American (Mexican) Spanish\n        A specified dialect must be valid for the base model. By default, the dialect\n        matches the language of the base model; for example, `en-US` for either of the US\n        English language models.\n        :param str description: A description of the new custom language model. Use a\n        localized description that matches the language of the custom model.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse\n        \"\"\"\n\n        if name is None:\n            raise ValueError('name must be provided')\n        if base_model_name is None:\n            raise ValueError('base_model_name must be provided')\n\n        headers = {}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('speech_to_text', 'V1',\n                                      'create_language_model')\n        headers.update(sdk_headers)\n\n        data = {\n            'name': name,\n            'base_model_name': base_model_name,\n            'dialect': dialect,\n            'description': description\n        }\n\n        url = '/v1/customizations'\n        response = self.request(\n            method='POST',\n            url=url,\n            headers=headers,\n            json=data,\n            accept_json=True)\n        return response"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef list_language_models(self, language=None, **kwargs):\n\n        headers = {}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('speech_to_text', 'V1',\n                                      'list_language_models')\n        headers.update(sdk_headers)\n\n        params = {'language': language}\n\n        url = '/v1/customizations'\n        response = self.request(\n            method='GET',\n            url=url,\n            headers=headers,\n            params=params,\n            accept_json=True)\n        return response", "response": "List custom language models."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ntraining a custom language model on the latest data.", "response": "def train_language_model(self,\n                             customization_id,\n                             word_type_to_add=None,\n                             customization_weight=None,\n                             **kwargs):\n        \"\"\"\n        Train a custom language model.\n\n        Initiates the training of a custom language model with new resources such as\n        corpora, grammars, and custom words. After adding, modifying, or deleting\n        resources for a custom language model, use this method to begin the actual\n        training of the model on the latest data. You can specify whether the custom\n        language model is to be trained with all words from its words resource or only\n        with words that were added or modified by the user directly. You must use\n        credentials for the instance of the service that owns a model to train it.\n        The training method is asynchronous. It can take on the order of minutes to\n        complete depending on the amount of data on which the service is being trained and\n        the current load on the service. The method returns an HTTP 200 response code to\n        indicate that the training process has begun.\n        You can monitor the status of the training by using the **Get a custom language\n        model** method to poll the model's status. Use a loop to check the status every 10\n        seconds. The method returns a `LanguageModel` object that includes `status` and\n        `progress` fields. A status of `available` means that the custom model is trained\n        and ready to use. The service cannot accept subsequent training requests or\n        requests to add new resources until the existing request completes.\n        Training can fail to start for the following reasons:\n        * The service is currently handling another request for the custom model, such as\n        another training request or a request to add a corpus or grammar to the model.\n        * No training data have been added to the custom model.\n        * One or more words that were added to the custom model have invalid sounds-like\n        pronunciations that you must fix.\n        **See also:** [Train the custom language\n        model](https://cloud.ibm.com/docs/services/speech-to-text/language-create.html#trainModel-language).\n\n        :param str customization_id: The customization ID (GUID) of the custom language\n        model that is to be used for the request. You must make the request with\n        credentials for the instance of the service that owns the custom model.\n        :param str word_type_to_add: The type of words from the custom language model's\n        words resource on which to train the model:\n        * `all` (the default) trains the model on all new words, regardless of whether\n        they were extracted from corpora or grammars or were added or modified by the\n        user.\n        * `user` trains the model only on new words that were added or modified by the\n        user directly. The model is not trained on new words extracted from corpora or\n        grammars.\n        :param float customization_weight: Specifies a customization weight for the custom\n        language model. The customization weight tells the service how much weight to give\n        to words from the custom language model compared to those from the base model for\n        speech recognition. Specify a value between 0.0 and 1.0; the default is 0.3.\n        The default value yields the best performance in general. Assign a higher value if\n        your audio makes frequent use of OOV words from the custom model. Use caution when\n        setting the weight: a higher value can improve the accuracy of phrases from the\n        custom model's domain, but it can negatively affect performance on non-domain\n        phrases.\n        The value that you assign is used for all recognition requests that use the model.\n        You can override it for any recognition request by specifying a customization\n        weight for that request.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse\n        \"\"\"\n\n        if customization_id is None:\n            raise ValueError('customization_id must be provided')\n\n        headers = {}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('speech_to_text', 'V1',\n                                      'train_language_model')\n        headers.update(sdk_headers)\n\n        params = {\n            'word_type_to_add': word_type_to_add,\n            'customization_weight': customization_weight\n        }\n\n        url = '/v1/customizations/{0}/train'.format(\n            *self._encode_path_vars(customization_id))\n        response = self.request(\n            method='POST',\n            url=url,\n            headers=headers,\n            params=params,\n            accept_json=True)\n        return response"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nadds a corpus to the custom language model.", "response": "def add_corpus(self,\n                   customization_id,\n                   corpus_name,\n                   corpus_file,\n                   allow_overwrite=None,\n                   **kwargs):\n        \"\"\"\n        Add a corpus.\n\n        Adds a single corpus text file of new training data to a custom language model.\n        Use multiple requests to submit multiple corpus text files. You must use\n        credentials for the instance of the service that owns a model to add a corpus to\n        it. Adding a corpus does not affect the custom language model until you train the\n        model for the new data by using the **Train a custom language model** method.\n        Submit a plain text file that contains sample sentences from the domain of\n        interest to enable the service to extract words in context. The more sentences you\n        add that represent the context in which speakers use words from the domain, the\n        better the service's recognition accuracy.\n        The call returns an HTTP 201 response code if the corpus is valid. The service\n        then asynchronously processes the contents of the corpus and automatically\n        extracts new words that it finds. This can take on the order of a minute or two to\n        complete depending on the total number of words and the number of new words in the\n        corpus, as well as the current load on the service. You cannot submit requests to\n        add additional resources to the custom model or to train the model until the\n        service's analysis of the corpus for the current request completes. Use the **List\n        a corpus** method to check the status of the analysis.\n        The service auto-populates the model's words resource with words from the corpus\n        that are not found in its base vocabulary. These are referred to as\n        out-of-vocabulary (OOV) words. You can use the **List custom words** method to\n        examine the words resource. You can use other words method to eliminate typos and\n        modify how words are pronounced as needed.\n        To add a corpus file that has the same name as an existing corpus, set the\n        `allow_overwrite` parameter to `true`; otherwise, the request fails. Overwriting\n        an existing corpus causes the service to process the corpus text file and extract\n        OOV words anew. Before doing so, it removes any OOV words associated with the\n        existing corpus from the model's words resource unless they were also added by\n        another corpus or grammar, or they have been modified in some way with the **Add\n        custom words** or **Add a custom word** method.\n        The service limits the overall amount of data that you can add to a custom model\n        to a maximum of 10 million total words from all sources combined. Also, you can\n        add no more than 30 thousand custom (OOV) words to a model. This includes words\n        that the service extracts from corpora and grammars, and words that you add\n        directly.\n        **See also:**\n        * [Working with\n        corpora](https://cloud.ibm.com/docs/services/speech-to-text/language-resource.html#workingCorpora)\n        * [Add corpora to the custom language\n        model](https://cloud.ibm.com/docs/services/speech-to-text/language-create.html#addCorpora).\n\n        :param str customization_id: The customization ID (GUID) of the custom language\n        model that is to be used for the request. You must make the request with\n        credentials for the instance of the service that owns the custom model.\n        :param str corpus_name: The name of the new corpus for the custom language model.\n        Use a localized name that matches the language of the custom model and reflects\n        the contents of the corpus.\n        * Include a maximum of 128 characters in the name.\n        * Do not include spaces, slashes, or backslashes in the name.\n        * Do not use the name of an existing corpus or grammar that is already defined for\n        the custom model.\n        * Do not use the name `user`, which is reserved by the service to denote custom\n        words that are added or modified by the user.\n        :param file corpus_file: A plain text file that contains the training data for the\n        corpus. Encode the file in UTF-8 if it contains non-ASCII characters; the service\n        assumes UTF-8 encoding if it encounters non-ASCII characters.\n        Make sure that you know the character encoding of the file. You must use that\n        encoding when working with the words in the custom language model. For more\n        information, see [Character\n        encoding](https://cloud.ibm.com/docs/services/speech-to-text/language-resource.html#charEncoding).\n        With the `curl` command, use the `--data-binary` option to upload the file for the\n        request.\n        :param bool allow_overwrite: If `true`, the specified corpus overwrites an\n        existing corpus with the same name. If `false`, the request fails if a corpus with\n        the same name already exists. The parameter has no effect if a corpus with the\n        same name does not already exist.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse\n        \"\"\"\n\n        if customization_id is None:\n            raise ValueError('customization_id must be provided')\n        if corpus_name is None:\n            raise ValueError('corpus_name must be provided')\n        if corpus_file is None:\n            raise ValueError('corpus_file must be provided')\n\n        headers = {}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('speech_to_text', 'V1', 'add_corpus')\n        headers.update(sdk_headers)\n\n        params = {'allow_overwrite': allow_overwrite}\n\n        form_data = {}\n        form_data['corpus_file'] = (None, corpus_file, 'text/plain')\n\n        url = '/v1/customizations/{0}/corpora/{1}'.format(\n            *self._encode_path_vars(customization_id, corpus_name))\n        response = self.request(\n            method='POST',\n            url=url,\n            headers=headers,\n            params=params,\n            files=form_data,\n            accept_json=True)\n        return response"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef add_word(self,\n                 customization_id,\n                 word_name,\n                 word=None,\n                 sounds_like=None,\n                 display_as=None,\n                 **kwargs):\n        \"\"\"\n        Add a custom word.\n\n        Adds a custom word to a custom language model. The service populates the words\n        resource for a custom model with out-of-vocabulary (OOV) words from each corpus or\n        grammar that is added to the model. You can use this method to add a word or to\n        modify an existing word in the words resource. The words resource for a model can\n        contain a maximum of 30 thousand custom (OOV) words. This includes words that the\n        service extracts from corpora and grammars and words that you add directly.\n        You must use credentials for the instance of the service that owns a model to add\n        or modify a custom word for the model. Adding or modifying a custom word does not\n        affect the custom model until you train the model for the new data by using the\n        **Train a custom language model** method.\n        Use the `word_name` parameter to specify the custom word that is to be added or\n        modified. Use the `CustomWord` object to provide one or both of the optional\n        `sounds_like` and `display_as` fields for the word.\n        * The `sounds_like` field provides an array of one or more pronunciations for the\n        word. Use the parameter to specify how the word can be pronounced by users. Use\n        the parameter for words that are difficult to pronounce, foreign words, acronyms,\n        and so on. For example, you might specify that the word `IEEE` can sound like `i\n        triple e`. You can specify a maximum of five sounds-like pronunciations for a\n        word.\n        * The `display_as` field provides a different way of spelling the word in a\n        transcript. Use the parameter when you want the word to appear different from its\n        usual representation or from its spelling in training data. For example, you might\n        indicate that the word `IBM(trademark)` is to be displayed as `IBM&trade;`.\n        If you add a custom word that already exists in the words resource for the custom\n        model, the new definition overwrites the existing data for the word. If the\n        service encounters an error, it does not add the word to the words resource. Use\n        the **List a custom word** method to review the word that you add.\n        **See also:**\n        * [Working with custom\n        words](https://cloud.ibm.com/docs/services/speech-to-text/language-resource.html#workingWords)\n        * [Add words to the custom language\n        model](https://cloud.ibm.com/docs/services/speech-to-text/language-create.html#addWords).\n\n        :param str customization_id: The customization ID (GUID) of the custom language\n        model that is to be used for the request. You must make the request with\n        credentials for the instance of the service that owns the custom model.\n        :param str word_name: The custom word that is to be added to or updated in the\n        custom language model. Do not include spaces in the word. Use a `-` (dash) or `_`\n        (underscore) to connect the tokens of compound words. URL-encode the word if it\n        includes non-ASCII characters. For more information, see [Character\n        encoding](https://cloud.ibm.com/docs/services/speech-to-text/language-resource.html#charEncoding).\n        :param str word: For the **Add custom words** method, you must specify the custom\n        word that is to be added to or updated in the custom model. Do not include spaces\n        in the word. Use a `-` (dash) or `_` (underscore) to connect the tokens of\n        compound words.\n        Omit this parameter for the **Add a custom word** method.\n        :param list[str] sounds_like: An array of sounds-like pronunciations for the\n        custom word. Specify how words that are difficult to pronounce, foreign words,\n        acronyms, and so on can be pronounced by users.\n        * For a word that is not in the service's base vocabulary, omit the parameter to\n        have the service automatically generate a sounds-like pronunciation for the word.\n        * For a word that is in the service's base vocabulary, use the parameter to\n        specify additional pronunciations for the word. You cannot override the default\n        pronunciation of a word; pronunciations you add augment the pronunciation from the\n        base vocabulary.\n        A word can have at most five sounds-like pronunciations. A pronunciation can\n        include at most 40 characters not including spaces.\n        :param str display_as: An alternative spelling for the custom word when it appears\n        in a transcript. Use the parameter when you want the word to have a spelling that\n        is different from its usual representation or from its spelling in corpora\n        training data.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse\n        \"\"\"\n\n        if customization_id is None:\n            raise ValueError('customization_id must be provided')\n        if word_name is None:\n            raise ValueError('word_name must be provided')\n\n        headers = {}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('speech_to_text', 'V1', 'add_word')\n        headers.update(sdk_headers)\n\n        data = {\n            'word': word,\n            'sounds_like': sounds_like,\n            'display_as': display_as\n        }\n\n        url = '/v1/customizations/{0}/words/{1}'.format(\n            *self._encode_path_vars(customization_id, word_name))\n        response = self.request(\n            method='PUT', url=url, headers=headers, json=data, accept_json=True)\n        return response", "response": "Add a custom word to the custom language model."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nadds a grammar to a custom language model.", "response": "def add_grammar(self,\n                    customization_id,\n                    grammar_name,\n                    grammar_file,\n                    content_type,\n                    allow_overwrite=None,\n                    **kwargs):\n        \"\"\"\n        Add a grammar.\n\n        Adds a single grammar file to a custom language model. Submit a plain text file in\n        UTF-8 format that defines the grammar. Use multiple requests to submit multiple\n        grammar files. You must use credentials for the instance of the service that owns\n        a model to add a grammar to it. Adding a grammar does not affect the custom\n        language model until you train the model for the new data by using the **Train a\n        custom language model** method.\n        The call returns an HTTP 201 response code if the grammar is valid. The service\n        then asynchronously processes the contents of the grammar and automatically\n        extracts new words that it finds. This can take a few seconds to complete\n        depending on the size and complexity of the grammar, as well as the current load\n        on the service. You cannot submit requests to add additional resources to the\n        custom model or to train the model until the service's analysis of the grammar for\n        the current request completes. Use the **Get a grammar** method to check the\n        status of the analysis.\n        The service populates the model's words resource with any word that is recognized\n        by the grammar that is not found in the model's base vocabulary. These are\n        referred to as out-of-vocabulary (OOV) words. You can use the **List custom\n        words** method to examine the words resource and use other words-related methods\n        to eliminate typos and modify how words are pronounced as needed.\n        To add a grammar that has the same name as an existing grammar, set the\n        `allow_overwrite` parameter to `true`; otherwise, the request fails. Overwriting\n        an existing grammar causes the service to process the grammar file and extract OOV\n        words anew. Before doing so, it removes any OOV words associated with the existing\n        grammar from the model's words resource unless they were also added by another\n        resource or they have been modified in some way with the **Add custom words** or\n        **Add a custom word** method.\n        The service limits the overall amount of data that you can add to a custom model\n        to a maximum of 10 million total words from all sources combined. Also, you can\n        add no more than 30 thousand OOV words to a model. This includes words that the\n        service extracts from corpora and grammars and words that you add directly.\n        **See also:**\n        * [Working with grammars](https://cloud.ibm.com/docs/services/speech-to-text/)\n        * [Add grammars to the custom language\n        model](https://cloud.ibm.com/docs/services/speech-to-text/).\n\n        :param str customization_id: The customization ID (GUID) of the custom language\n        model that is to be used for the request. You must make the request with\n        credentials for the instance of the service that owns the custom model.\n        :param str grammar_name: The name of the new grammar for the custom language\n        model. Use a localized name that matches the language of the custom model and\n        reflects the contents of the grammar.\n        * Include a maximum of 128 characters in the name.\n        * Do not include spaces, slashes, or backslashes in the name.\n        * Do not use the name of an existing grammar or corpus that is already defined for\n        the custom model.\n        * Do not use the name `user`, which is reserved by the service to denote custom\n        words that are added or modified by the user.\n        :param str grammar_file: A plain text file that contains the grammar in the format\n        specified by the `Content-Type` header. Encode the file in UTF-8 (ASCII is a\n        subset of UTF-8). Using any other encoding can lead to issues when compiling the\n        grammar or to unexpected results in decoding. The service ignores an encoding that\n        is specified in the header of the grammar.\n        :param str content_type: The format (MIME type) of the grammar file:\n        * `application/srgs` for Augmented Backus-Naur Form (ABNF), which uses a\n        plain-text representation that is similar to traditional BNF grammars.\n        * `application/srgs+xml` for XML Form, which uses XML elements to represent the\n        grammar.\n        :param bool allow_overwrite: If `true`, the specified grammar overwrites an\n        existing grammar with the same name. If `false`, the request fails if a grammar\n        with the same name already exists. The parameter has no effect if a grammar with\n        the same name does not already exist.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse\n        \"\"\"\n\n        if customization_id is None:\n            raise ValueError('customization_id must be provided')\n        if grammar_name is None:\n            raise ValueError('grammar_name must be provided')\n        if grammar_file is None:\n            raise ValueError('grammar_file must be provided')\n        if content_type is None:\n            raise ValueError('content_type must be provided')\n\n        headers = {'Content-Type': content_type}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('speech_to_text', 'V1', 'add_grammar')\n        headers.update(sdk_headers)\n\n        params = {'allow_overwrite': allow_overwrite}\n\n        data = grammar_file\n\n        url = '/v1/customizations/{0}/grammars/{1}'.format(\n            *self._encode_path_vars(customization_id, grammar_name))\n        response = self.request(\n            method='POST',\n            url=url,\n            headers=headers,\n            params=params,\n            data=data,\n            accept_json=True)\n        return response"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nadd an audio resource to the acoustic model.", "response": "def add_audio(self,\n                  customization_id,\n                  audio_name,\n                  audio_resource,\n                  contained_content_type=None,\n                  allow_overwrite=None,\n                  content_type=None,\n                  **kwargs):\n        \"\"\"\n        Add an audio resource.\n\n        Adds an audio resource to a custom acoustic model. Add audio content that reflects\n        the acoustic characteristics of the audio that you plan to transcribe. You must\n        use credentials for the instance of the service that owns a model to add an audio\n        resource to it. Adding audio data does not affect the custom acoustic model until\n        you train the model for the new data by using the **Train a custom acoustic\n        model** method.\n        You can add individual audio files or an archive file that contains multiple audio\n        files. Adding multiple audio files via a single archive file is significantly more\n        efficient than adding each file individually. You can add audio resources in any\n        format that the service supports for speech recognition.\n        You can use this method to add any number of audio resources to a custom model by\n        calling the method once for each audio or archive file. But the addition of one\n        audio resource must be fully complete before you can add another. You must add a\n        minimum of 10 minutes and a maximum of 100 hours of audio that includes speech,\n        not just silence, to a custom acoustic model before you can train it. No audio\n        resource, audio- or archive-type, can be larger than 100 MB. To add an audio\n        resource that has the same name as an existing audio resource, set the\n        `allow_overwrite` parameter to `true`; otherwise, the request fails.\n        The method is asynchronous. It can take several seconds to complete depending on\n        the duration of the audio and, in the case of an archive file, the total number of\n        audio files being processed. The service returns a 201 response code if the audio\n        is valid. It then asynchronously analyzes the contents of the audio file or files\n        and automatically extracts information about the audio such as its length,\n        sampling rate, and encoding. You cannot submit requests to add additional audio\n        resources to a custom acoustic model, or to train the model, until the service's\n        analysis of all audio files for the current request completes.\n        To determine the status of the service's analysis of the audio, use the **Get an\n        audio resource** method to poll the status of the audio. The method accepts the\n        customization ID of the custom model and the name of the audio resource, and it\n        returns the status of the resource. Use a loop to check the status of the audio\n        every few seconds until it becomes `ok`.\n        **See also:** [Add audio to the custom acoustic\n        model](https://cloud.ibm.com/docs/services/speech-to-text/acoustic-create.html#addAudio).\n        ### Content types for audio-type resources\n         You can add an individual audio file in any format that the service supports for\n        speech recognition. For an audio-type resource, use the `Content-Type` parameter\n        to specify the audio format (MIME type) of the audio file, including specifying\n        the sampling rate, channels, and endianness where indicated.\n        * `audio/alaw` (Specify the sampling rate (`rate`) of the audio.)\n        * `audio/basic` (Use only with narrowband models.)\n        * `audio/flac`\n        * `audio/g729` (Use only with narrowband models.)\n        * `audio/l16` (Specify the sampling rate (`rate`) and optionally the number of\n        channels (`channels`) and endianness (`endianness`) of the audio.)\n        * `audio/mp3`\n        * `audio/mpeg`\n        * `audio/mulaw` (Specify the sampling rate (`rate`) of the audio.)\n        * `audio/ogg` (The service automatically detects the codec of the input audio.)\n        * `audio/ogg;codecs=opus`\n        * `audio/ogg;codecs=vorbis`\n        * `audio/wav` (Provide audio with a maximum of nine channels.)\n        * `audio/webm` (The service automatically detects the codec of the input audio.)\n        * `audio/webm;codecs=opus`\n        * `audio/webm;codecs=vorbis`\n        The sampling rate of an audio file must match the sampling rate of the base model\n        for the custom model: for broadband models, at least 16 kHz; for narrowband\n        models, at least 8 kHz. If the sampling rate of the audio is higher than the\n        minimum required rate, the service down-samples the audio to the appropriate rate.\n        If the sampling rate of the audio is lower than the minimum required rate, the\n        service labels the audio file as `invalid`.\n         **See also:** [Audio\n        formats](https://cloud.ibm.com/docs/services/speech-to-text/audio-formats.html).\n        ### Content types for archive-type resources\n         You can add an archive file (**.zip** or **.tar.gz** file) that contains audio\n        files in any format that the service supports for speech recognition. For an\n        archive-type resource, use the `Content-Type` parameter to specify the media type\n        of the archive file:\n        * `application/zip` for a **.zip** file\n        * `application/gzip` for a **.tar.gz** file.\n        When you add an archive-type resource, the `Contained-Content-Type` header is\n        optional depending on the format of the files that you are adding:\n        * For audio files of type `audio/alaw`, `audio/basic`, `audio/l16`, or\n        `audio/mulaw`, you must use the `Contained-Content-Type` header to specify the\n        format of the contained audio files. Include the `rate`, `channels`, and\n        `endianness` parameters where necessary. In this case, all audio files contained\n        in the archive file must have the same audio format.\n        * For audio files of all other types, you can omit the `Contained-Content-Type`\n        header. In this case, the audio files contained in the archive file can have any\n        of the formats not listed in the previous bullet. The audio files do not need to\n        have the same format.\n        Do not use the `Contained-Content-Type` header when adding an audio-type resource.\n        ### Naming restrictions for embedded audio files\n         The name of an audio file that is embedded within an archive-type resource must\n        meet the following restrictions:\n        * Include a maximum of 128 characters in the file name; this includes the file\n        extension.\n        * Do not include spaces, slashes, or backslashes in the file name.\n        * Do not use the name of an audio file that has already been added to the custom\n        model as part of an archive-type resource.\n\n        :param str customization_id: The customization ID (GUID) of the custom acoustic\n        model that is to be used for the request. You must make the request with\n        credentials for the instance of the service that owns the custom model.\n        :param str audio_name: The name of the new audio resource for the custom acoustic\n        model. Use a localized name that matches the language of the custom model and\n        reflects the contents of the resource.\n        * Include a maximum of 128 characters in the name.\n        * Do not include spaces, slashes, or backslashes in the name.\n        * Do not use the name of an audio resource that has already been added to the\n        custom model.\n        :param file audio_resource: The audio resource that is to be added to the custom\n        acoustic model, an individual audio file or an archive file.\n        :param str contained_content_type: **For an archive-type resource,** specify the\n        format of the audio files that are contained in the archive file if they are of\n        type `audio/alaw`, `audio/basic`, `audio/l16`, or `audio/mulaw`. Include the\n        `rate`, `channels`, and `endianness` parameters where necessary. In this case, all\n        audio files that are contained in the archive file must be of the indicated type.\n        For all other audio formats, you can omit the header. In this case, the audio\n        files can be of multiple types as long as they are not of the types listed in the\n        previous paragraph.\n        The parameter accepts all of the audio formats that are supported for use with\n        speech recognition. For more information, see **Content types for audio-type\n        resources** in the method description.\n        **For an audio-type resource,** omit the header.\n        :param bool allow_overwrite: If `true`, the specified audio resource overwrites an\n        existing audio resource with the same name. If `false`, the request fails if an\n        audio resource with the same name already exists. The parameter has no effect if\n        an audio resource with the same name does not already exist.\n        :param str content_type: For an audio-type resource, the format (MIME type) of the\n        audio. For more information, see **Content types for audio-type resources** in the\n        method description.\n        For an archive-type resource, the media type of the archive file. For more\n        information, see **Content types for archive-type resources** in the method\n        description.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse\n        \"\"\"\n\n        if customization_id is None:\n            raise ValueError('customization_id must be provided')\n        if audio_name is None:\n            raise ValueError('audio_name must be provided')\n        if audio_resource is None:\n            raise ValueError('audio_resource must be provided')\n\n        headers = {\n            'Contained-Content-Type': contained_content_type,\n            'Content-Type': content_type\n        }\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('speech_to_text', 'V1', 'add_audio')\n        headers.update(sdk_headers)\n\n        params = {'allow_overwrite': allow_overwrite}\n\n        data = audio_resource\n\n        url = '/v1/acoustic_customizations/{0}/audio/{1}'.format(\n            *self._encode_path_vars(customization_id, audio_name))\n        response = self.request(\n            method='POST',\n            url=url,\n            headers=headers,\n            params=params,\n            data=data,\n            accept_json=True)\n        return response"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self,\n                   'customization_id') and self.customization_id is not None:\n            _dict['customization_id'] = self.customization_id\n        if hasattr(self, 'created') and self.created is not None:\n            _dict['created'] = self.created\n        if hasattr(self, 'language') and self.language is not None:\n            _dict['language'] = self.language\n        if hasattr(self, 'versions') and self.versions is not None:\n            _dict['versions'] = self.versions\n        if hasattr(self, 'owner') and self.owner is not None:\n            _dict['owner'] = self.owner\n        if hasattr(self, 'name') and self.name is not None:\n            _dict['name'] = self.name\n        if hasattr(self, 'description') and self.description is not None:\n            _dict['description'] = self.description\n        if hasattr(self,\n                   'base_model_name') and self.base_model_name is not None:\n            _dict['base_model_name'] = self.base_model_name\n        if hasattr(self, 'status') and self.status is not None:\n            _dict['status'] = self.status\n        if hasattr(self, 'progress') and self.progress is not None:\n            _dict['progress'] = self.progress\n        if hasattr(self, 'warnings') and self.warnings is not None:\n            _dict['warnings'] = self.warnings\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'customizations' in _dict:\n            args['customizations'] = [\n                AcousticModel._from_dict(x)\n                for x in (_dict.get('customizations'))\n            ]\n        else:\n            raise ValueError(\n                'Required property \\'customizations\\' not present in AcousticModels JSON'\n            )\n        return cls(**args)", "response": "Initialize a AcousticModels object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'customizations') and self.customizations is not None:\n            _dict['customizations'] = [\n                x._to_dict() for x in self.customizations\n            ]\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'type') and self.type is not None:\n            _dict['type'] = self.type\n        if hasattr(self, 'codec') and self.codec is not None:\n            _dict['codec'] = self.codec\n        if hasattr(self, 'frequency') and self.frequency is not None:\n            _dict['frequency'] = self.frequency\n        if hasattr(self, 'compression') and self.compression is not None:\n            _dict['compression'] = self.compression\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'duration' in _dict:\n            args['duration'] = _dict.get('duration')\n        if 'name' in _dict:\n            args['name'] = _dict.get('name')\n        if 'details' in _dict:\n            args['details'] = AudioDetails._from_dict(_dict.get('details'))\n        if 'status' in _dict:\n            args['status'] = _dict.get('status')\n        if 'container' in _dict:\n            args['container'] = AudioResource._from_dict(_dict.get('container'))\n        if 'audio' in _dict:\n            args['audio'] = [\n                AudioResource._from_dict(x) for x in (_dict.get('audio'))\n            ]\n        return cls(**args)", "response": "Initialize a AudioListing object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'duration') and self.duration is not None:\n            _dict['duration'] = self.duration\n        if hasattr(self, 'name') and self.name is not None:\n            _dict['name'] = self.name\n        if hasattr(self, 'details') and self.details is not None:\n            _dict['details'] = self.details._to_dict()\n        if hasattr(self, 'status') and self.status is not None:\n            _dict['status'] = self.status\n        if hasattr(self, 'container') and self.container is not None:\n            _dict['container'] = self.container._to_dict()\n        if hasattr(self, 'audio') and self.audio is not None:\n            _dict['audio'] = [x._to_dict() for x in self.audio]\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'duration') and self.duration is not None:\n            _dict['duration'] = self.duration\n        if hasattr(self, 'name') and self.name is not None:\n            _dict['name'] = self.name\n        if hasattr(self, 'details') and self.details is not None:\n            _dict['details'] = self.details._to_dict()\n        if hasattr(self, 'status') and self.status is not None:\n            _dict['status'] = self.status\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'total_minutes_of_audio' in _dict:\n            args['total_minutes_of_audio'] = _dict.get('total_minutes_of_audio')\n        else:\n            raise ValueError(\n                'Required property \\'total_minutes_of_audio\\' not present in AudioResources JSON'\n            )\n        if 'audio' in _dict:\n            args['audio'] = [\n                AudioResource._from_dict(x) for x in (_dict.get('audio'))\n            ]\n        else:\n            raise ValueError(\n                'Required property \\'audio\\' not present in AudioResources JSON'\n            )\n        return cls(**args)", "response": "Initialize a AudioResources object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'total_minutes_of_audio'\n                  ) and self.total_minutes_of_audio is not None:\n            _dict['total_minutes_of_audio'] = self.total_minutes_of_audio\n        if hasattr(self, 'audio') and self.audio is not None:\n            _dict['audio'] = [x._to_dict() for x in self.audio]\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ninitializing a Corpora object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a Corpora object from a json dictionary.\"\"\"\n        args = {}\n        if 'corpora' in _dict:\n            args['corpora'] = [\n                Corpus._from_dict(x) for x in (_dict.get('corpora'))\n            ]\n        else:\n            raise ValueError(\n                'Required property \\'corpora\\' not present in Corpora JSON')\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'corpora') and self.corpora is not None:\n            _dict['corpora'] = [x._to_dict() for x in self.corpora]\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'word') and self.word is not None:\n            _dict['word'] = self.word\n        if hasattr(self, 'sounds_like') and self.sounds_like is not None:\n            _dict['sounds_like'] = self.sounds_like\n        if hasattr(self, 'display_as') and self.display_as is not None:\n            _dict['display_as'] = self.display_as\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'name') and self.name is not None:\n            _dict['name'] = self.name\n        if hasattr(self, 'out_of_vocabulary_words'\n                  ) and self.out_of_vocabulary_words is not None:\n            _dict['out_of_vocabulary_words'] = self.out_of_vocabulary_words\n        if hasattr(self, 'status') and self.status is not None:\n            _dict['status'] = self.status\n        if hasattr(self, 'error') and self.error is not None:\n            _dict['error'] = self.error\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'grammars' in _dict:\n            args['grammars'] = [\n                Grammar._from_dict(x) for x in (_dict.get('grammars'))\n            ]\n        else:\n            raise ValueError(\n                'Required property \\'grammars\\' not present in Grammars JSON')\n        return cls(**args)", "response": "Initialize a Grammars object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'grammars') and self.grammars is not None:\n            _dict['grammars'] = [x._to_dict() for x in self.grammars]\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self,\n                   'normalized_text') and self.normalized_text is not None:\n            _dict['normalized_text'] = self.normalized_text\n        if hasattr(self, 'start_time') and self.start_time is not None:\n            _dict['start_time'] = self.start_time\n        if hasattr(self, 'end_time') and self.end_time is not None:\n            _dict['end_time'] = self.end_time\n        if hasattr(self, 'confidence') and self.confidence is not None:\n            _dict['confidence'] = self.confidence\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'customizations' in _dict:\n            args['customizations'] = [\n                LanguageModel._from_dict(x)\n                for x in (_dict.get('customizations'))\n            ]\n        else:\n            raise ValueError(\n                'Required property \\'customizations\\' not present in LanguageModels JSON'\n            )\n        return cls(**args)", "response": "Initialize a LanguageModels object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'id' in _dict:\n            args['id'] = _dict.get('id')\n        else:\n            raise ValueError(\n                'Required property \\'id\\' not present in RecognitionJob JSON')\n        if 'status' in _dict:\n            args['status'] = _dict.get('status')\n        else:\n            raise ValueError(\n                'Required property \\'status\\' not present in RecognitionJob JSON'\n            )\n        if 'created' in _dict:\n            args['created'] = _dict.get('created')\n        else:\n            raise ValueError(\n                'Required property \\'created\\' not present in RecognitionJob JSON'\n            )\n        if 'updated' in _dict:\n            args['updated'] = _dict.get('updated')\n        if 'url' in _dict:\n            args['url'] = _dict.get('url')\n        if 'user_token' in _dict:\n            args['user_token'] = _dict.get('user_token')\n        if 'results' in _dict:\n            args['results'] = [\n                SpeechRecognitionResults._from_dict(x)\n                for x in (_dict.get('results'))\n            ]\n        if 'warnings' in _dict:\n            args['warnings'] = _dict.get('warnings')\n        return cls(**args)", "response": "Initialize a RecognitionJob object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'id') and self.id is not None:\n            _dict['id'] = self.id\n        if hasattr(self, 'status') and self.status is not None:\n            _dict['status'] = self.status\n        if hasattr(self, 'created') and self.created is not None:\n            _dict['created'] = self.created\n        if hasattr(self, 'updated') and self.updated is not None:\n            _dict['updated'] = self.updated\n        if hasattr(self, 'url') and self.url is not None:\n            _dict['url'] = self.url\n        if hasattr(self, 'user_token') and self.user_token is not None:\n            _dict['user_token'] = self.user_token\n        if hasattr(self, 'results') and self.results is not None:\n            _dict['results'] = [x._to_dict() for x in self.results]\n        if hasattr(self, 'warnings') and self.warnings is not None:\n            _dict['warnings'] = self.warnings\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ninitializing a RecognitionJobs object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a RecognitionJobs object from a json dictionary.\"\"\"\n        args = {}\n        if 'recognitions' in _dict:\n            args['recognitions'] = [\n                RecognitionJob._from_dict(x)\n                for x in (_dict.get('recognitions'))\n            ]\n        else:\n            raise ValueError(\n                'Required property \\'recognitions\\' not present in RecognitionJobs JSON'\n            )\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'recognitions') and self.recognitions is not None:\n            _dict['recognitions'] = [x._to_dict() for x in self.recognitions]\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'status') and self.status is not None:\n            _dict['status'] = self.status\n        if hasattr(self, 'url') and self.url is not None:\n            _dict['url'] = self.url\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'from_') and self.from_ is not None:\n            _dict['from'] = self.from_\n        if hasattr(self, 'to') and self.to is not None:\n            _dict['to'] = self.to\n        if hasattr(self, 'speaker') and self.speaker is not None:\n            _dict['speaker'] = self.speaker\n        if hasattr(self, 'confidence') and self.confidence is not None:\n            _dict['confidence'] = self.confidence\n        if hasattr(self, 'final_results') and self.final_results is not None:\n            _dict['final'] = self.final_results\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'name') and self.name is not None:\n            _dict['name'] = self.name\n        if hasattr(self, 'language') and self.language is not None:\n            _dict['language'] = self.language\n        if hasattr(self, 'rate') and self.rate is not None:\n            _dict['rate'] = self.rate\n        if hasattr(self, 'url') and self.url is not None:\n            _dict['url'] = self.url\n        if hasattr(\n                self,\n                'supported_features') and self.supported_features is not None:\n            _dict['supported_features'] = self.supported_features._to_dict()\n        if hasattr(self, 'description') and self.description is not None:\n            _dict['description'] = self.description\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'models' in _dict:\n            args['models'] = [\n                SpeechModel._from_dict(x) for x in (_dict.get('models'))\n            ]\n        else:\n            raise ValueError(\n                'Required property \\'models\\' not present in SpeechModels JSON')\n        return cls(**args)", "response": "Initialize a SpeechModels object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'transcript') and self.transcript is not None:\n            _dict['transcript'] = self.transcript\n        if hasattr(self, 'confidence') and self.confidence is not None:\n            _dict['confidence'] = self.confidence\n        if hasattr(self, 'timestamps') and self.timestamps is not None:\n            _dict['timestamps'] = self.timestamps\n        if hasattr(self,\n                   'word_confidence') and self.word_confidence is not None:\n            _dict['word_confidence'] = self.word_confidence\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ninitialize a SpeechRecognitionResult object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a SpeechRecognitionResult object from a json dictionary.\"\"\"\n        args = {}\n        if 'final' in _dict or 'final_results' in _dict:\n            args['final_results'] = _dict.get('final') or _dict.get(\n                'final_results')\n        else:\n            raise ValueError(\n                'Required property \\'final\\' not present in SpeechRecognitionResult JSON'\n            )\n        if 'alternatives' in _dict:\n            args['alternatives'] = [\n                SpeechRecognitionAlternative._from_dict(x)\n                for x in (_dict.get('alternatives'))\n            ]\n        else:\n            raise ValueError(\n                'Required property \\'alternatives\\' not present in SpeechRecognitionResult JSON'\n            )\n        if 'keywords_result' in _dict:\n            args['keywords_result'] = _dict.get('keywords_result')\n        if 'word_alternatives' in _dict:\n            args['word_alternatives'] = [\n                WordAlternativeResults._from_dict(x)\n                for x in (_dict.get('word_alternatives'))\n            ]\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'final_results') and self.final_results is not None:\n            _dict['final'] = self.final_results\n        if hasattr(self, 'alternatives') and self.alternatives is not None:\n            _dict['alternatives'] = [x._to_dict() for x in self.alternatives]\n        if hasattr(self,\n                   'keywords_result') and self.keywords_result is not None:\n            _dict['keywords_result'] = self.keywords_result\n        if hasattr(self,\n                   'word_alternatives') and self.word_alternatives is not None:\n            _dict['word_alternatives'] = [\n                x._to_dict() for x in self.word_alternatives\n            ]\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ninitializing a SpeechRecognitionResults object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a SpeechRecognitionResults object from a json dictionary.\"\"\"\n        args = {}\n        if 'results' in _dict:\n            args['results'] = [\n                SpeechRecognitionResult._from_dict(x)\n                for x in (_dict.get('results'))\n            ]\n        if 'result_index' in _dict:\n            args['result_index'] = _dict.get('result_index')\n        if 'speaker_labels' in _dict:\n            args['speaker_labels'] = [\n                SpeakerLabelsResult._from_dict(x)\n                for x in (_dict.get('speaker_labels'))\n            ]\n        if 'warnings' in _dict:\n            args['warnings'] = _dict.get('warnings')\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'results') and self.results is not None:\n            _dict['results'] = [x._to_dict() for x in self.results]\n        if hasattr(self, 'result_index') and self.result_index is not None:\n            _dict['result_index'] = self.result_index\n        if hasattr(self, 'speaker_labels') and self.speaker_labels is not None:\n            _dict['speaker_labels'] = [\n                x._to_dict() for x in self.speaker_labels\n            ]\n        if hasattr(self, 'warnings') and self.warnings is not None:\n            _dict['warnings'] = self.warnings\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ninitializes a SupportedFeatures object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a SupportedFeatures object from a json dictionary.\"\"\"\n        args = {}\n        if 'custom_language_model' in _dict:\n            args['custom_language_model'] = _dict.get('custom_language_model')\n        else:\n            raise ValueError(\n                'Required property \\'custom_language_model\\' not present in SupportedFeatures JSON'\n            )\n        if 'speaker_labels' in _dict:\n            args['speaker_labels'] = _dict.get('speaker_labels')\n        else:\n            raise ValueError(\n                'Required property \\'speaker_labels\\' not present in SupportedFeatures JSON'\n            )\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'custom_language_model'\n                  ) and self.custom_language_model is not None:\n            _dict['custom_language_model'] = self.custom_language_model\n        if hasattr(self, 'speaker_labels') and self.speaker_labels is not None:\n            _dict['speaker_labels'] = self.speaker_labels\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'word' in _dict:\n            args['word'] = _dict.get('word')\n        else:\n            raise ValueError(\n                'Required property \\'word\\' not present in Word JSON')\n        if 'sounds_like' in _dict:\n            args['sounds_like'] = _dict.get('sounds_like')\n        else:\n            raise ValueError(\n                'Required property \\'sounds_like\\' not present in Word JSON')\n        if 'display_as' in _dict:\n            args['display_as'] = _dict.get('display_as')\n        else:\n            raise ValueError(\n                'Required property \\'display_as\\' not present in Word JSON')\n        if 'count' in _dict:\n            args['count'] = _dict.get('count')\n        else:\n            raise ValueError(\n                'Required property \\'count\\' not present in Word JSON')\n        if 'source' in _dict:\n            args['source'] = _dict.get('source')\n        else:\n            raise ValueError(\n                'Required property \\'source\\' not present in Word JSON')\n        if 'error' in _dict:\n            args['error'] = [\n                WordError._from_dict(x) for x in (_dict.get('error'))\n            ]\n        return cls(**args)", "response": "Initialize a Word object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'word') and self.word is not None:\n            _dict['word'] = self.word\n        if hasattr(self, 'sounds_like') and self.sounds_like is not None:\n            _dict['sounds_like'] = self.sounds_like\n        if hasattr(self, 'display_as') and self.display_as is not None:\n            _dict['display_as'] = self.display_as\n        if hasattr(self, 'count') and self.count is not None:\n            _dict['count'] = self.count\n        if hasattr(self, 'source') and self.source is not None:\n            _dict['source'] = self.source\n        if hasattr(self, 'error') and self.error is not None:\n            _dict['error'] = [x._to_dict() for x in self.error]\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'confidence') and self.confidence is not None:\n            _dict['confidence'] = self.confidence\n        if hasattr(self, 'word') and self.word is not None:\n            _dict['word'] = self.word\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'start_time' in _dict:\n            args['start_time'] = _dict.get('start_time')\n        else:\n            raise ValueError(\n                'Required property \\'start_time\\' not present in WordAlternativeResults JSON'\n            )\n        if 'end_time' in _dict:\n            args['end_time'] = _dict.get('end_time')\n        else:\n            raise ValueError(\n                'Required property \\'end_time\\' not present in WordAlternativeResults JSON'\n            )\n        if 'alternatives' in _dict:\n            args['alternatives'] = [\n                WordAlternativeResult._from_dict(x)\n                for x in (_dict.get('alternatives'))\n            ]\n        else:\n            raise ValueError(\n                'Required property \\'alternatives\\' not present in WordAlternativeResults JSON'\n            )\n        return cls(**args)", "response": "Initialize a WordAlternativeResults object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'start_time') and self.start_time is not None:\n            _dict['start_time'] = self.start_time\n        if hasattr(self, 'end_time') and self.end_time is not None:\n            _dict['end_time'] = self.end_time\n        if hasattr(self, 'alternatives') and self.alternatives is not None:\n            _dict['alternatives'] = [x._to_dict() for x in self.alternatives]\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'element') and self.element is not None:\n            _dict['element'] = self.element\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'words' in _dict:\n            args['words'] = [Word._from_dict(x) for x in (_dict.get('words'))]\n        else:\n            raise ValueError(\n                'Required property \\'words\\' not present in Words JSON')\n        return cls(**args)", "response": "Initialize a Words object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'words') and self.words is not None:\n            _dict['words'] = [x._to_dict() for x in self.words]\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nupdate the user emotion tone with the primary emotion", "response": "def updateEmotionTone(user, emotionTone, maintainHistory):\n    \"\"\"\n    updateEmotionTone updates the user emotion tone with the primary emotion -\n    the emotion tone that has a score greater than or equal to the\n    EMOTION_SCORE_THRESHOLD; otherwise primary emotion will be 'neutral'\n    @param user a json object representing user information (tone) to be used in\n    conversing with the Conversation Service\n    @param emotionTone a json object containing the emotion tones in the payload\n    returned by the Tone Analyzer\n    \"\"\"\n    maxScore = 0.0\n    primaryEmotion = None\n    primaryEmotionScore = None\n\n    for tone in emotionTone['tones']:\n        if tone['score'] > maxScore:\n            maxScore = tone['score']\n            primaryEmotion = tone['tone_name'].lower()\n            primaryEmotionScore = tone['score']\n\n    if maxScore <= PRIMARY_EMOTION_SCORE_THRESHOLD:\n        primaryEmotion = 'neutral'\n        primaryEmotionScore = None\n\n    # update user emotion tone\n    user['tone']['emotion']['current'] = primaryEmotion\n\n    if maintainHistory:\n        if 'history' not in user['tone']['emotion']:\n            user['tone']['emotion']['history'] = []\n            user['tone']['emotion']['history'].append({\n                'tone_name': primaryEmotion,\n                'score': primaryEmotionScore\n            })"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nupdates the user with the writing tones interpreted based on the specified thresholds.", "response": "def updateWritingTone(user, writingTone, maintainHistory):\n    \"\"\"\n    updateWritingTone updates the user with the writing tones interpreted based\n    on the specified thresholds\n    @param: user a json object representing user information (tone) to be used\n    in conversing with the Conversation Service\n    @param: writingTone a json object containing the writing tones in the\n    payload returned by the Tone Analyzer\n    \"\"\"\n    currentWriting = []\n    currentWritingObject = []\n\n    # Process each writing tone and determine if it is high or low\n    for tone in writingTone['tones']:\n        if tone['score'] >= WRITING_HIGH_SCORE_THRESHOLD:\n            currentWriting.append(tone['tone_name'].lower() + '_high')\n            currentWritingObject.append({\n                'tone_name': tone['tone_name'].lower(),\n                'score': tone['score'],\n                'interpretation': 'likely high'\n            })\n        elif tone['score'] <= WRITING_NO_SCORE_THRESHOLD:\n            currentWritingObject.append({\n                'tone_name': tone['tone_name'].lower(),\n                'score': tone['score'],\n                'interpretation': 'no evidence'\n            })\n        else:\n            currentWritingObject.append({\n                'tone_name': tone['tone_name'].lower(),\n                'score': tone['score'],\n                'interpretation': 'likely medium'\n            })\n\n    # update user writing tone\n    user['tone']['writing']['current'] = currentWriting\n    if maintainHistory:\n        if 'history' not in user['tone']['writing']:\n            user['tone']['writing']['history'] = []\n    user['tone']['writing']['history'].append(currentWritingObject)"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nupdates the user with the social tones interpreted based on the specified thresholds.", "response": "def updateSocialTone(user, socialTone, maintainHistory):\n    \"\"\"\n    updateSocialTone updates the user with the social tones interpreted based on\n    the specified thresholds\n    @param user a json object representing user information (tone) to be used in\n    conversing with the Conversation Service\n    @param socialTone a json object containing the social tones in the payload\n    returned by the Tone Analyzer\n    \"\"\"\n    currentSocial = []\n    currentSocialObject = []\n\n    # Process each social tone and determine if it is high or low\n    for tone in socialTone['tones']:\n        if tone['score'] >= SOCIAL_HIGH_SCORE_THRESHOLD:\n            currentSocial.append(tone['tone_name'].lower() + '_high')\n            currentSocialObject.append({\n                'tone_name': tone['tone_name'].lower(),\n                'score': tone['score'],\n                'interpretation': 'likely high'\n            })\n        elif tone['score'] <= SOCIAL_LOW_SCORE_THRESHOLD:\n            currentSocial.append(tone['tone_name'].lower() + '_low')\n            currentSocialObject.append({\n                'tone_name': tone['tone_name'].lower(),\n                'score': tone['score'],\n                'interpretation': 'likely low'\n            })\n        else:\n            currentSocialObject.append({\n                'tone_name': tone['tone_name'].lower(),\n                'score': tone['score'],\n                'interpretation': 'likely medium'\n            })\n\n    # update user social tone\n    user['tone']['social']['current'] = currentSocial\n    if maintainHistory:\n        if not user['tone']['social']['current']:\n            user['tone']['social']['current'] = []\n        user['tone']['social']['current'].append(currentSocialObject)"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nsynthesizes a text to audio.", "response": "def synthesize(self,\n                   text,\n                   voice=None,\n                   customization_id=None,\n                   accept=None,\n                   **kwargs):\n        \"\"\"\n        Synthesize audio.\n\n        Synthesizes text to audio that is spoken in the specified voice. The service bases\n        its understanding of the language for the input text on the specified voice. Use a\n        voice that matches the language of the input text.\n        The method accepts a maximum of 5 KB of input text in the body of the request, and\n        8 KB for the URL and headers. The 5 KB limit includes any SSML tags that you\n        specify. The service returns the synthesized audio stream as an array of bytes.\n        **See also:** [The HTTP\n        interface](https://cloud.ibm.com/docs/services/text-to-speech/http.html).\n        ### Audio formats (accept types)\n         The service can return audio in the following formats (MIME types).\n        * Where indicated, you can optionally specify the sampling rate (`rate`) of the\n        audio. You must specify a sampling rate for the `audio/l16` and `audio/mulaw`\n        formats. A specified sampling rate must lie in the range of 8 kHz to 192 kHz.\n        * For the `audio/l16` format, you can optionally specify the endianness\n        (`endianness`) of the audio: `endianness=big-endian` or\n        `endianness=little-endian`.\n        Use the `Accept` header or the `accept` parameter to specify the requested format\n        of the response audio. If you omit an audio format altogether, the service returns\n        the audio in Ogg format with the Opus codec (`audio/ogg;codecs=opus`). The service\n        always returns single-channel audio.\n        * `audio/basic`\n          The service returns audio with a sampling rate of 8000 Hz.\n        * `audio/flac`\n          You can optionally specify the `rate` of the audio. The default sampling rate is\n        22,050 Hz.\n        * `audio/l16`\n          You must specify the `rate` of the audio. You can optionally specify the\n        `endianness` of the audio. The default endianness is `little-endian`.\n        * `audio/mp3`\n          You can optionally specify the `rate` of the audio. The default sampling rate is\n        22,050 Hz.\n        * `audio/mpeg`\n          You can optionally specify the `rate` of the audio. The default sampling rate is\n        22,050 Hz.\n        * `audio/mulaw`\n          You must specify the `rate` of the audio.\n        * `audio/ogg`\n          The service returns the audio in the `vorbis` codec. You can optionally specify\n        the `rate` of the audio. The default sampling rate is 22,050 Hz.\n        * `audio/ogg;codecs=opus`\n          You can optionally specify the `rate` of the audio. The default sampling rate is\n        22,050 Hz.\n        * `audio/ogg;codecs=vorbis`\n          You can optionally specify the `rate` of the audio. The default sampling rate is\n        22,050 Hz.\n        * `audio/wav`\n          You can optionally specify the `rate` of the audio. The default sampling rate is\n        22,050 Hz.\n        * `audio/webm`\n          The service returns the audio in the `opus` codec. The service returns audio\n        with a sampling rate of 48,000 Hz.\n        * `audio/webm;codecs=opus`\n          The service returns audio with a sampling rate of 48,000 Hz.\n        * `audio/webm;codecs=vorbis`\n          You can optionally specify the `rate` of the audio. The default sampling rate is\n        22,050 Hz.\n        For more information about specifying an audio format, including additional\n        details about some of the formats, see [Audio\n        formats](https://cloud.ibm.com/docs/services/text-to-speech/audio-formats.html).\n        ### Warning messages\n         If a request includes invalid query parameters, the service returns a `Warnings`\n        response header that provides messages about the invalid parameters. The warning\n        includes a descriptive message and a list of invalid argument strings. For\n        example, a message such as `\\\"Unknown arguments:\\\"` or `\\\"Unknown url query\n        arguments:\\\"` followed by a list of the form `\\\"{invalid_arg_1},\n        {invalid_arg_2}.\\\"` The request succeeds despite the warnings.\n\n        :param str text: The text to synthesize.\n        :param str voice: The voice to use for synthesis.\n        :param str customization_id: The customization ID (GUID) of a custom voice model\n        to use for the synthesis. If a custom voice model is specified, it is guaranteed\n        to work only if it matches the language of the indicated voice. You must make the\n        request with service credentials created for the instance of the service that owns\n        the custom model. Omit the parameter to use the specified voice with no\n        customization.\n        :param str accept: The requested format (MIME type) of the audio. You can use the\n        `Accept` header or the `accept` parameter to specify the audio format. For more\n        information about specifying an audio format, see **Audio formats (accept types)**\n        in the method description.\n        Default: `audio/ogg;codecs=opus`.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse\n        \"\"\"\n\n        if text is None:\n            raise ValueError('text must be provided')\n\n        headers = {'Accept': accept}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('text_to_speech', 'V1', 'synthesize')\n        headers.update(sdk_headers)\n\n        params = {'voice': voice, 'customization_id': customization_id}\n\n        data = {'text': text}\n\n        url = '/v1/synthesize'\n        response = self.request(\n            method='POST',\n            url=url,\n            headers=headers,\n            params=params,\n            json=data,\n            accept_json=False)\n        return response"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nupdate the custom voice model with the specified customization ID name description and words.", "response": "def update_voice_model(self,\n                           customization_id,\n                           name=None,\n                           description=None,\n                           words=None,\n                           **kwargs):\n        \"\"\"\n        Update a custom model.\n\n        Updates information for the specified custom voice model. You can update metadata\n        such as the name and description of the voice model. You can also update the words\n        in the model and their translations. Adding a new translation for a word that\n        already exists in a custom model overwrites the word's existing translation. A\n        custom model can contain no more than 20,000 entries. You must use credentials for\n        the instance of the service that owns a model to update it.\n        You can define sounds-like or phonetic translations for words. A sounds-like\n        translation consists of one or more words that, when combined, sound like the\n        word. Phonetic translations are based on the SSML phoneme format for representing\n        a word. You can specify them in standard International Phonetic Alphabet (IPA)\n        representation\n          <code>&lt;phoneme alphabet=\\\"ipa\\\"\n        ph=\\\"t&#601;m&#712;&#593;to\\\"&gt;&lt;/phoneme&gt;</code>\n          or in the proprietary IBM Symbolic Phonetic Representation (SPR)\n          <code>&lt;phoneme alphabet=\\\"ibm\\\"\n        ph=\\\"1gAstroEntxrYFXs\\\"&gt;&lt;/phoneme&gt;</code>\n        **Note:** This method is currently a beta release.\n        **See also:**\n        * [Updating a custom\n        model](https://cloud.ibm.com/docs/services/text-to-speech/custom-models.html#cuModelsUpdate)\n        * [Adding words to a Japanese custom\n        model](https://cloud.ibm.com/docs/services/text-to-speech/custom-entries.html#cuJapaneseAdd)\n        * [Understanding\n        customization](https://cloud.ibm.com/docs/services/text-to-speech/custom-intro.html).\n\n        :param str customization_id: The customization ID (GUID) of the custom voice\n        model. You must make the request with service credentials created for the instance\n        of the service that owns the custom model.\n        :param str name: A new name for the custom voice model.\n        :param str description: A new description for the custom voice model.\n        :param list[Word] words: An array of `Word` objects that provides the words and\n        their translations that are to be added or updated for the custom voice model.\n        Pass an empty array to make no additions or updates.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse\n        \"\"\"\n\n        if customization_id is None:\n            raise ValueError('customization_id must be provided')\n        if words is not None:\n            words = [self._convert_model(x, Word) for x in words]\n\n        headers = {}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('text_to_speech', 'V1',\n                                      'update_voice_model')\n        headers.update(sdk_headers)\n\n        data = {'name': name, 'description': description, 'words': words}\n\n        url = '/v1/customizations/{0}'.format(\n            *self._encode_path_vars(customization_id))\n        response = self.request(\n            method='POST',\n            url=url,\n            headers=headers,\n            json=data,\n            accept_json=True)\n        return response"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nadd a word to the specified custom model.", "response": "def add_word(self,\n                 customization_id,\n                 word,\n                 translation,\n                 part_of_speech=None,\n                 **kwargs):\n        \"\"\"\n        Add a custom word.\n\n        Adds a single word and its translation to the specified custom voice model. Adding\n        a new translation for a word that already exists in a custom model overwrites the\n        word's existing translation. A custom model can contain no more than 20,000\n        entries. You must use credentials for the instance of the service that owns a\n        model to add a word to it.\n        You can define sounds-like or phonetic translations for words. A sounds-like\n        translation consists of one or more words that, when combined, sound like the\n        word. Phonetic translations are based on the SSML phoneme format for representing\n        a word. You can specify them in standard International Phonetic Alphabet (IPA)\n        representation\n          <code>&lt;phoneme alphabet=\\\"ipa\\\"\n        ph=\\\"t&#601;m&#712;&#593;to\\\"&gt;&lt;/phoneme&gt;</code>\n          or in the proprietary IBM Symbolic Phonetic Representation (SPR)\n          <code>&lt;phoneme alphabet=\\\"ibm\\\"\n        ph=\\\"1gAstroEntxrYFXs\\\"&gt;&lt;/phoneme&gt;</code>\n        **Note:** This method is currently a beta release.\n        **See also:**\n        * [Adding a single word to a custom\n        model](https://cloud.ibm.com/docs/services/text-to-speech/custom-entries.html#cuWordAdd)\n        * [Adding words to a Japanese custom\n        model](https://cloud.ibm.com/docs/services/text-to-speech/custom-entries.html#cuJapaneseAdd)\n        * [Understanding\n        customization](https://cloud.ibm.com/docs/services/text-to-speech/custom-intro.html).\n\n        :param str customization_id: The customization ID (GUID) of the custom voice\n        model. You must make the request with service credentials created for the instance\n        of the service that owns the custom model.\n        :param str word: The word that is to be added or updated for the custom voice\n        model.\n        :param str translation: The phonetic or sounds-like translation for the word. A\n        phonetic translation is based on the SSML format for representing the phonetic\n        string of a word either as an IPA translation or as an IBM SPR translation. A\n        sounds-like is one or more words that, when combined, sound like the word.\n        :param str part_of_speech: **Japanese only.** The part of speech for the word. The\n        service uses the value to produce the correct intonation for the word. You can\n        create only a single entry, with or without a single part of speech, for any word;\n        you cannot create multiple entries with different parts of speech for the same\n        word. For more information, see [Working with Japanese\n        entries](https://cloud.ibm.com/docs/services/text-to-speech/custom-rules.html#jaNotes).\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse\n        \"\"\"\n\n        if customization_id is None:\n            raise ValueError('customization_id must be provided')\n        if word is None:\n            raise ValueError('word must be provided')\n        if translation is None:\n            raise ValueError('translation must be provided')\n\n        headers = {}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('text_to_speech', 'V1', 'add_word')\n        headers.update(sdk_headers)\n\n        data = {'translation': translation, 'part_of_speech': part_of_speech}\n\n        url = '/v1/customizations/{0}/words/{1}'.format(\n            *self._encode_path_vars(customization_id, word))\n        response = self.request(\n            method='PUT',\n            url=url,\n            headers=headers,\n            json=data,\n            accept_json=False)\n        return response"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ndelete labeled data. Deletes all data that is associated with a specified customer ID. The method deletes all data for the customer ID, regardless of the method by which the information was added. The method has no effect if no data is associated with the customer ID. You must issue the request with credentials for the same instance of the service that was used to associate the customer ID with the data. You associate a customer ID with data by passing the `X-Watson-Metadata` header with a request that passes the data. **See also:** [Information security](https://cloud.ibm.com/docs/services/text-to-speech/information-security.html). :param str customer_id: The customer ID for which all data is to be deleted. :param dict headers: A `dict` containing the request headers :return: A `DetailedResponse` containing the result, headers and HTTP status code. :rtype: DetailedResponse", "response": "def delete_user_data(self, customer_id, **kwargs):\n        \"\"\"\n        Delete labeled data.\n\n        Deletes all data that is associated with a specified customer ID. The method\n        deletes all data for the customer ID, regardless of the method by which the\n        information was added. The method has no effect if no data is associated with the\n        customer ID. You must issue the request with credentials for the same instance of\n        the service that was used to associate the customer ID with the data.\n        You associate a customer ID with data by passing the `X-Watson-Metadata` header\n        with a request that passes the data.\n        **See also:** [Information\n        security](https://cloud.ibm.com/docs/services/text-to-speech/information-security.html).\n\n        :param str customer_id: The customer ID for which all data is to be deleted.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse\n        \"\"\"\n\n        if customer_id is None:\n            raise ValueError('customer_id must be provided')\n\n        headers = {}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('text_to_speech', 'V1',\n                                      'delete_user_data')\n        headers.update(sdk_headers)\n\n        params = {'customer_id': customer_id}\n\n        url = '/v1/user_data'\n        response = self.request(\n            method='DELETE',\n            url=url,\n            headers=headers,\n            params=params,\n            accept_json=False)\n        return response"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'pronunciation') and self.pronunciation is not None:\n            _dict['pronunciation'] = self.pronunciation\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'custom_pronunciation'\n                  ) and self.custom_pronunciation is not None:\n            _dict['custom_pronunciation'] = self.custom_pronunciation\n        if hasattr(self, 'voice_transformation'\n                  ) and self.voice_transformation is not None:\n            _dict['voice_transformation'] = self.voice_transformation\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'url') and self.url is not None:\n            _dict['url'] = self.url\n        if hasattr(self, 'gender') and self.gender is not None:\n            _dict['gender'] = self.gender\n        if hasattr(self, 'name') and self.name is not None:\n            _dict['name'] = self.name\n        if hasattr(self, 'language') and self.language is not None:\n            _dict['language'] = self.language\n        if hasattr(self, 'description') and self.description is not None:\n            _dict['description'] = self.description\n        if hasattr(self, 'customizable') and self.customizable is not None:\n            _dict['customizable'] = self.customizable\n        if hasattr(\n                self,\n                'supported_features') and self.supported_features is not None:\n            _dict['supported_features'] = self.supported_features._to_dict()\n        if hasattr(self, 'customization') and self.customization is not None:\n            _dict['customization'] = self.customization._to_dict()\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ninitialize a VoiceModel object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a VoiceModel object from a json dictionary.\"\"\"\n        args = {}\n        if 'customization_id' in _dict:\n            args['customization_id'] = _dict.get('customization_id')\n        else:\n            raise ValueError(\n                'Required property \\'customization_id\\' not present in VoiceModel JSON'\n            )\n        if 'name' in _dict:\n            args['name'] = _dict.get('name')\n        if 'language' in _dict:\n            args['language'] = _dict.get('language')\n        if 'owner' in _dict:\n            args['owner'] = _dict.get('owner')\n        if 'created' in _dict:\n            args['created'] = _dict.get('created')\n        if 'last_modified' in _dict:\n            args['last_modified'] = _dict.get('last_modified')\n        if 'description' in _dict:\n            args['description'] = _dict.get('description')\n        if 'words' in _dict:\n            args['words'] = [Word._from_dict(x) for x in (_dict.get('words'))]\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self,\n                   'customization_id') and self.customization_id is not None:\n            _dict['customization_id'] = self.customization_id\n        if hasattr(self, 'name') and self.name is not None:\n            _dict['name'] = self.name\n        if hasattr(self, 'language') and self.language is not None:\n            _dict['language'] = self.language\n        if hasattr(self, 'owner') and self.owner is not None:\n            _dict['owner'] = self.owner\n        if hasattr(self, 'created') and self.created is not None:\n            _dict['created'] = self.created\n        if hasattr(self, 'last_modified') and self.last_modified is not None:\n            _dict['last_modified'] = self.last_modified\n        if hasattr(self, 'description') and self.description is not None:\n            _dict['description'] = self.description\n        if hasattr(self, 'words') and self.words is not None:\n            _dict['words'] = [x._to_dict() for x in self.words]\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'customizations' in _dict:\n            args['customizations'] = [\n                VoiceModel._from_dict(x) for x in (_dict.get('customizations'))\n            ]\n        else:\n            raise ValueError(\n                'Required property \\'customizations\\' not present in VoiceModels JSON'\n            )\n        return cls(**args)", "response": "Initialize a VoiceModels object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ninitializes a Voices object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a Voices object from a json dictionary.\"\"\"\n        args = {}\n        if 'voices' in _dict:\n            args['voices'] = [\n                Voice._from_dict(x) for x in (_dict.get('voices'))\n            ]\n        else:\n            raise ValueError(\n                'Required property \\'voices\\' not present in Voices JSON')\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'voices') and self.voices is not None:\n            _dict['voices'] = [x._to_dict() for x in self.voices]\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'word') and self.word is not None:\n            _dict['word'] = self.word\n        if hasattr(self, 'translation') and self.translation is not None:\n            _dict['translation'] = self.translation\n        if hasattr(self, 'part_of_speech') and self.part_of_speech is not None:\n            _dict['part_of_speech'] = self.part_of_speech\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ninvoke Tone Analyzer service to get the tone information for the user s input text and update the user s tone in the payload.", "response": "def invokeToneConversation(payload, maintainToneHistoryInContext):\n    \"\"\"\n     invokeToneConversation calls the Tone Analyzer service to get the\n     tone information for the user's input text (input['text'] in the payload\n     json object), adds/updates the user's tone in the payload's context,\n     and sends the payload to the\n     conversation service to get a response which is printed to screen.\n     :param payload: a json object containing the basic information needed to\n     converse with the Conversation Service's message endpoint.\n     :param maintainHistoryInContext:\n\n\n     Note: as indicated below, the console.log statements can be replaced\n     with application-specific code to process the err or data object\n     returned by the Conversation Service.\n    \"\"\"\n    tone = tone_analyzer.tone(tone_input=payload['input'], content_type='application/json').get_result()\n    conversation_payload = tone_detection.\\\n        updateUserTone(payload, tone, maintainToneHistoryInContext)\n    response = assistant.message(workspace_id=workspace_id,\n                                 input=conversation_payload['input'],\n                                 context=conversation_payload['context']).get_result()\n    print(json.dumps(response, indent=2))"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nconvert a document to HTML.", "response": "def convert_to_html(self,\n                        file,\n                        filename=None,\n                        file_content_type=None,\n                        model=None,\n                        **kwargs):\n        \"\"\"\n        Convert document to HTML.\n\n        Converts a document to HTML.\n\n        :param file file: The document to convert.\n        :param str filename: The filename for file.\n        :param str file_content_type: The content type of file.\n        :param str model: The analysis model to be used by the service. For the **Element\n        classification** and **Compare two documents** methods, the default is\n        `contracts`. For the **Extract tables** method, the default is `tables`. These\n        defaults apply to the standalone methods as well as to the methods' use in\n        batch-processing requests.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse\n        \"\"\"\n\n        if file is None:\n            raise ValueError('file must be provided')\n\n        headers = {}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('compare-comply', 'V1', 'convert_to_html')\n        headers.update(sdk_headers)\n\n        params = {'version': self.version, 'model': model}\n\n        form_data = {}\n        if not filename and hasattr(file, 'name'):\n            filename = basename(file.name)\n        if not filename:\n            raise ValueError('filename must be provided')\n        form_data['file'] = (filename, file, file_content_type or\n                             'application/octet-stream')\n\n        url = '/v1/html_conversion'\n        response = self.request(\n            method='POST',\n            url=url,\n            headers=headers,\n            params=params,\n            files=form_data,\n            accept_json=True)\n        return response"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef classify_elements(self,\n                          file,\n                          file_content_type=None,\n                          model=None,\n                          **kwargs):\n        \"\"\"\n        Classify the elements of a document.\n\n        Analyzes the structural and semantic elements of a document.\n\n        :param file file: The document to classify.\n        :param str file_content_type: The content type of file.\n        :param str model: The analysis model to be used by the service. For the **Element\n        classification** and **Compare two documents** methods, the default is\n        `contracts`. For the **Extract tables** method, the default is `tables`. These\n        defaults apply to the standalone methods as well as to the methods' use in\n        batch-processing requests.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse\n        \"\"\"\n\n        if file is None:\n            raise ValueError('file must be provided')\n\n        headers = {}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('compare-comply', 'V1',\n                                      'classify_elements')\n        headers.update(sdk_headers)\n\n        params = {'version': self.version, 'model': model}\n\n        form_data = {}\n        form_data['file'] = (None, file, file_content_type or\n                             'application/octet-stream')\n\n        url = '/v1/element_classification'\n        response = self.request(\n            method='POST',\n            url=url,\n            headers=headers,\n            params=params,\n            files=form_data,\n            accept_json=True)\n        return response", "response": "Classify the elements of a document."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncompare two input documents.", "response": "def compare_documents(self,\n                          file_1,\n                          file_2,\n                          file_1_content_type=None,\n                          file_2_content_type=None,\n                          file_1_label=None,\n                          file_2_label=None,\n                          model=None,\n                          **kwargs):\n        \"\"\"\n        Compare two documents.\n\n        Compares two input documents. Documents must be in the same format.\n\n        :param file file_1: The first document to compare.\n        :param file file_2: The second document to compare.\n        :param str file_1_content_type: The content type of file_1.\n        :param str file_2_content_type: The content type of file_2.\n        :param str file_1_label: A text label for the first document.\n        :param str file_2_label: A text label for the second document.\n        :param str model: The analysis model to be used by the service. For the **Element\n        classification** and **Compare two documents** methods, the default is\n        `contracts`. For the **Extract tables** method, the default is `tables`. These\n        defaults apply to the standalone methods as well as to the methods' use in\n        batch-processing requests.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse\n        \"\"\"\n\n        if file_1 is None:\n            raise ValueError('file_1 must be provided')\n        if file_2 is None:\n            raise ValueError('file_2 must be provided')\n\n        headers = {}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('compare-comply', 'V1',\n                                      'compare_documents')\n        headers.update(sdk_headers)\n\n        params = {\n            'version': self.version,\n            'file_1_label': file_1_label,\n            'file_2_label': file_2_label,\n            'model': model\n        }\n\n        form_data = {}\n        form_data['file_1'] = (None, file_1, file_1_content_type or\n                               'application/octet-stream')\n        form_data['file_2'] = (None, file_2, file_2_content_type or\n                               'application/octet-stream')\n\n        url = '/v1/comparison'\n        response = self.request(\n            method='POST',\n            url=url,\n            headers=headers,\n            params=params,\n            files=form_data,\n            accept_json=True)\n        return response"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nadd feedback to a specific entry in the training model.", "response": "def add_feedback(self, feedback_data, user_id=None, comment=None, **kwargs):\n        \"\"\"\n        Add feedback.\n\n        Adds feedback in the form of _labels_ from a subject-matter expert (SME) to a\n        governing document.\n        **Important:** Feedback is not immediately incorporated into the training model,\n        nor is it guaranteed to be incorporated at a later date. Instead, submitted\n        feedback is used to suggest future updates to the training model.\n\n        :param FeedbackDataInput feedback_data: Feedback data for submission.\n        :param str user_id: An optional string identifying the user.\n        :param str comment: An optional comment on or description of the feedback.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse\n        \"\"\"\n\n        if feedback_data is None:\n            raise ValueError('feedback_data must be provided')\n        feedback_data = self._convert_model(feedback_data, FeedbackDataInput)\n\n        headers = {}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('compare-comply', 'V1', 'add_feedback')\n        headers.update(sdk_headers)\n\n        params = {'version': self.version}\n\n        data = {\n            'feedback_data': feedback_data,\n            'user_id': user_id,\n            'comment': comment\n        }\n\n        url = '/v1/feedback'\n        response = self.request(\n            method='POST',\n            url=url,\n            headers=headers,\n            params=params,\n            json=data,\n            accept_json=True)\n        return response"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get_feedback(self, feedback_id, model=None, **kwargs):\n\n        if feedback_id is None:\n            raise ValueError('feedback_id must be provided')\n\n        headers = {}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('compare-comply', 'V1', 'get_feedback')\n        headers.update(sdk_headers)\n\n        params = {'version': self.version, 'model': model}\n\n        url = '/v1/feedback/{0}'.format(*self._encode_path_vars(feedback_id))\n        response = self.request(\n            method='GET',\n            url=url,\n            headers=headers,\n            params=params,\n            accept_json=True)\n        return response", "response": "Get a specific feedback entry."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nlist the feedback in a document. Lists the feedback in a document. :param str feedback_type: An optional string that filters the output to include only feedback with the specified feedback type. The only permitted value is `element_classification`. :param date before: An optional string in the format `YYYY-MM-DD` that filters the output to include only feedback that was added before the specified date. :param date after: An optional string in the format `YYYY-MM-DD` that filters the output to include only feedback that was added after the specified date. :param str document_title: An optional string that filters the output to include only feedback from the document with the specified `document_title`. :param str model_id: An optional string that filters the output to include only feedback with the specified `model_id`. The only permitted value is `contracts`. :param str model_version: An optional string that filters the output to include only feedback with the specified `model_version`. :param str category_removed: An optional string in the form of a comma-separated list of categories. If this is specified, the service filters the output to include only feedback that has at least one category from the list removed. :param str category_added: An optional string in the form of a comma-separated list of categories. If this is specified, the service filters the output to include only feedback that has at least one category from the list added. :param str category_not_changed: An optional string in the form of a comma-separated list of categories. If this is specified, the service filters the output to include only feedback that has at least one category from the list unchanged. :param str type_removed: An optional string of comma-separated `nature`:`party` pairs. If this is specified, the service filters the output to include only feedback that has at least one `nature`:`party` pair from the list removed. :param str type_added: An optional string of comma-separated `nature`:`party` pairs. If this is specified, the service filters the output to include only feedback that has at least one `nature`:`party` pair from the list removed. :param str type_not_changed: An optional string of comma-separated `nature`:`party` pairs. If this is specified, the service filters the output to include only feedback that has at least one `nature`:`party` pair from the list unchanged. :param int page_limit: An optional integer specifying the number of documents that you want the service to return. :param str cursor: An optional string that returns the set of documents after the previous set. Use this parameter with the `page_limit` parameter. :param str sort: An optional comma-separated list of fields in the document to sort on. You can optionally specify the sort direction by prefixing the value of the field with `-` for descending order or `+` for ascending order (the default). Currently permitted sorting fields are `created`, `user_id`, and `document_title`. :param bool include_total: An optional boolean value. If specified as `true`, the `pagination` object in the output includes a value called `total` that gives the total count of feedback created. :param dict headers: A `dict` containing the request headers :return: A `DetailedResponse` containing the result, headers and HTTP status code. :rtype: DetailedResponse", "response": "def list_feedback(self,\n                      feedback_type=None,\n                      before=None,\n                      after=None,\n                      document_title=None,\n                      model_id=None,\n                      model_version=None,\n                      category_removed=None,\n                      category_added=None,\n                      category_not_changed=None,\n                      type_removed=None,\n                      type_added=None,\n                      type_not_changed=None,\n                      page_limit=None,\n                      cursor=None,\n                      sort=None,\n                      include_total=None,\n                      **kwargs):\n        \"\"\"\n        List the feedback in a document.\n\n        Lists the feedback in a document.\n\n        :param str feedback_type: An optional string that filters the output to include\n        only feedback with the specified feedback type. The only permitted value is\n        `element_classification`.\n        :param date before: An optional string in the format `YYYY-MM-DD` that filters the\n        output to include only feedback that was added before the specified date.\n        :param date after: An optional string in the format `YYYY-MM-DD` that filters the\n        output to include only feedback that was added after the specified date.\n        :param str document_title: An optional string that filters the output to include\n        only feedback from the document with the specified `document_title`.\n        :param str model_id: An optional string that filters the output to include only\n        feedback with the specified `model_id`. The only permitted value is `contracts`.\n        :param str model_version: An optional string that filters the output to include\n        only feedback with the specified `model_version`.\n        :param str category_removed: An optional string in the form of a comma-separated\n        list of categories. If this is specified, the service filters the output to\n        include only feedback that has at least one category from the list removed.\n        :param str category_added: An optional string in the form of a comma-separated\n        list of categories. If this is specified, the service filters the output to\n        include only feedback that has at least one category from the list added.\n        :param str category_not_changed: An optional string in the form of a\n        comma-separated list of categories. If this is specified, the service filters the\n        output to include only feedback that has at least one category from the list\n        unchanged.\n        :param str type_removed: An optional string of comma-separated `nature`:`party`\n        pairs. If this is specified, the service filters the output to include only\n        feedback that has at least one `nature`:`party` pair from the list removed.\n        :param str type_added: An optional string of comma-separated `nature`:`party`\n        pairs. If this is specified, the service filters the output to include only\n        feedback that has at least one `nature`:`party` pair from the list removed.\n        :param str type_not_changed: An optional string of comma-separated\n        `nature`:`party` pairs. If this is specified, the service filters the output to\n        include only feedback that has at least one `nature`:`party` pair from the list\n        unchanged.\n        :param int page_limit: An optional integer specifying the number of documents that\n        you want the service to return.\n        :param str cursor: An optional string that returns the set of documents after the\n        previous set. Use this parameter with the `page_limit` parameter.\n        :param str sort: An optional comma-separated list of fields in the document to\n        sort on. You can optionally specify the sort direction by prefixing the value of\n        the field with `-` for descending order or `+` for ascending order (the default).\n        Currently permitted sorting fields are `created`, `user_id`, and `document_title`.\n        :param bool include_total: An optional boolean value. If specified as `true`, the\n        `pagination` object in the output includes a value called `total` that gives the\n        total count of feedback created.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse\n        \"\"\"\n\n        headers = {}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('compare-comply', 'V1', 'list_feedback')\n        headers.update(sdk_headers)\n\n        params = {\n            'version': self.version,\n            'feedback_type': feedback_type,\n            'before': before,\n            'after': after,\n            'document_title': document_title,\n            'model_id': model_id,\n            'model_version': model_version,\n            'category_removed': category_removed,\n            'category_added': category_added,\n            'category_not_changed': category_not_changed,\n            'type_removed': type_removed,\n            'type_added': type_added,\n            'type_not_changed': type_not_changed,\n            'page_limit': page_limit,\n            'cursor': cursor,\n            'sort': sort,\n            'include_total': include_total\n        }\n\n        url = '/v1/feedback'\n        response = self.request(\n            method='GET',\n            url=url,\n            headers=headers,\n            params=params,\n            accept_json=True)\n        return response"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nsubmitting a batch - processing request for the given function and the given input document sets.", "response": "def create_batch(self,\n                     function,\n                     input_credentials_file,\n                     input_bucket_location,\n                     input_bucket_name,\n                     output_credentials_file,\n                     output_bucket_location,\n                     output_bucket_name,\n                     model=None,\n                     **kwargs):\n        \"\"\"\n        Submit a batch-processing request.\n\n        Run Compare and Comply methods over a collection of input documents.\n        **Important:** Batch processing requires the use of the [IBM Cloud Object Storage\n        service](https://cloud.ibm.com/docs/services/cloud-object-storage/about-cos.html#about-ibm-cloud-object-storage).\n        The use of IBM Cloud Object Storage with Compare and Comply is discussed at [Using\n        batch\n        processing](https://cloud.ibm.com/docs/services/compare-comply/batching.html#before-you-batch).\n\n        :param str function: The Compare and Comply method to run across the submitted\n        input documents.\n        :param file input_credentials_file: A JSON file containing the input Cloud Object\n        Storage credentials. At a minimum, the credentials must enable `READ` permissions\n        on the bucket defined by the `input_bucket_name` parameter.\n        :param str input_bucket_location: The geographical location of the Cloud Object\n        Storage input bucket as listed on the **Endpoint** tab of your Cloud Object\n        Storage instance; for example, `us-geo`, `eu-geo`, or `ap-geo`.\n        :param str input_bucket_name: The name of the Cloud Object Storage input bucket.\n        :param file output_credentials_file: A JSON file that lists the Cloud Object\n        Storage output credentials. At a minimum, the credentials must enable `READ` and\n        `WRITE` permissions on the bucket defined by the `output_bucket_name` parameter.\n        :param str output_bucket_location: The geographical location of the Cloud Object\n        Storage output bucket as listed on the **Endpoint** tab of your Cloud Object\n        Storage instance; for example, `us-geo`, `eu-geo`, or `ap-geo`.\n        :param str output_bucket_name: The name of the Cloud Object Storage output bucket.\n        :param str model: The analysis model to be used by the service. For the **Element\n        classification** and **Compare two documents** methods, the default is\n        `contracts`. For the **Extract tables** method, the default is `tables`. These\n        defaults apply to the standalone methods as well as to the methods' use in\n        batch-processing requests.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse\n        \"\"\"\n\n        if function is None:\n            raise ValueError('function must be provided')\n        if input_credentials_file is None:\n            raise ValueError('input_credentials_file must be provided')\n        if input_bucket_location is None:\n            raise ValueError('input_bucket_location must be provided')\n        if input_bucket_name is None:\n            raise ValueError('input_bucket_name must be provided')\n        if output_credentials_file is None:\n            raise ValueError('output_credentials_file must be provided')\n        if output_bucket_location is None:\n            raise ValueError('output_bucket_location must be provided')\n        if output_bucket_name is None:\n            raise ValueError('output_bucket_name must be provided')\n\n        headers = {}\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('compare-comply', 'V1', 'create_batch')\n        headers.update(sdk_headers)\n\n        params = {'version': self.version, 'function': function, 'model': model}\n\n        form_data = {}\n        form_data['input_credentials_file'] = (None, input_credentials_file,\n                                               'application/json')\n        form_data['input_bucket_location'] = (None, input_bucket_location,\n                                              'text/plain')\n        form_data['input_bucket_name'] = (None, input_bucket_name, 'text/plain')\n        form_data['output_credentials_file'] = (None, output_credentials_file,\n                                                'application/json')\n        form_data['output_bucket_location'] = (None, output_bucket_location,\n                                               'text/plain')\n        form_data['output_bucket_name'] = (None, output_bucket_name,\n                                           'text/plain')\n\n        url = '/v1/batches'\n        response = self.request(\n            method='POST',\n            url=url,\n            headers=headers,\n            params=params,\n            files=form_data,\n            accept_json=True)\n        return response"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ninitialize a AlignedElement object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a AlignedElement object from a json dictionary.\"\"\"\n        args = {}\n        if 'element_pair' in _dict:\n            args['element_pair'] = [\n                ElementPair._from_dict(x) for x in (_dict.get('element_pair'))\n            ]\n        if 'identical_text' in _dict:\n            args['identical_text'] = _dict.get('identical_text')\n        if 'provenance_ids' in _dict:\n            args['provenance_ids'] = _dict.get('provenance_ids')\n        if 'significant_elements' in _dict:\n            args['significant_elements'] = _dict.get('significant_elements')\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'element_pair') and self.element_pair is not None:\n            _dict['element_pair'] = [x._to_dict() for x in self.element_pair]\n        if hasattr(self, 'identical_text') and self.identical_text is not None:\n            _dict['identical_text'] = self.identical_text\n        if hasattr(self, 'provenance_ids') and self.provenance_ids is not None:\n            _dict['provenance_ids'] = self.provenance_ids\n        if hasattr(self, 'significant_elements'\n                  ) and self.significant_elements is not None:\n            _dict['significant_elements'] = self.significant_elements\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a json dictionary representing this object.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'function') and self.function is not None:\n            _dict['function'] = self.function\n        if hasattr(self, 'input_bucket_location'\n                  ) and self.input_bucket_location is not None:\n            _dict['input_bucket_location'] = self.input_bucket_location\n        if hasattr(self,\n                   'input_bucket_name') and self.input_bucket_name is not None:\n            _dict['input_bucket_name'] = self.input_bucket_name\n        if hasattr(self, 'output_bucket_location'\n                  ) and self.output_bucket_location is not None:\n            _dict['output_bucket_location'] = self.output_bucket_location\n        if hasattr(\n                self,\n                'output_bucket_name') and self.output_bucket_name is not None:\n            _dict['output_bucket_name'] = self.output_bucket_name\n        if hasattr(self, 'batch_id') and self.batch_id is not None:\n            _dict['batch_id'] = self.batch_id\n        if hasattr(self,\n                   'document_counts') and self.document_counts is not None:\n            _dict['document_counts'] = self.document_counts._to_dict()\n        if hasattr(self, 'status') and self.status is not None:\n            _dict['status'] = self.status\n        if hasattr(self, 'created') and self.created is not None:\n            _dict['created'] = datetime_to_string(self.created)\n        if hasattr(self, 'updated') and self.updated is not None:\n            _dict['updated'] = datetime_to_string(self.updated)\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ninitialize a Batches object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a Batches object from a json dictionary.\"\"\"\n        args = {}\n        if 'batches' in _dict:\n            args['batches'] = [\n                BatchStatus._from_dict(x) for x in (_dict.get('batches'))\n            ]\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'batches') and self.batches is not None:\n            _dict['batches'] = [x._to_dict() for x in self.batches]\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'cell_id' in _dict:\n            args['cell_id'] = _dict.get('cell_id')\n        if 'location' in _dict:\n            args['location'] = Location._from_dict(_dict.get('location'))\n        if 'text' in _dict:\n            args['text'] = _dict.get('text')\n        if 'row_index_begin' in _dict:\n            args['row_index_begin'] = _dict.get('row_index_begin')\n        if 'row_index_end' in _dict:\n            args['row_index_end'] = _dict.get('row_index_end')\n        if 'column_index_begin' in _dict:\n            args['column_index_begin'] = _dict.get('column_index_begin')\n        if 'column_index_end' in _dict:\n            args['column_index_end'] = _dict.get('column_index_end')\n        if 'row_header_ids' in _dict:\n            args['row_header_ids'] = [\n                RowHeaderIds._from_dict(x)\n                for x in (_dict.get('row_header_ids'))\n            ]\n        if 'row_header_texts' in _dict:\n            args['row_header_texts'] = [\n                RowHeaderTexts._from_dict(x)\n                for x in (_dict.get('row_header_texts'))\n            ]\n        if 'row_header_texts_normalized' in _dict:\n            args['row_header_texts_normalized'] = [\n                RowHeaderTextsNormalized._from_dict(x)\n                for x in (_dict.get('row_header_texts_normalized'))\n            ]\n        if 'column_header_ids' in _dict:\n            args['column_header_ids'] = [\n                ColumnHeaderIds._from_dict(x)\n                for x in (_dict.get('column_header_ids'))\n            ]\n        if 'column_header_texts' in _dict:\n            args['column_header_texts'] = [\n                ColumnHeaderTexts._from_dict(x)\n                for x in (_dict.get('column_header_texts'))\n            ]\n        if 'column_header_texts_normalized' in _dict:\n            args['column_header_texts_normalized'] = [\n                ColumnHeaderTextsNormalized._from_dict(x)\n                for x in (_dict.get('column_header_texts_normalized'))\n            ]\n        if 'attributes' in _dict:\n            args['attributes'] = [\n                Attribute._from_dict(x) for x in (_dict.get('attributes'))\n            ]\n        return cls(**args)", "response": "Initialize a BodyCells object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'cell_id') and self.cell_id is not None:\n            _dict['cell_id'] = self.cell_id\n        if hasattr(self, 'location') and self.location is not None:\n            _dict['location'] = self.location._to_dict()\n        if hasattr(self, 'text') and self.text is not None:\n            _dict['text'] = self.text\n        if hasattr(self,\n                   'row_index_begin') and self.row_index_begin is not None:\n            _dict['row_index_begin'] = self.row_index_begin\n        if hasattr(self, 'row_index_end') and self.row_index_end is not None:\n            _dict['row_index_end'] = self.row_index_end\n        if hasattr(\n                self,\n                'column_index_begin') and self.column_index_begin is not None:\n            _dict['column_index_begin'] = self.column_index_begin\n        if hasattr(self,\n                   'column_index_end') and self.column_index_end is not None:\n            _dict['column_index_end'] = self.column_index_end\n        if hasattr(self, 'row_header_ids') and self.row_header_ids is not None:\n            _dict['row_header_ids'] = [\n                x._to_dict() for x in self.row_header_ids\n            ]\n        if hasattr(self,\n                   'row_header_texts') and self.row_header_texts is not None:\n            _dict['row_header_texts'] = [\n                x._to_dict() for x in self.row_header_texts\n            ]\n        if hasattr(self, 'row_header_texts_normalized'\n                  ) and self.row_header_texts_normalized is not None:\n            _dict['row_header_texts_normalized'] = [\n                x._to_dict() for x in self.row_header_texts_normalized\n            ]\n        if hasattr(self,\n                   'column_header_ids') and self.column_header_ids is not None:\n            _dict['column_header_ids'] = [\n                x._to_dict() for x in self.column_header_ids\n            ]\n        if hasattr(\n                self,\n                'column_header_texts') and self.column_header_texts is not None:\n            _dict['column_header_texts'] = [\n                x._to_dict() for x in self.column_header_texts\n            ]\n        if hasattr(self, 'column_header_texts_normalized'\n                  ) and self.column_header_texts_normalized is not None:\n            _dict['column_header_texts_normalized'] = [\n                x._to_dict() for x in self.column_header_texts_normalized\n            ]\n        if hasattr(self, 'attributes') and self.attributes is not None:\n            _dict['attributes'] = [x._to_dict() for x in self.attributes]\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'label') and self.label is not None:\n            _dict['label'] = self.label\n        if hasattr(self, 'provenance_ids') and self.provenance_ids is not None:\n            _dict['provenance_ids'] = self.provenance_ids\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ninitializes a ClassifyReturn object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a ClassifyReturn object from a json dictionary.\"\"\"\n        args = {}\n        if 'document' in _dict:\n            args['document'] = Document._from_dict(_dict.get('document'))\n        if 'model_id' in _dict:\n            args['model_id'] = _dict.get('model_id')\n        if 'model_version' in _dict:\n            args['model_version'] = _dict.get('model_version')\n        if 'elements' in _dict:\n            args['elements'] = [\n                Element._from_dict(x) for x in (_dict.get('elements'))\n            ]\n        if 'tables' in _dict:\n            args['tables'] = [\n                Tables._from_dict(x) for x in (_dict.get('tables'))\n            ]\n        if 'document_structure' in _dict:\n            args['document_structure'] = DocStructure._from_dict(\n                _dict.get('document_structure'))\n        if 'parties' in _dict:\n            args['parties'] = [\n                Parties._from_dict(x) for x in (_dict.get('parties'))\n            ]\n        if 'effective_dates' in _dict:\n            args['effective_dates'] = [\n                EffectiveDates._from_dict(x)\n                for x in (_dict.get('effective_dates'))\n            ]\n        if 'contract_amounts' in _dict:\n            args['contract_amounts'] = [\n                ContractAmts._from_dict(x)\n                for x in (_dict.get('contract_amounts'))\n            ]\n        if 'termination_dates' in _dict:\n            args['termination_dates'] = [\n                TerminationDates._from_dict(x)\n                for x in (_dict.get('termination_dates'))\n            ]\n        if 'contract_type' in _dict:\n            args['contract_type'] = [\n                ContractType._from_dict(x) for x in (_dict.get('contract_type'))\n            ]\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'document') and self.document is not None:\n            _dict['document'] = self.document._to_dict()\n        if hasattr(self, 'model_id') and self.model_id is not None:\n            _dict['model_id'] = self.model_id\n        if hasattr(self, 'model_version') and self.model_version is not None:\n            _dict['model_version'] = self.model_version\n        if hasattr(self, 'elements') and self.elements is not None:\n            _dict['elements'] = [x._to_dict() for x in self.elements]\n        if hasattr(self, 'tables') and self.tables is not None:\n            _dict['tables'] = [x._to_dict() for x in self.tables]\n        if hasattr(\n                self,\n                'document_structure') and self.document_structure is not None:\n            _dict['document_structure'] = self.document_structure._to_dict()\n        if hasattr(self, 'parties') and self.parties is not None:\n            _dict['parties'] = [x._to_dict() for x in self.parties]\n        if hasattr(self,\n                   'effective_dates') and self.effective_dates is not None:\n            _dict['effective_dates'] = [\n                x._to_dict() for x in self.effective_dates\n            ]\n        if hasattr(self,\n                   'contract_amounts') and self.contract_amounts is not None:\n            _dict['contract_amounts'] = [\n                x._to_dict() for x in self.contract_amounts\n            ]\n        if hasattr(self,\n                   'termination_dates') and self.termination_dates is not None:\n            _dict['termination_dates'] = [\n                x._to_dict() for x in self.termination_dates\n            ]\n        if hasattr(self, 'contract_type') and self.contract_type is not None:\n            _dict['contract_type'] = [x._to_dict() for x in self.contract_type]\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'id') and self.id is not None:\n            _dict['id'] = self.id\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self,\n                   'text_normalized') and self.text_normalized is not None:\n            _dict['text_normalized'] = self.text_normalized\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'model_id' in _dict:\n            args['model_id'] = _dict.get('model_id')\n        if 'model_version' in _dict:\n            args['model_version'] = _dict.get('model_version')\n        if 'documents' in _dict:\n            args['documents'] = [\n                Document._from_dict(x) for x in (_dict.get('documents'))\n            ]\n        if 'aligned_elements' in _dict:\n            args['aligned_elements'] = [\n                AlignedElement._from_dict(x)\n                for x in (_dict.get('aligned_elements'))\n            ]\n        if 'unaligned_elements' in _dict:\n            args['unaligned_elements'] = [\n                UnalignedElement._from_dict(x)\n                for x in (_dict.get('unaligned_elements'))\n            ]\n        return cls(**args)", "response": "Initialize a CompareReturn object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'model_id') and self.model_id is not None:\n            _dict['model_id'] = self.model_id\n        if hasattr(self, 'model_version') and self.model_version is not None:\n            _dict['model_version'] = self.model_version\n        if hasattr(self, 'documents') and self.documents is not None:\n            _dict['documents'] = [x._to_dict() for x in self.documents]\n        if hasattr(self,\n                   'aligned_elements') and self.aligned_elements is not None:\n            _dict['aligned_elements'] = [\n                x._to_dict() for x in self.aligned_elements\n            ]\n        if hasattr(\n                self,\n                'unaligned_elements') and self.unaligned_elements is not None:\n            _dict['unaligned_elements'] = [\n                x._to_dict() for x in self.unaligned_elements\n            ]\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'name') and self.name is not None:\n            _dict['name'] = self.name\n        if hasattr(self, 'role') and self.role is not None:\n            _dict['role'] = self.role\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'total') and self.total is not None:\n            _dict['total'] = self.total\n        if hasattr(self, 'pending') and self.pending is not None:\n            _dict['pending'] = self.pending\n        if hasattr(self, 'successful') and self.successful is not None:\n            _dict['successful'] = self.successful\n        if hasattr(self, 'failed') and self.failed is not None:\n            _dict['failed'] = self.failed\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'section_titles' in _dict:\n            args['section_titles'] = [\n                SectionTitles._from_dict(x)\n                for x in (_dict.get('section_titles'))\n            ]\n        if 'leading_sentences' in _dict:\n            args['leading_sentences'] = [\n                LeadingSentence._from_dict(x)\n                for x in (_dict.get('leading_sentences'))\n            ]\n        return cls(**args)", "response": "Initialize a DocStructure object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'section_titles') and self.section_titles is not None:\n            _dict['section_titles'] = [\n                x._to_dict() for x in self.section_titles\n            ]\n        if hasattr(self,\n                   'leading_sentences') and self.leading_sentences is not None:\n            _dict['leading_sentences'] = [\n                x._to_dict() for x in self.leading_sentences\n            ]\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ninitializes a Element object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a Element object from a json dictionary.\"\"\"\n        args = {}\n        if 'location' in _dict:\n            args['location'] = Location._from_dict(_dict.get('location'))\n        if 'text' in _dict:\n            args['text'] = _dict.get('text')\n        if 'types' in _dict:\n            args['types'] = [\n                TypeLabel._from_dict(x) for x in (_dict.get('types'))\n            ]\n        if 'categories' in _dict:\n            args['categories'] = [\n                Category._from_dict(x) for x in (_dict.get('categories'))\n            ]\n        if 'attributes' in _dict:\n            args['attributes'] = [\n                Attribute._from_dict(x) for x in (_dict.get('attributes'))\n            ]\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'begin') and self.begin is not None:\n            _dict['begin'] = self.begin\n        if hasattr(self, 'end') and self.end is not None:\n            _dict['end'] = self.end\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ninitialize a FeedbackDataInput object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a FeedbackDataInput object from a json dictionary.\"\"\"\n        args = {}\n        if 'feedback_type' in _dict:\n            args['feedback_type'] = _dict.get('feedback_type')\n        else:\n            raise ValueError(\n                'Required property \\'feedback_type\\' not present in FeedbackDataInput JSON'\n            )\n        if 'document' in _dict:\n            args['document'] = ShortDoc._from_dict(_dict.get('document'))\n        if 'model_id' in _dict:\n            args['model_id'] = _dict.get('model_id')\n        if 'model_version' in _dict:\n            args['model_version'] = _dict.get('model_version')\n        if 'location' in _dict:\n            args['location'] = Location._from_dict(_dict.get('location'))\n        else:\n            raise ValueError(\n                'Required property \\'location\\' not present in FeedbackDataInput JSON'\n            )\n        if 'text' in _dict:\n            args['text'] = _dict.get('text')\n        else:\n            raise ValueError(\n                'Required property \\'text\\' not present in FeedbackDataInput JSON'\n            )\n        if 'original_labels' in _dict:\n            args['original_labels'] = OriginalLabelsIn._from_dict(\n                _dict.get('original_labels'))\n        else:\n            raise ValueError(\n                'Required property \\'original_labels\\' not present in FeedbackDataInput JSON'\n            )\n        if 'updated_labels' in _dict:\n            args['updated_labels'] = UpdatedLabelsIn._from_dict(\n                _dict.get('updated_labels'))\n        else:\n            raise ValueError(\n                'Required property \\'updated_labels\\' not present in FeedbackDataInput JSON'\n            )\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'feedback_type' in _dict:\n            args['feedback_type'] = _dict.get('feedback_type')\n        if 'document' in _dict:\n            args['document'] = ShortDoc._from_dict(_dict.get('document'))\n        if 'model_id' in _dict:\n            args['model_id'] = _dict.get('model_id')\n        if 'model_version' in _dict:\n            args['model_version'] = _dict.get('model_version')\n        if 'location' in _dict:\n            args['location'] = Location._from_dict(_dict.get('location'))\n        if 'text' in _dict:\n            args['text'] = _dict.get('text')\n        if 'original_labels' in _dict:\n            args['original_labels'] = OriginalLabelsOut._from_dict(\n                _dict.get('original_labels'))\n        if 'updated_labels' in _dict:\n            args['updated_labels'] = UpdatedLabelsOut._from_dict(\n                _dict.get('updated_labels'))\n        if 'pagination' in _dict:\n            args['pagination'] = Pagination._from_dict(_dict.get('pagination'))\n        return cls(**args)", "response": "Initialize a FeedbackDataOutput object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'feedback_type') and self.feedback_type is not None:\n            _dict['feedback_type'] = self.feedback_type\n        if hasattr(self, 'document') and self.document is not None:\n            _dict['document'] = self.document._to_dict()\n        if hasattr(self, 'model_id') and self.model_id is not None:\n            _dict['model_id'] = self.model_id\n        if hasattr(self, 'model_version') and self.model_version is not None:\n            _dict['model_version'] = self.model_version\n        if hasattr(self, 'location') and self.location is not None:\n            _dict['location'] = self.location._to_dict()\n        if hasattr(self, 'text') and self.text is not None:\n            _dict['text'] = self.text\n        if hasattr(self,\n                   'original_labels') and self.original_labels is not None:\n            _dict['original_labels'] = self.original_labels._to_dict()\n        if hasattr(self, 'updated_labels') and self.updated_labels is not None:\n            _dict['updated_labels'] = self.updated_labels._to_dict()\n        if hasattr(self, 'pagination') and self.pagination is not None:\n            _dict['pagination'] = self.pagination._to_dict()\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'status') and self.status is not None:\n            _dict['status'] = self.status\n        if hasattr(self, 'message') and self.message is not None:\n            _dict['message'] = self.message\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ninitialize a FeedbackList object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a FeedbackList object from a json dictionary.\"\"\"\n        args = {}\n        if 'feedback' in _dict:\n            args['feedback'] = [\n                GetFeedback._from_dict(x) for x in (_dict.get('feedback'))\n            ]\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'feedback') and self.feedback is not None:\n            _dict['feedback'] = [x._to_dict() for x in self.feedback]\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'feedback_id') and self.feedback_id is not None:\n            _dict['feedback_id'] = self.feedback_id\n        if hasattr(self, 'user_id') and self.user_id is not None:\n            _dict['user_id'] = self.user_id\n        if hasattr(self, 'comment') and self.comment is not None:\n            _dict['comment'] = self.comment\n        if hasattr(self, 'created') and self.created is not None:\n            _dict['created'] = datetime_to_string(self.created)\n        if hasattr(self, 'feedback_data') and self.feedback_data is not None:\n            _dict['feedback_data'] = self.feedback_data._to_dict()\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'num_pages') and self.num_pages is not None:\n            _dict['num_pages'] = self.num_pages\n        if hasattr(self, 'author') and self.author is not None:\n            _dict['author'] = self.author\n        if hasattr(self,\n                   'publication_date') and self.publication_date is not None:\n            _dict['publication_date'] = self.publication_date\n        if hasattr(self, 'title') and self.title is not None:\n            _dict['title'] = self.title\n        if hasattr(self, 'html') and self.html is not None:\n            _dict['html'] = self.html\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'cell_id') and self.cell_id is not None:\n            _dict['cell_id'] = self.cell_id\n        if hasattr(self, 'location') and self.location is not None:\n            _dict['location'] = self.location._to_dict()\n        if hasattr(self, 'text') and self.text is not None:\n            _dict['text'] = self.text\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'key' in _dict:\n            args['key'] = Key._from_dict(_dict.get('key'))\n        if 'value' in _dict:\n            args['value'] = Value._from_dict(_dict.get('value'))\n        return cls(**args)", "response": "Initialize a KeyValuePair object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'key') and self.key is not None:\n            _dict['key'] = self.key._to_dict()\n        if hasattr(self, 'value') and self.value is not None:\n            _dict['value'] = self.value._to_dict()\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'nature') and self.nature is not None:\n            _dict['nature'] = self.nature\n        if hasattr(self, 'party') and self.party is not None:\n            _dict['party'] = self.party\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ninitialize a LeadingSentence object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a LeadingSentence object from a json dictionary.\"\"\"\n        args = {}\n        if 'text' in _dict:\n            args['text'] = _dict.get('text')\n        if 'location' in _dict:\n            args['location'] = Location._from_dict(_dict.get('location'))\n        if 'element_locations' in _dict:\n            args['element_locations'] = [\n                ElementLocations._from_dict(x)\n                for x in (_dict.get('element_locations'))\n            ]\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ninitializing a OriginalLabelsIn object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a OriginalLabelsIn object from a json dictionary.\"\"\"\n        args = {}\n        if 'types' in _dict:\n            args['types'] = [\n                TypeLabel._from_dict(x) for x in (_dict.get('types'))\n            ]\n        else:\n            raise ValueError(\n                'Required property \\'types\\' not present in OriginalLabelsIn JSON'\n            )\n        if 'categories' in _dict:\n            args['categories'] = [\n                Category._from_dict(x) for x in (_dict.get('categories'))\n            ]\n        else:\n            raise ValueError(\n                'Required property \\'categories\\' not present in OriginalLabelsIn JSON'\n            )\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'types') and self.types is not None:\n            _dict['types'] = [x._to_dict() for x in self.types]\n        if hasattr(self, 'categories') and self.categories is not None:\n            _dict['categories'] = [x._to_dict() for x in self.categories]\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'refresh_cursor') and self.refresh_cursor is not None:\n            _dict['refresh_cursor'] = self.refresh_cursor\n        if hasattr(self, 'next_cursor') and self.next_cursor is not None:\n            _dict['next_cursor'] = self.next_cursor\n        if hasattr(self, 'refresh_url') and self.refresh_url is not None:\n            _dict['refresh_url'] = self.refresh_url\n        if hasattr(self, 'next_url') and self.next_url is not None:\n            _dict['next_url'] = self.next_url\n        if hasattr(self, 'total') and self.total is not None:\n            _dict['total'] = self.total\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'party' in _dict:\n            args['party'] = _dict.get('party')\n        if 'importance' in _dict:\n            args['importance'] = _dict.get('importance')\n        if 'role' in _dict:\n            args['role'] = _dict.get('role')\n        if 'addresses' in _dict:\n            args['addresses'] = [\n                Address._from_dict(x) for x in (_dict.get('addresses'))\n            ]\n        if 'contacts' in _dict:\n            args['contacts'] = [\n                Contact._from_dict(x) for x in (_dict.get('contacts'))\n            ]\n        return cls(**args)", "response": "Initialize a Parties object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'party') and self.party is not None:\n            _dict['party'] = self.party\n        if hasattr(self, 'importance') and self.importance is not None:\n            _dict['importance'] = self.importance\n        if hasattr(self, 'role') and self.role is not None:\n            _dict['role'] = self.role\n        if hasattr(self, 'addresses') and self.addresses is not None:\n            _dict['addresses'] = [x._to_dict() for x in self.addresses]\n        if hasattr(self, 'contacts') and self.contacts is not None:\n            _dict['contacts'] = [x._to_dict() for x in self.contacts]\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'text') and self.text is not None:\n            _dict['text'] = self.text\n        if hasattr(self, 'location') and self.location is not None:\n            _dict['location'] = self.location._to_dict()\n        if hasattr(self, 'level') and self.level is not None:\n            _dict['level'] = self.level\n        if hasattr(self,\n                   'element_locations') and self.element_locations is not None:\n            _dict['element_locations'] = [\n                x._to_dict() for x in self.element_locations\n            ]\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'title') and self.title is not None:\n            _dict['title'] = self.title\n        if hasattr(self, 'hash') and self.hash is not None:\n            _dict['hash'] = self.hash\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'cell_id') and self.cell_id is not None:\n            _dict['cell_id'] = self.cell_id\n        if hasattr(self, 'location') and self.location is not None:\n            _dict['location'] = self.location\n        if hasattr(self, 'text') and self.text is not None:\n            _dict['text'] = self.text\n        if hasattr(self,\n                   'row_index_begin') and self.row_index_begin is not None:\n            _dict['row_index_begin'] = self.row_index_begin\n        if hasattr(self, 'row_index_end') and self.row_index_end is not None:\n            _dict['row_index_end'] = self.row_index_end\n        if hasattr(\n                self,\n                'column_index_begin') and self.column_index_begin is not None:\n            _dict['column_index_begin'] = self.column_index_begin\n        if hasattr(self,\n                   'column_index_end') and self.column_index_end is not None:\n            _dict['column_index_end'] = self.column_index_end\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'document' in _dict:\n            args['document'] = DocInfo._from_dict(_dict.get('document'))\n        if 'model_id' in _dict:\n            args['model_id'] = _dict.get('model_id')\n        if 'model_version' in _dict:\n            args['model_version'] = _dict.get('model_version')\n        if 'tables' in _dict:\n            args['tables'] = [\n                Tables._from_dict(x) for x in (_dict.get('tables'))\n            ]\n        return cls(**args)", "response": "Initialize a TableReturn object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'document') and self.document is not None:\n            _dict['document'] = self.document._to_dict()\n        if hasattr(self, 'model_id') and self.model_id is not None:\n            _dict['model_id'] = self.model_id\n        if hasattr(self, 'model_version') and self.model_version is not None:\n            _dict['model_version'] = self.model_version\n        if hasattr(self, 'tables') and self.tables is not None:\n            _dict['tables'] = [x._to_dict() for x in self.tables]\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'location' in _dict:\n            args['location'] = Location._from_dict(_dict.get('location'))\n        if 'text' in _dict:\n            args['text'] = _dict.get('text')\n        if 'section_title' in _dict:\n            args['section_title'] = SectionTitle._from_dict(\n                _dict.get('section_title'))\n        if 'table_headers' in _dict:\n            args['table_headers'] = [\n                TableHeaders._from_dict(x) for x in (_dict.get('table_headers'))\n            ]\n        if 'row_headers' in _dict:\n            args['row_headers'] = [\n                RowHeaders._from_dict(x) for x in (_dict.get('row_headers'))\n            ]\n        if 'column_headers' in _dict:\n            args['column_headers'] = [\n                ColumnHeaders._from_dict(x)\n                for x in (_dict.get('column_headers'))\n            ]\n        if 'key_value_pairs' in _dict:\n            args['key_value_pairs'] = [\n                KeyValuePair._from_dict(x)\n                for x in (_dict.get('key_value_pairs'))\n            ]\n        if 'body_cells' in _dict:\n            args['body_cells'] = [\n                BodyCells._from_dict(x) for x in (_dict.get('body_cells'))\n            ]\n        return cls(**args)", "response": "Initialize a Tables object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'location') and self.location is not None:\n            _dict['location'] = self.location._to_dict()\n        if hasattr(self, 'text') and self.text is not None:\n            _dict['text'] = self.text\n        if hasattr(self, 'section_title') and self.section_title is not None:\n            _dict['section_title'] = self.section_title._to_dict()\n        if hasattr(self, 'table_headers') and self.table_headers is not None:\n            _dict['table_headers'] = [x._to_dict() for x in self.table_headers]\n        if hasattr(self, 'row_headers') and self.row_headers is not None:\n            _dict['row_headers'] = [x._to_dict() for x in self.row_headers]\n        if hasattr(self, 'column_headers') and self.column_headers is not None:\n            _dict['column_headers'] = [\n                x._to_dict() for x in self.column_headers\n            ]\n        if hasattr(self,\n                   'key_value_pairs') and self.key_value_pairs is not None:\n            _dict['key_value_pairs'] = [\n                x._to_dict() for x in self.key_value_pairs\n            ]\n        if hasattr(self, 'body_cells') and self.body_cells is not None:\n            _dict['body_cells'] = [x._to_dict() for x in self.body_cells]\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'label') and self.label is not None:\n            _dict['label'] = self.label._to_dict()\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'document_label' in _dict:\n            args['document_label'] = _dict.get('document_label')\n        if 'location' in _dict:\n            args['location'] = Location._from_dict(_dict.get('location'))\n        if 'text' in _dict:\n            args['text'] = _dict.get('text')\n        if 'types' in _dict:\n            args['types'] = [\n                TypeLabelComparison._from_dict(x) for x in (_dict.get('types'))\n            ]\n        if 'categories' in _dict:\n            args['categories'] = [\n                CategoryComparison._from_dict(x)\n                for x in (_dict.get('categories'))\n            ]\n        if 'attributes' in _dict:\n            args['attributes'] = [\n                Attribute._from_dict(x) for x in (_dict.get('attributes'))\n            ]\n        return cls(**args)", "response": "Initialize a UnalignedElement object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'document_label') and self.document_label is not None:\n            _dict['document_label'] = self.document_label\n        if hasattr(self, 'location') and self.location is not None:\n            _dict['location'] = self.location._to_dict()\n        if hasattr(self, 'text') and self.text is not None:\n            _dict['text'] = self.text\n        if hasattr(self, 'types') and self.types is not None:\n            _dict['types'] = [x._to_dict() for x in self.types]\n        if hasattr(self, 'categories') and self.categories is not None:\n            _dict['categories'] = [x._to_dict() for x in self.categories]\n        if hasattr(self, 'attributes') and self.attributes is not None:\n            _dict['attributes'] = [x._to_dict() for x in self.attributes]\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nsends audio for speech recognition using the web sockets.", "response": "def recognize_using_websocket(self,\n                                  audio,\n                                  content_type,\n                                  recognize_callback,\n                                  model=None,\n                                  language_customization_id=None,\n                                  acoustic_customization_id=None,\n                                  customization_weight=None,\n                                  base_model_version=None,\n                                  inactivity_timeout=None,\n                                  interim_results=None,\n                                  keywords=None,\n                                  keywords_threshold=None,\n                                  max_alternatives=None,\n                                  word_alternatives_threshold=None,\n                                  word_confidence=None,\n                                  timestamps=None,\n                                  profanity_filter=None,\n                                  smart_formatting=None,\n                                  speaker_labels=None,\n                                  http_proxy_host=None,\n                                  http_proxy_port=None,\n                                  customization_id=None,\n                                  grammar_name=None,\n                                  redaction=None,\n                                  **kwargs):\n        \"\"\"\n        Sends audio for speech recognition using web sockets.\n\n        :param AudioSource audio: The audio to transcribe in the format specified by the\n        `Content-Type` header.\n        :param str content_type: The type of the input: audio/basic, audio/flac,\n        audio/l16, audio/mp3, audio/mpeg, audio/mulaw, audio/ogg, audio/ogg;codecs=opus,\n        audio/ogg;codecs=vorbis, audio/wav, audio/webm, audio/webm;codecs=opus, or\n        audio/webm;codecs=vorbis.\n        :param RecognizeCallback recognize_callback: The callback method for the websocket.\n        :param str model: The identifier of the model that is to be used for the\n        recognition request or, for the **Create a session** method, with the new session.\n        :param str language_customization_id: The customization ID (GUID) of a custom\n        language model that is to be used with the recognition request. The base model of\n        the specified custom language model must match the model specified with the\n        `model` parameter. You must make the request with service credentials created for\n        the instance of the service that owns the custom model. By default, no custom\n        language model is used. See [Custom\n        models](https://console.bluemix.net/docs/services/speech-to-text/input.html#custom).\n        **Note:** Use this parameter instead of the deprecated `customization_id`\n        parameter.\n        :param str acoustic_customization_id: The customization ID (GUID) of a custom\n        acoustic model that is to be used with the recognition request or, for the\n        **Create a session** method, with the new session. The base model of the specified\n        custom acoustic model must match the model specified with the `model` parameter.\n        You must make the request with service credentials created for the instance of the\n        service that owns the custom model. By default, no custom acoustic model is used.\n        :param float customization_weight: If you specify the customization ID (GUID) of a\n        custom language model with the recognition request or, for sessions, with the\n        **Create a session** method, the customization weight tells the service how much\n        weight to give to words from the custom language model compared to those from the\n        base model for the current request.\n        Specify a value between 0.0 and 1.0. Unless a different customization weight was\n        specified for the custom model when it was trained, the default value is 0.3. A\n        customization weight that you specify overrides a weight that was specified when\n        the custom model was trained.\n        The default value yields the best performance in general. Assign a higher value if\n        your audio makes frequent use of OOV words from the custom model. Use caution when\n        setting the weight: a higher value can improve the accuracy of phrases from the\n        custom model's domain, but it can negatively affect performance on non-domain\n        phrases.\n        :param str base_model_version: The version of the specified base model that is to\n        be used with recognition request or, for the **Create a session** method, with the\n        new session. Multiple versions of a base model can exist when a model is updated\n        for internal improvements. The parameter is intended primarily for use with custom\n        models that have been upgraded for a new base model. The default value depends on\n        whether the parameter is used with or without a custom model. For more\n        information, see [Base model\n        version](https://console.bluemix.net/docs/services/speech-to-text/input.html#version).\n        :param int inactivity_timeout: The time in seconds after which, if only silence\n        (no speech) is detected in submitted audio, the connection is closed with a 400\n        error. Useful for stopping audio submission from a live microphone when a user\n        simply walks away. Use `-1` for infinity.\n        :param list[str] keywords: An array of keyword strings to spot in the audio. Each\n        keyword string can include one or more tokens. Keywords are spotted only in the\n        final hypothesis, not in interim results. If you specify any keywords, you must\n        also specify a keywords threshold. You can spot a maximum of 1000 keywords. Omit\n        the parameter or specify an empty array if you do not need to spot keywords.\n        :param float keywords_threshold: A confidence value that is the lower bound for\n        spotting a keyword. A word is considered to match a keyword if its confidence is\n        greater than or equal to the threshold. Specify a probability between 0 and 1\n        inclusive. No keyword spotting is performed if you omit the parameter. If you\n        specify a threshold, you must also specify one or more keywords.\n        :param int max_alternatives: The maximum number of alternative transcripts to be\n        returned. By default, a single transcription is returned.\n        :param float word_alternatives_threshold: A confidence value that is the lower\n        bound for identifying a hypothesis as a possible word alternative (also known as\n        \\\"Confusion Networks\\\"). An alternative word is considered if its confidence is\n        greater than or equal to the threshold. Specify a probability between 0 and 1\n        inclusive. No alternative words are computed if you omit the parameter.\n        :param bool word_confidence: If `true`, a confidence measure in the range of 0 to\n        1 is returned for each word. By default, no word confidence measures are returned.\n        :param bool timestamps: If `true`, time alignment is returned for each word. By\n        default, no timestamps are returned.\n        :param bool profanity_filter: If `true` (the default), filters profanity from all\n        output except for keyword results by replacing inappropriate words with a series\n        of asterisks. Set the parameter to `false` to return results with no censoring.\n        Applies to US English transcription only.\n        :param bool smart_formatting: If `true`, converts dates, times, series of digits\n        and numbers, phone numbers, currency values, and internet addresses into more\n        readable, conventional representations in the final transcript of a recognition\n        request. For US English, also converts certain keyword strings to punctuation\n        symbols. By default, no smart formatting is performed. Applies to US English and\n        Spanish transcription only.\n        :param bool speaker_labels: If `true`, the response includes labels that identify\n        which words were spoken by which participants in a multi-person exchange. By\n        default, no speaker labels are returned. Setting `speaker_labels` to `true` forces\n        the `timestamps` parameter to be `true`, regardless of whether you specify `false`\n        for the parameter.\n        To determine whether a language model supports speaker labels, use the **Get\n        models** method and check that the attribute `speaker_labels` is set to `true`.\n        You can also refer to [Speaker\n        labels](https://console.bluemix.net/docs/services/speech-to-text/output.html#speaker_labels).\n        :param str http_proxy_host: http proxy host name.\n        :param str http_proxy_port: http proxy port. If not set, set to 80.\n        :param str customization_id: **Deprecated.** Use the `language_customization_id`\n        parameter to specify the customization ID (GUID) of a custom language model that\n        is to be used with the recognition request. Do not specify both parameters with a\n        request.\n        :param str grammar_name: The name of a grammar that is to be used with the\n        recognition request. If you specify a grammar, you must also use the\n        `language_customization_id` parameter to specify the name of the custom language\n        model for which the grammar is defined. The service recognizes only strings that\n        are recognized by the specified grammar; it does not recognize other custom words\n        from the model's words resource. See\n        [Grammars](https://cloud.ibm.com/docs/services/speech-to-text/output.html).\n        :param bool redaction: If `true`, the service redacts, or masks, numeric data from\n        final transcripts. The feature redacts any number that has three or more\n        consecutive digits by replacing each digit with an `X` character. It is intended\n        to redact sensitive numeric data, such as credit card numbers. By default, the\n        service performs no redaction.\n        When you enable redaction, the service automatically enables smart formatting,\n        regardless of whether you explicitly disable that feature. To ensure maximum\n        security, the service also disables keyword spotting (ignores the `keywords` and\n        `keywords_threshold` parameters) and returns only a single final transcript\n        (forces the `max_alternatives` parameter to be `1`).\n        **Note:** Applies to US English, Japanese, and Korean transcription only.\n        See [Numeric\n        redaction](https://cloud.ibm.com/docs/services/speech-to-text/output.html#redaction).\n        :param dict headers: A `dict` containing the request headers\n        :return: A `dict` containing the `SpeechRecognitionResults` response.\n        :rtype: dict\n        \"\"\"\n        if audio is None:\n            raise ValueError('audio must be provided')\n        if not isinstance(audio, AudioSource):\n            raise Exception(\n                'audio is not of type AudioSource. Import the class from ibm_watson.websocket')\n        if content_type is None:\n            raise ValueError('content_type must be provided')\n        if recognize_callback is None:\n            raise ValueError('recognize_callback must be provided')\n        if not isinstance(recognize_callback, RecognizeCallback):\n            raise Exception(\n                'Callback is not a derived class of RecognizeCallback')\n\n        headers = {}\n        if self.default_headers is not None:\n            headers = self.default_headers.copy()\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n\n        if self.token_manager:\n            access_token = self.token_manager.get_token()\n            headers['Authorization'] = '{0} {1}'.format(BEARER, access_token)\n        else:\n            authstring = \"{0}:{1}\".format(self.username, self.password)\n            base64_authorization = base64.b64encode(authstring.encode('utf-8')).decode('utf-8')\n            headers['Authorization'] = 'Basic {0}'.format(base64_authorization)\n\n        url = self.url.replace('https:', 'wss:')\n        params = {\n            'model': model,\n            'customization_id': customization_id,\n            'acoustic_customization_id': acoustic_customization_id,\n            'customization_weight': customization_weight,\n            'base_model_version': base_model_version,\n            'language_customization_id': language_customization_id\n        }\n        params = dict([(k, v) for k, v in params.items() if v is not None])\n        url += '/v1/recognize?{0}'.format(urlencode(params))\n\n        options = {\n            'content_type': content_type,\n            'inactivity_timeout': inactivity_timeout,\n            'interim_results': interim_results,\n            'keywords': keywords,\n            'keywords_threshold': keywords_threshold,\n            'max_alternatives': max_alternatives,\n            'word_alternatives_threshold': word_alternatives_threshold,\n            'word_confidence': word_confidence,\n            'timestamps': timestamps,\n            'profanity_filter': profanity_filter,\n            'smart_formatting': smart_formatting,\n            'speaker_labels': speaker_labels,\n            'grammar_name': grammar_name,\n            'redaction': redaction\n        }\n        options = dict([(k, v) for k, v in options.items() if v is not None])\n\n        RecognizeListener(audio,\n                          options,\n                          recognize_callback,\n                          url,\n                          headers,\n                          http_proxy_host,\n                          http_proxy_port,\n                          self.verify)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef send_text(self):\n        def run(*args):\n            \"\"\"Background process to send the text\"\"\"\n            self.ws_client.send(json.dumps(self.options).encode('utf8'))\n\n            time.sleep(TEN_MILLISECONDS)\n\n        thread.start_new_thread(run, ())", "response": "Sends the text message to the server"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef send(self, data, opcode=websocket.ABNF.OPCODE_TEXT):\n        self.ws_client.send(data, opcode)", "response": "Send a message to the server."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef send_audio(self, ws):\n        def run(*args):\n            \"\"\"Background process to stream the data\"\"\"\n            if not self.audio_source.is_buffer:\n                while True:\n                    chunk = self.audio_source.input.read(ONE_KB)\n                    if not chunk:\n                        break\n                    self.ws_client.send(chunk, websocket.ABNF.OPCODE_BINARY)\n                    time.sleep(TEN_MILLISECONDS)\n\n                self.audio_source.input.close()\n            else:\n                while True:\n                    try:\n                        if not self.audio_source.input.empty():\n                            chunk = self.audio_source.input.get()\n                            self.ws_client.send(chunk, websocket.ABNF.OPCODE_BINARY)\n                            time.sleep(TEN_MILLISECONDS)\n                        if self.audio_source.input.empty():\n                            if self.audio_source.is_recording:\n                                time.sleep(TEN_MILLISECONDS)\n                            else:\n                                break\n                    except Exception:\n                        if self.audio_source.is_recording:\n                            time.sleep(TEN_MILLISECONDS)\n                        else:\n                            break\n\n            time.sleep(TEN_MILLISECONDS)\n            self.ws_client.send(self.build_closing_message(), websocket.ABNF.OPCODE_TEXT)\n\n        thread.start_new_thread(run, ())", "response": "Stream audio to server"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef on_open(self, ws):\n        self.callback.on_connected()\n\n        # Send initialization message\n        init_data = self.build_start_message(self.options)\n        self.ws_client.send(json.dumps(init_data).encode('utf8'), websocket.ABNF.OPCODE_TEXT)", "response": "Callback executed when a connection is opened to the server."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef on_data(self, ws, message, message_type, fin):\n\n        try:\n            json_object = json.loads(message)\n        except Exception:\n            self.on_error(ws, 'Unable to parse received message.')\n\n        if 'error' in json_object:\n            # Only call on_error() if a real error occurred. The STT service sends\n            # {\"error\" : \"No speech detected for 5s\"} for valid timeouts, configured by\n            # options.inactivity_timeout\n            error = json_object['error']\n            if error.startswith(TIMEOUT_PREFIX):\n                self.callback.on_inactivity_timeout(error)\n            else:\n                self.on_error(ws, error)\n\n        # if uninitialized, receive the initialization response from the server\n        elif 'state' in json_object:\n            if not self.isListening:\n                self.isListening = True\n                self.callback.on_listening()\n                self.send_audio(ws)\n            else:\n                # close the connection\n                self.callback.on_close()\n                ws.close()\n\n        # if in streaming\n        elif 'results' in json_object or 'speaker_labels' in json_object:\n            hypothesis = ''\n            if 'results' in json_object:\n                hypothesis = json_object['results'][0]['alternatives'][0][\n                    'transcript']\n                b_final = (json_object['results'][0]['final'] is True)\n                transcripts = self.extract_transcripts(\n                    json_object['results'][0]['alternatives'])\n\n                if b_final:\n                    self.callback.on_transcription(transcripts)\n\n                self.callback.on_hypothesis(hypothesis)\n            self.callback.on_data(json_object)", "response": "Callback executed when a message is received from the server."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef profile(self,\n                content,\n                accept,\n                content_language=None,\n                accept_language=None,\n                raw_scores=None,\n                csv_headers=None,\n                consumption_preferences=None,\n                content_type=None,\n                **kwargs):\n        \"\"\"\n        Get profile.\n\n        Generates a personality profile for the author of the input text. The service\n        accepts a maximum of 20 MB of input content, but it requires much less text to\n        produce an accurate profile. The service can analyze text in Arabic, English,\n        Japanese, Korean, or Spanish. It can return its results in a variety of languages.\n        **See also:**\n        * [Requesting a\n        profile](https://cloud.ibm.com/docs/services/personality-insights/input.html)\n        * [Providing sufficient\n        input](https://cloud.ibm.com/docs/services/personality-insights/input.html#sufficient)\n        ### Content types\n         You can provide input content as plain text (`text/plain`), HTML (`text/html`),\n        or JSON (`application/json`) by specifying the **Content-Type** parameter. The\n        default is `text/plain`.\n        * Per the JSON specification, the default character encoding for JSON content is\n        effectively always UTF-8.\n        * Per the HTTP specification, the default encoding for plain text and HTML is\n        ISO-8859-1 (effectively, the ASCII character set).\n        When specifying a content type of plain text or HTML, include the `charset`\n        parameter to indicate the character encoding of the input text; for example,\n        `Content-Type: text/plain;charset=utf-8`.\n        **See also:** [Specifying request and response\n        formats](https://cloud.ibm.com/docs/services/personality-insights/input.html#formats)\n        ### Accept types\n         You must request a response as JSON (`application/json`) or comma-separated\n        values (`text/csv`) by specifying the **Accept** parameter. CSV output includes a\n        fixed number of columns. Set the **csv_headers** parameter to `true` to request\n        optional column headers for CSV output.\n        **See also:**\n        * [Understanding a JSON\n        profile](https://cloud.ibm.com/docs/services/personality-insights/output.html)\n        * [Understanding a CSV\n        profile](https://cloud.ibm.com/docs/services/personality-insights/output-csv.html).\n\n        :param Content content: A maximum of 20 MB of content to analyze, though the\n        service requires much less text; for more information, see [Providing sufficient\n        input](https://cloud.ibm.com/docs/services/personality-insights/input.html#sufficient).\n        For JSON input, provide an object of type `Content`.\n        :param str accept: The type of the response. For more information, see **Accept\n        types** in the method description.\n        :param str content_language: The language of the input text for the request:\n        Arabic, English, Japanese, Korean, or Spanish. Regional variants are treated as\n        their parent language; for example, `en-US` is interpreted as `en`.\n        The effect of the **Content-Language** parameter depends on the **Content-Type**\n        parameter. When **Content-Type** is `text/plain` or `text/html`,\n        **Content-Language** is the only way to specify the language. When\n        **Content-Type** is `application/json`, **Content-Language** overrides a language\n        specified with the `language` parameter of a `ContentItem` object, and content\n        items that specify a different language are ignored; omit this parameter to base\n        the language on the specification of the content items. You can specify any\n        combination of languages for **Content-Language** and **Accept-Language**.\n        :param str accept_language: The desired language of the response. For\n        two-character arguments, regional variants are treated as their parent language;\n        for example, `en-US` is interpreted as `en`. You can specify any combination of\n        languages for the input and response content.\n        :param bool raw_scores: Indicates whether a raw score in addition to a normalized\n        percentile is returned for each characteristic; raw scores are not compared with a\n        sample population. By default, only normalized percentiles are returned.\n        :param bool csv_headers: Indicates whether column labels are returned with a CSV\n        response. By default, no column labels are returned. Applies only when the\n        response type is CSV (`text/csv`).\n        :param bool consumption_preferences: Indicates whether consumption preferences are\n        returned with the results. By default, no consumption preferences are returned.\n        :param str content_type: The type of the input. For more information, see\n        **Content types** in the method description.\n        Default: `text/plain`.\n        :param dict headers: A `dict` containing the request headers\n        :return: A `DetailedResponse` containing the result, headers and HTTP status code.\n        :rtype: DetailedResponse\n        \"\"\"\n\n        if content is None:\n            raise ValueError('content must be provided')\n        if accept is None:\n            raise ValueError('accept must be provided')\n        if isinstance(content, Content):\n            content = self._convert_model(content, Content)\n\n        headers = {\n            'Accept': accept,\n            'Content-Language': content_language,\n            'Accept-Language': accept_language,\n            'Content-Type': content_type\n        }\n        if 'headers' in kwargs:\n            headers.update(kwargs.get('headers'))\n        sdk_headers = get_sdk_headers('personality_insights', 'V3', 'profile')\n        headers.update(sdk_headers)\n\n        params = {\n            'version': self.version,\n            'raw_scores': raw_scores,\n            'csv_headers': csv_headers,\n            'consumption_preferences': consumption_preferences\n        }\n\n        if content_type == 'application/json' and isinstance(content, dict):\n            data = json.dumps(content)\n        else:\n            data = content\n\n        url = '/v3/profile'\n        response = self.request(\n            method='POST',\n            url=url,\n            headers=headers,\n            params=params,\n            data=data,\n            accept_json=(accept is None or accept == 'application/json'))\n        return response", "response": "This function generates a personality profile for the author of the input text."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'trait_id') and self.trait_id is not None:\n            _dict['trait_id'] = self.trait_id\n        if hasattr(self, 'name') and self.name is not None:\n            _dict['name'] = self.name\n        if hasattr(self, 'category') and self.category is not None:\n            _dict['category'] = self.category\n        if hasattr(self, 'percentage') and self.percentage is not None:\n            _dict['percentage'] = self.percentage\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'consumption_preference_id'\n                  ) and self.consumption_preference_id is not None:\n            _dict['consumption_preference_id'] = self.consumption_preference_id\n        if hasattr(self, 'name') and self.name is not None:\n            _dict['name'] = self.name\n        if hasattr(self, 'score') and self.score is not None:\n            _dict['score'] = self.score\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'consumption_preference_category_id' in _dict:\n            args['consumption_preference_category_id'] = _dict.get(\n                'consumption_preference_category_id')\n        else:\n            raise ValueError(\n                'Required property \\'consumption_preference_category_id\\' not present in ConsumptionPreferencesCategory JSON'\n            )\n        if 'name' in _dict:\n            args['name'] = _dict.get('name')\n        else:\n            raise ValueError(\n                'Required property \\'name\\' not present in ConsumptionPreferencesCategory JSON'\n            )\n        if 'consumption_preferences' in _dict:\n            args['consumption_preferences'] = [\n                ConsumptionPreferences._from_dict(x)\n                for x in (_dict.get('consumption_preferences'))\n            ]\n        else:\n            raise ValueError(\n                'Required property \\'consumption_preferences\\' not present in ConsumptionPreferencesCategory JSON'\n            )\n        return cls(**args)", "response": "Initialize a ConsumptionPreferencesCategory object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'consumption_preference_category_id'\n                  ) and self.consumption_preference_category_id is not None:\n            _dict[\n                'consumption_preference_category_id'] = self.consumption_preference_category_id\n        if hasattr(self, 'name') and self.name is not None:\n            _dict['name'] = self.name\n        if hasattr(self, 'consumption_preferences'\n                  ) and self.consumption_preferences is not None:\n            _dict['consumption_preferences'] = [\n                x._to_dict() for x in self.consumption_preferences\n            ]\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _from_dict(cls, _dict):\n        args = {}\n        if 'contentItems' in _dict:\n            args['content_items'] = [\n                ContentItem._from_dict(x) for x in (_dict.get('contentItems'))\n            ]\n        else:\n            raise ValueError(\n                'Required property \\'contentItems\\' not present in Content JSON'\n            )\n        return cls(**args)", "response": "Initialize a Content object from a json dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'content_items') and self.content_items is not None:\n            _dict['contentItems'] = [x._to_dict() for x in self.content_items]\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'content') and self.content is not None:\n            _dict['content'] = self.content\n        if hasattr(self, 'id') and self.id is not None:\n            _dict['id'] = self.id\n        if hasattr(self, 'created') and self.created is not None:\n            _dict['created'] = self.created\n        if hasattr(self, 'updated') and self.updated is not None:\n            _dict['updated'] = self.updated\n        if hasattr(self, 'contenttype') and self.contenttype is not None:\n            _dict['contenttype'] = self.contenttype\n        if hasattr(self, 'language') and self.language is not None:\n            _dict['language'] = self.language\n        if hasattr(self, 'parentid') and self.parentid is not None:\n            _dict['parentid'] = self.parentid\n        if hasattr(self, 'reply') and self.reply is not None:\n            _dict['reply'] = self.reply\n        if hasattr(self, 'forward') and self.forward is not None:\n            _dict['forward'] = self.forward\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ninitializing a Profile object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a Profile object from a json dictionary.\"\"\"\n        args = {}\n        if 'processed_language' in _dict:\n            args['processed_language'] = _dict.get('processed_language')\n        else:\n            raise ValueError(\n                'Required property \\'processed_language\\' not present in Profile JSON'\n            )\n        if 'word_count' in _dict:\n            args['word_count'] = _dict.get('word_count')\n        else:\n            raise ValueError(\n                'Required property \\'word_count\\' not present in Profile JSON')\n        if 'word_count_message' in _dict:\n            args['word_count_message'] = _dict.get('word_count_message')\n        if 'personality' in _dict:\n            args['personality'] = [\n                Trait._from_dict(x) for x in (_dict.get('personality'))\n            ]\n        else:\n            raise ValueError(\n                'Required property \\'personality\\' not present in Profile JSON')\n        if 'needs' in _dict:\n            args['needs'] = [Trait._from_dict(x) for x in (_dict.get('needs'))]\n        else:\n            raise ValueError(\n                'Required property \\'needs\\' not present in Profile JSON')\n        if 'values' in _dict:\n            args['values'] = [\n                Trait._from_dict(x) for x in (_dict.get('values'))\n            ]\n        else:\n            raise ValueError(\n                'Required property \\'values\\' not present in Profile JSON')\n        if 'behavior' in _dict:\n            args['behavior'] = [\n                Behavior._from_dict(x) for x in (_dict.get('behavior'))\n            ]\n        if 'consumption_preferences' in _dict:\n            args['consumption_preferences'] = [\n                ConsumptionPreferencesCategory._from_dict(x)\n                for x in (_dict.get('consumption_preferences'))\n            ]\n        if 'warnings' in _dict:\n            args['warnings'] = [\n                Warning._from_dict(x) for x in (_dict.get('warnings'))\n            ]\n        else:\n            raise ValueError(\n                'Required property \\'warnings\\' not present in Profile JSON')\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(\n                self,\n                'processed_language') and self.processed_language is not None:\n            _dict['processed_language'] = self.processed_language\n        if hasattr(self, 'word_count') and self.word_count is not None:\n            _dict['word_count'] = self.word_count\n        if hasattr(\n                self,\n                'word_count_message') and self.word_count_message is not None:\n            _dict['word_count_message'] = self.word_count_message\n        if hasattr(self, 'personality') and self.personality is not None:\n            _dict['personality'] = [x._to_dict() for x in self.personality]\n        if hasattr(self, 'needs') and self.needs is not None:\n            _dict['needs'] = [x._to_dict() for x in self.needs]\n        if hasattr(self, 'values') and self.values is not None:\n            _dict['values'] = [x._to_dict() for x in self.values]\n        if hasattr(self, 'behavior') and self.behavior is not None:\n            _dict['behavior'] = [x._to_dict() for x in self.behavior]\n        if hasattr(self, 'consumption_preferences'\n                  ) and self.consumption_preferences is not None:\n            _dict['consumption_preferences'] = [\n                x._to_dict() for x in self.consumption_preferences\n            ]\n        if hasattr(self, 'warnings') and self.warnings is not None:\n            _dict['warnings'] = [x._to_dict() for x in self.warnings]\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ninitialize a Trait object from a json dictionary.", "response": "def _from_dict(cls, _dict):\n        \"\"\"Initialize a Trait object from a json dictionary.\"\"\"\n        args = {}\n        if 'trait_id' in _dict:\n            args['trait_id'] = _dict.get('trait_id')\n        else:\n            raise ValueError(\n                'Required property \\'trait_id\\' not present in Trait JSON')\n        if 'name' in _dict:\n            args['name'] = _dict.get('name')\n        else:\n            raise ValueError(\n                'Required property \\'name\\' not present in Trait JSON')\n        if 'category' in _dict:\n            args['category'] = _dict.get('category')\n        else:\n            raise ValueError(\n                'Required property \\'category\\' not present in Trait JSON')\n        if 'percentile' in _dict:\n            args['percentile'] = _dict.get('percentile')\n        else:\n            raise ValueError(\n                'Required property \\'percentile\\' not present in Trait JSON')\n        if 'raw_score' in _dict:\n            args['raw_score'] = _dict.get('raw_score')\n        if 'significant' in _dict:\n            args['significant'] = _dict.get('significant')\n        if 'children' in _dict:\n            args['children'] = [\n                Trait._from_dict(x) for x in (_dict.get('children'))\n            ]\n        return cls(**args)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn a json dictionary representing this model.", "response": "def _to_dict(self):\n        \"\"\"Return a json dictionary representing this model.\"\"\"\n        _dict = {}\n        if hasattr(self, 'trait_id') and self.trait_id is not None:\n            _dict['trait_id'] = self.trait_id\n        if hasattr(self, 'name') and self.name is not None:\n            _dict['name'] = self.name\n        if hasattr(self, 'category') and self.category is not None:\n            _dict['category'] = self.category\n        if hasattr(self, 'percentile') and self.percentile is not None:\n            _dict['percentile'] = self.percentile\n        if hasattr(self, 'raw_score') and self.raw_score is not None:\n            _dict['raw_score'] = self.raw_score\n        if hasattr(self, 'significant') and self.significant is not None:\n            _dict['significant'] = self.significant\n        if hasattr(self, 'children') and self.children is not None:\n            _dict['children'] = [x._to_dict() for x in self.children]\n        return _dict"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _to_dict(self):\n        _dict = {}\n        if hasattr(self, 'warning_id') and self.warning_id is not None:\n            _dict['warning_id'] = self.warning_id\n        if hasattr(self, 'message') and self.message is not None:\n            _dict['message'] = self.message\n        return _dict", "response": "Return a json dictionary representing this model."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ndelete all chunks for a symbol.", "response": "def delete(self, symbol, date_range=None):\n        \"\"\"\n        Delete all chunks for a symbol.\n\n        Which are, for the moment, fully contained in the passed in\n        date_range.\n\n        Parameters\n        ----------\n        symbol : `str`\n            symbol name for the item\n        date_range : `date.DateRange`\n            DateRange to delete ticks in\n        \"\"\"\n        query = {SYMBOL: symbol}\n        date_range = to_pandas_closed_closed(date_range)\n        if date_range is not None:\n            assert date_range.start and date_range.end\n            query[START] = {'$gte': date_range.start}\n            query[END] = {'$lte': date_range.end}\n        else:\n            # delete metadata on complete deletion\n            self._metadata.delete_one({SYMBOL: symbol})\n        return self._collection.delete_many(query)"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns the mongo read preference given an allow_secondary argument", "response": "def _read_preference(self, allow_secondary):\n        \"\"\" Return the mongo read preference given an 'allow_secondary' argument\n        \"\"\"\n        allow_secondary = self._allow_secondary if allow_secondary is None else allow_secondary\n        return ReadPreference.NEAREST if allow_secondary else ReadPreference.PRIMARY"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nread the data for the specified symbol from the version store.", "response": "def read(self, symbol, date_range=None, columns=None, include_images=False, allow_secondary=None,\n             _target_tick_count=0):\n        \"\"\"\n        Read data for the named symbol.  Returns a VersionedItem object with\n        a data and metdata element (as passed into write).\n\n        Parameters\n        ----------\n        symbol : `str`\n            symbol name for the item\n        date_range : `date.DateRange`\n            Returns ticks in the specified DateRange\n        columns : `list` of `str`\n            Columns (fields) to return from the tickstore\n        include_images : `bool`\n            Should images (/snapshots) be included in the read\n        allow_secondary : `bool` or `None`\n            Override the default behavior for allowing reads from secondary members of a cluster:\n            `None` : use the settings from the top-level `Arctic` object used to query this version store.\n            `True` : allow reads from secondary members\n            `False` : only allow reads from primary members\n\n        Returns\n        -------\n        pandas.DataFrame of data\n        \"\"\"\n        perf_start = dt.now()\n        rtn = {}\n        column_set = set()\n\n        multiple_symbols = not isinstance(symbol, string_types)\n\n        date_range = to_pandas_closed_closed(date_range)\n        query = self._symbol_query(symbol)\n        query.update(self._mongo_date_range_query(symbol, date_range))\n\n        if columns:\n            projection = dict([(SYMBOL, 1),\n                               (INDEX, 1),\n                               (START, 1),\n                               (VERSION, 1),\n                               (IMAGE_DOC, 1)] +\n                              [(COLUMNS + '.%s' % c, 1) for c in columns])\n            column_set.update([c for c in columns if c != 'SYMBOL'])\n        else:\n            projection = dict([(SYMBOL, 1),\n                               (INDEX, 1),\n                               (START, 1),\n                               (VERSION, 1),\n                               (COLUMNS, 1),\n                               (IMAGE_DOC, 1)])\n\n        column_dtypes = {}\n        ticks_read = 0\n        data_coll = self._collection.with_options(read_preference=self._read_preference(allow_secondary))\n        for b in data_coll.find(query, projection=projection).sort([(START, pymongo.ASCENDING)],):\n            data = self._read_bucket(b, column_set, column_dtypes,\n                                     multiple_symbols or (columns is not None and 'SYMBOL' in columns),\n                                     include_images, columns)\n            for k, v in iteritems(data):\n                try:\n                    rtn[k].append(v)\n                except KeyError:\n                    rtn[k] = [v]\n            # For testing\n            ticks_read += len(data[INDEX])\n            if _target_tick_count and ticks_read > _target_tick_count:\n                break\n\n        if not rtn:\n            raise NoDataFoundException(\"No Data found for {} in range: {}\".format(symbol, date_range))\n        rtn = self._pad_and_fix_dtypes(rtn, column_dtypes)\n\n        index = pd.to_datetime(np.concatenate(rtn[INDEX]), utc=True, unit='ms')\n        if columns is None:\n            columns = [x for x in rtn.keys() if x not in (INDEX, 'SYMBOL')]\n        if multiple_symbols and 'SYMBOL' not in columns:\n            columns = ['SYMBOL', ] + columns\n\n        if len(index) > 0:\n            arrays = [np.concatenate(rtn[k]) for k in columns]\n        else:\n            arrays = [[] for _ in columns]\n\n        if multiple_symbols:\n            sort = np.argsort(index, kind='mergesort')\n            index = index[sort]\n            arrays = [a[sort] for a in arrays]\n\n        t = (dt.now() - perf_start).total_seconds()\n        logger.info(\"Got data in %s secs, creating DataFrame...\" % t)\n        mgr = _arrays_to_mgr(arrays, columns, index, columns, dtype=None)\n        rtn = pd.DataFrame(mgr)\n        # Present data in the user's default TimeZone\n        rtn.index = rtn.index.tz_convert(mktz())\n\n        t = (dt.now() - perf_start).total_seconds()\n        ticks = len(rtn)\n        rate = int(ticks / t) if t != 0 else float(\"nan\")\n        logger.info(\"%d rows in %s secs: %s ticks/sec\" % (ticks, t, rate))\n        if not rtn.index.is_monotonic:\n            logger.error(\"TimeSeries data is out of order, sorting!\")\n            rtn = rtn.sort_index(kind='mergesort')\n        if date_range:\n            # FIXME: support DateRange.interval...\n            rtn = rtn.loc[date_range.start:date_range.end]\n        return rtn"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nwrite a list of market data events to the tickstore.", "response": "def write(self, symbol, data, initial_image=None, metadata=None):\n        \"\"\"\n        Writes a list of market data events.\n\n        Parameters\n        ----------\n        symbol : `str`\n            symbol name for the item\n        data : list of dicts or a pandas.DataFrame\n            List of ticks to store to the tick-store.\n            if a list of dicts, each dict must contain a 'index' datetime\n            if a pandas.DataFrame the index must be a Timestamp that can be converted to a datetime.\n            Index names will not be preserved.\n        initial_image : dict\n            Dict of the initial image at the start of the document. If this contains a 'index' entry it is\n            assumed to be the time of the timestamp of the index\n        metadata: dict\n            optional user defined metadata - one per symbol\n        \"\"\"\n        pandas = False\n        # Check for overlapping data\n        if isinstance(data, list):\n            start = data[0]['index']\n            end = data[-1]['index']\n        elif isinstance(data, pd.DataFrame):\n            start = data.index[0].to_pydatetime()\n            end = data.index[-1].to_pydatetime()\n            pandas = True\n        else:\n            raise UnhandledDtypeException(\"Can't persist type %s to tickstore\" % type(data))\n        self._assert_nonoverlapping_data(symbol, to_dt(start), to_dt(end))\n\n        if pandas:\n            buckets = self._pandas_to_buckets(data, symbol, initial_image)\n        else:\n            buckets = self._to_buckets(data, symbol, initial_image)\n        self._write(buckets)\n\n        if metadata:\n            self._metadata.replace_one({SYMBOL: symbol},\n                                       {SYMBOL: symbol, META: metadata},\n                                       upsert=True)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _str_dtype(dtype):\n        assert dtype.byteorder != '>'\n        if dtype.kind == 'i':\n            assert dtype.itemsize == 8\n            return 'int64'\n        elif dtype.kind == 'f':\n            assert dtype.itemsize == 8\n            return 'float64'\n        elif dtype.kind == 'U':\n            return 'U%d' % (dtype.itemsize / 4)\n        else:\n            raise UnhandledDtypeException(\"Bad dtype '%s'\" % dtype)", "response": "Return a string representation of a dtype."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef max_date(self, symbol):\n        res = self._collection.find_one({SYMBOL: symbol}, projection={ID: 0, END: 1},\n                                        sort=[(START, pymongo.DESCENDING)])\n        if res is None:\n            raise NoDataFoundException(\"No Data found for {}\".format(symbol))\n        return utc_dt_to_local_dt(res[END])", "response": "Returns the maximum datetime stored for a particular symbol"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns the minimum datetime stored for a particular symbol", "response": "def min_date(self, symbol):\n        \"\"\"\n        Return the minimum datetime stored for a particular symbol\n\n        Parameters\n        ----------\n        symbol : `str`\n            symbol name for the item\n        \"\"\"\n        res = self._collection.find_one({SYMBOL: symbol}, projection={ID: 0, START: 1},\n                                        sort=[(START, pymongo.ASCENDING)])\n        if res is None:\n            raise NoDataFoundException(\"No Data found for {}\".format(symbol))\n        return utc_dt_to_local_dt(res[START])"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef to_chunks(self, df, **kwargs):\n        if len(df) > 0:\n            yield b'NA', b'NA', b'NA', df", "response": "Yields the nans of the data in a list of tuples."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nauthenticating with a database.", "response": "def authenticate(db, user, password):\n    \"\"\"\n    Return True / False on authentication success.\n\n    PyMongo 2.6 changed the auth API to raise on Auth failure.\n    \"\"\"\n    try:\n        logger.debug(\"Authenticating {} with {}\".format(db, user))\n        return db.authenticate(user, password)\n    except OperationFailure as e:\n        logger.debug(\"Auth Error %s\" % e)\n    return False"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_auth(host, app_name, database_name):\n    from .hooks import _get_auth_hook\n    return _get_auth_hook(host, app_name, database_name)", "response": "Returns an authentication hook to allow plugging in custom credential providers"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn a Arctic library for the given connection string.", "response": "def get_arctic_lib(connection_string, **kwargs):\n    \"\"\"\n    Returns a mongo library for the given connection string\n\n    Parameters\n    ---------\n    connection_string: `str`\n        Format must be one of the following:\n            library@trading for known mongo servers\n            library@hostname:port\n\n    Returns:\n    --------\n    Arctic library\n    \"\"\"\n    m = CONNECTION_STR.match(connection_string)\n    if not m:\n        raise ValueError(\"connection string incorrectly formed: %s\" % connection_string)\n    library, host = m.group(1), m.group(2)\n    return _get_arctic(host, **kwargs)[library]"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef do_db_auth(host, connection, db_name):\n    admin_creds = get_auth(host, 'admin', 'admin')\n    user_creds = get_auth(host, 'arctic', db_name)\n\n    # Attempt to authenticate the connection\n    # Try at 'admin level' first as this allows us to enableSharding, which we want\n    if admin_creds is None:\n        # Get ordinary credentials for authenticating against the DB\n        if user_creds is None:\n            logger.error(\"You need credentials for db '%s' on '%s', or admin credentials\" % (db_name, host))\n            return False\n        if not authenticate(connection[db_name], user_creds.user, user_creds.password):\n            logger.error(\"Failed to authenticate to db '%s' on '%s', using user credentials\" % (db_name, host))\n            return False\n        return True\n    elif not authenticate(connection.admin, admin_creds.user, admin_creds.password):\n        logger.error(\"Failed to authenticate to '%s' as Admin. Giving up.\" % (host))\n        return False\n    # Ensure we attempt to auth against the user DB, for non-priviledged users to get access\n    authenticate(connection[db_name], user_creds.user, user_creds.password)\n    return True", "response": "Attempts to authenticate against the mongo instance."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef fancy_group_by(df, grouping_level=0, aggregate_level=1, method='last', max_=None, min_=None, within=None):\n    if method not in ('first', 'last'):\n        raise ValueError('Invalid method')\n\n    if isinstance(aggregate_level, six.string_types):\n        aggregate_level = df.index.names.index(aggregate_level)\n\n    # Trim any rows outside the aggregate value bounds\n    if max_ is not None or min_ is not None or within is not None:\n        agg_idx = df.index.get_level_values(aggregate_level)\n        mask = np.full(len(agg_idx), True, dtype='b1')\n        if max_ is not None:\n            mask &= (agg_idx <= max_)\n        if min_ is not None:\n            mask &= (agg_idx >= min_)\n        if within is not None:\n            group_idx = df.index.get_level_values(grouping_level)\n            if isinstance(agg_idx, pd.DatetimeIndex):\n                mask &= (group_idx >= agg_idx.shift(-1, freq=within))\n            else:\n                mask &= (group_idx >= (agg_idx - within))\n        df = df.loc[mask]\n\n    # The sort order must be correct in order of grouping_level -> aggregate_level for the aggregation methods\n    # to work properly. We can check the sortdepth to see if this is in fact the case and resort if necessary.\n    # TODO: this might need tweaking if the levels are around the wrong way\n    if df.index.lexsort_depth < (aggregate_level + 1):\n        df = df.sort_index(level=grouping_level)\n\n    gb = df.groupby(level=grouping_level)\n    if method == 'last':\n        return gb.last()\n    return gb.first()", "response": "Dataframe group - by operation that supports aggregating by different methods on the index."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef multi_index_insert_row(df, index_row, values_row):\n    row_index = pd.MultiIndex(levels=[[i] for i in index_row],\n                              labels=[[0] for i in index_row])\n    row = pd.DataFrame(values_row, index=row_index, columns=df.columns)\n    df = pd.concat((df, row))\n    if df.index.lexsort_depth == len(index_row) and df.index[-2] < df.index[-1]:\n        # We've just appended a row to an already-sorted dataframe\n        return df\n    # The df wasn't sorted or the row has to be put in the middle somewhere\n    return df.sort_index()", "response": "Return a new dataframe with a row inserted for a multi - index dataframe."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef insert_at(df, sample_date, values):\n    observed_dt = dt(datetime.now())\n    return multi_index_insert_row(df, [sample_date, observed_dt], values)", "response": "Insert some values into a bi - temporal dataframe."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef register_library_type(name, type_):\n    if name in LIBRARY_TYPES:\n        raise ArcticException(\"Library %s already registered as %s\" % (name, LIBRARY_TYPES[name]))\n    LIBRARY_TYPES[name] = type_", "response": "Register a Arctic Library Type handler"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a list of Arctic library names from a cached collection.", "response": "def _list_libraries_cached(self, newer_than_secs=-1):\n        \"\"\"\n        Returns\n        -------\n        List of Arctic library names from a cached collection (global per mongo cluster) in mongo.\n        Long term list_libraries should have a use_cached argument.\n        \"\"\"\n        _ = self._conn  # Ensures the connection exists and cache is initialized with it.\n        cache_data = self._cache.get('list_libraries', newer_than_secs)\n        if cache_data:\n            logger.debug('Library names are in cache.')\n            return cache_data\n\n        return self._list_libraries()"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef library_exists(self, library):\n        exists = False\n        try:\n            # This forces auth errors, and to fall back to the slower \"list_collections\"\n            ArcticLibraryBinding(self, library).get_library_type()\n            # This will obtain the library, if no exception thrown we have verified its existence\n            self.get_library(library)\n            exists = True\n        except OperationFailure:\n            exists = library in self.list_libraries()\n        except LibraryNotFoundException:\n            pass\n        return exists", "response": "Check whether a given library exists."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncreate an Arctic Library or a particular type.", "response": "def initialize_library(self, library, lib_type=VERSION_STORE, **kwargs):\n        \"\"\"\n        Create an Arctic Library or a particular type.\n\n        Parameters\n        ----------\n        library : `str`\n            The name of the library. e.g. 'library' or 'user.library'\n\n        lib_type : `str`\n            The type of the library.  e.g. arctic.VERSION_STORE or arctic.TICK_STORE\n            Or any type registered with register_library_type\n            Default: arctic.VERSION_STORE\n\n        kwargs :\n            Arguments passed to the Library type for initialization.\n        \"\"\"\n        lib = ArcticLibraryBinding(self, library)\n        # check that we don't create too many namespaces\n        # can be disabled check_library_count=False\n        check_library_count = kwargs.pop('check_library_count', True)\n        if len(self._conn[lib.database_name].list_collection_names()) > 5000 and check_library_count:\n            raise ArcticException(\"Too many namespaces %s, not creating: %s\" %\n                                  (len(self._conn[lib.database_name].list_collection_names()), library))\n        lib.set_library_type(lib_type)\n        LIBRARY_TYPES[lib_type].initialize_library(lib, **kwargs)\n        # Add a 10G quota just in case the user is calling this with API.\n        if not lib.get_quota():\n            lib.set_quota(10 * 1024 * 1024 * 1024)\n\n        self._cache.append('list_libraries', self._sanitize_lib_name(library))"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ndeleting an Arctic Library and all associated collections in the MongoDB.", "response": "def delete_library(self, library):\n        \"\"\"\n        Delete an Arctic Library, and all associated collections in the MongoDB.\n\n        Parameters\n        ----------\n        library : `str`\n            The name of the library. e.g. 'library' or 'user.library'\n        \"\"\"\n        lib = ArcticLibraryBinding(self, library)\n        colname = lib.get_top_level_collection().name\n        if not [c for c in lib._db.list_collection_names(False) if re.match(r\"^{}([\\.].*)?$\".format(colname), c)]:\n            logger.info('Nothing to delete. Arctic library %s does not exist.' % colname)\n        logger.info('Dropping collection: %s' % colname)\n        lib._db.drop_collection(colname)\n        for coll in lib._db.list_collection_names():\n            if coll.startswith(colname + '.'):\n                logger.info('Dropping collection: %s' % coll)\n                lib._db.drop_collection(coll)\n        if library in self._library_cache:\n            del self._library_cache[library]\n            del self._library_cache[lib.get_name()]\n\n        self._cache.delete_item_from_key('list_libraries', self._sanitize_lib_name(library))"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get_library(self, library):\n        if library in self._library_cache:\n            return self._library_cache[library]\n\n        try:\n            error = None\n            lib = ArcticLibraryBinding(self, library)\n            lib_type = lib.get_library_type()\n        except (OperationFailure, AutoReconnect) as e:\n            error = e\n\n        if error:\n            raise LibraryNotFoundException(\"Library %s was not correctly initialized in %s.\\nReason: %r)\" %\n                                           (library, self, error))\n        elif not lib_type:\n            raise LibraryNotFoundException(\"Library %s was not correctly initialized in %s.\" %\n                                           (library, self))\n        elif lib_type not in LIBRARY_TYPES:\n            raise LibraryNotFoundException(\"Couldn't load LibraryType '%s' for '%s' (has the class been registered?)\" %\n                                           (lib_type, library))\n        instance = LIBRARY_TYPES[lib_type](lib)\n        self._library_cache[library] = instance\n        # The library official name may be different from 'library': e.g. 'library' vs 'user.library'\n        self._library_cache[lib.get_name()] = instance\n        return self._library_cache[library]", "response": "Returns the object with the specified name. Can generally use slicing to return the library instance."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef rename_library(self, from_lib, to_lib):\n        to_colname = to_lib\n        if '.' in from_lib and '.' in to_lib:\n            if from_lib.split('.')[0] != to_lib.split('.')[0]:\n                raise ValueError(\"Collection can only be renamed in the same database\")\n            to_colname = to_lib.split('.')[1]\n\n        lib = ArcticLibraryBinding(self, from_lib)\n        colname = lib.get_top_level_collection().name\n\n        logger.info('Renaming collection: %s' % colname)\n        lib._db[colname].rename(to_colname)\n        for coll in lib._db.list_collection_names():\n            if coll.startswith(colname + '.'):\n                lib._db[coll].rename(coll.replace(colname, to_colname))\n\n        if from_lib in self._library_cache:\n            del self._library_cache[from_lib]\n            del self._library_cache[lib.get_name()]\n\n        self._cache.update_item_for_key(\n            'list_libraries', self._sanitize_lib_name(from_lib), self._sanitize_lib_name(to_lib))", "response": "Renames a library\n\n        Parameters\n        ----------\n        from_lib: str\n            The name of the library to be renamed\n        to_lib: str\n            The new name of the library"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns the canonical database name and library for the passed in string library.", "response": "def _parse_db_lib(cls, library):\n        \"\"\"\n        Returns the canonical (database_name, library) for the passed in\n        string 'library'.\n        \"\"\"\n        database_name = library.split('.', 2)\n        if len(database_name) == 2:\n            library = database_name[1]\n            if database_name[0].startswith(cls.DB_PREFIX):\n                database_name = database_name[0]\n            else:\n                database_name = cls.DB_PREFIX + '_' + database_name[0]\n        else:\n            database_name = cls.DB_PREFIX\n        return database_name, library"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef set_quota(self, quota_bytes):\n        self.set_library_metadata(ArcticLibraryBinding.QUOTA, quota_bytes)\n        self.quota = quota_bytes\n        self.quota_countdown = 0", "response": "Set a quota on this user library."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef check_quota(self):\n        # Don't check on every write, that would be slow\n        if self.quota_countdown > 0:\n            self.quota_countdown -= 1\n            return\n\n        # Re-cache the quota after the countdown\n        self.quota = self.get_library_metadata(ArcticLibraryBinding.QUOTA)\n        if self.quota is None or self.quota == 0:\n            self.quota = 0\n            return\n\n        # Figure out whether the user has exceeded their quota\n        library = self.arctic[self.get_name()]\n        stats = library.stats()\n\n        def to_gigabytes(bytes_):\n            return bytes_ / 1024. / 1024. / 1024.\n\n        # Have we exceeded our quota?\n        size = stats['totals']['size']\n        count = stats['totals']['count']\n        if size >= self.quota:\n            raise QuotaExceededException(\"Mongo Quota Exceeded: %s %.3f / %.0f GB used\" % (\n                '.'.join([self.database_name, self.library]),\n                to_gigabytes(size),\n                to_gigabytes(self.quota)))\n\n        # Quota not exceeded, print an informational message and return\n        try:\n            avg_size = size // count if count > 1 else 100 * 1024\n            remaining = self.quota - size\n            remaining_count = remaining / avg_size\n            if remaining_count < 100 or float(remaining) / self.quota < 0.1:\n                logger.warning(\"Mongo Quota: %s %.3f / %.0f GB used\" % (\n                    '.'.join([self.database_name, self.library]),\n                    to_gigabytes(size),\n                    to_gigabytes(self.quota)))\n            else:\n                logger.info(\"Mongo Quota: %s %.3f / %.0f GB used\" % (\n                    '.'.join([self.database_name, self.library]),\n                    to_gigabytes(size),\n                    to_gigabytes(self.quota)))\n\n            # Set-up a timer to prevent us for checking for a few writes.\n            # This will check every average half-life\n            self.quota_countdown = int(max(remaining_count // 2, 1))\n        except Exception as e:\n            logger.warning(\"Encountered an exception while calculating quota statistics: %s\" % str(e))", "response": "Check whether the user is within the quota."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _checksum(self, fields, data):\n        sha = hashlib.sha1()\n        for field in fields:\n            sha.update(field)\n        sha.update(data)\n        return Binary(sha.digest())", "response": "Checksum the passed in dictionary"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef delete(self, symbol, chunk_range=None, audit=None):\n        if chunk_range is not None:\n            sym = self._get_symbol_info(symbol)\n            # read out chunks that fall within the range and filter out\n            # data within the range\n            df = self.read(symbol, chunk_range=chunk_range, filter_data=False)\n            row_adjust = len(df)\n            if not df.empty:\n                df = CHUNKER_MAP[sym[CHUNKER]].exclude(df, chunk_range)\n\n                # remove chunks, and update any remaining data\n                query = {SYMBOL: symbol}\n                query.update(CHUNKER_MAP[sym[CHUNKER]].to_mongo(chunk_range))\n                self._collection.delete_many(query)\n                self._mdata.delete_many(query)\n                self.update(symbol, df)\n\n                # update symbol metadata (rows and chunk count)\n                sym = self._get_symbol_info(symbol)\n                sym[LEN] -= row_adjust\n                sym[CHUNK_COUNT] = mongo_count(self._collection, filter={SYMBOL: symbol})\n                self._symbols.replace_one({SYMBOL: symbol}, sym)\n\n        else:\n            query = {SYMBOL: symbol}\n            self._collection.delete_many(query)\n            self._symbols.delete_many(query)\n            self._mdata.delete_many(query)\n\n        if audit is not None:\n            audit['symbol'] = symbol\n            if chunk_range is not None:\n                audit['rows_deleted'] = row_adjust\n                audit['action'] = 'range delete'\n            else:\n                audit['action'] = 'symbol delete'\n\n            self._audit.insert_one(audit)", "response": "Delete all chunks for a symbol or optionally a range of chunks within a range."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning a list of all symbols in the library", "response": "def list_symbols(self, partial_match=None):\n        \"\"\"\n        Returns all symbols in the library\n\n        Parameters\n        ----------\n        partial: None or str\n            if not none, use this string to do a partial match on symbol names\n\n        Returns\n        -------\n        list of str\n        \"\"\"\n        symbols = self._symbols.distinct(SYMBOL)\n        if partial_match is None:\n            return symbols\n        return [x for x in symbols if partial_match in x]"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nrename a symbol in the cache", "response": "def rename(self, from_symbol, to_symbol, audit=None):\n        \"\"\"\n        Rename a symbol\n\n        Parameters\n        ----------\n        from_symbol: str\n            the existing symbol that will be renamed\n        to_symbol: str\n            the new symbol name\n        audit: dict\n            audit information\n        \"\"\"\n\n        sym = self._get_symbol_info(from_symbol)\n        if not sym:\n            raise NoDataFoundException('No data found for %s' % (from_symbol))\n\n        if self._get_symbol_info(to_symbol) is not None:\n            raise Exception('Symbol %s already exists' % (to_symbol))\n\n        mongo_retry(self._collection.update_many)({SYMBOL: from_symbol},\n                                                  {'$set': {SYMBOL: to_symbol}})\n        mongo_retry(self._symbols.update_one)({SYMBOL: from_symbol},\n                                              {'$set': {SYMBOL: to_symbol}})\n        mongo_retry(self._mdata.update_many)({SYMBOL: from_symbol},\n                                             {'$set': {SYMBOL: to_symbol}})\n        mongo_retry(self._audit.update_many)({'symbol': from_symbol},\n                                             {'$set': {'symbol': to_symbol}})\n        if audit is not None:\n            audit['symbol'] = to_symbol\n            audit['action'] = 'symbol rename'\n            audit['old_symbol'] = from_symbol\n            self._audit.insert_one(audit)"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreads data for a given symbol from the database.", "response": "def read(self, symbol, chunk_range=None, filter_data=True, **kwargs):\n        \"\"\"\n        Reads data for a given symbol from the database.\n\n        Parameters\n        ----------\n        symbol: str, or list of str\n            the symbol(s) to retrieve\n        chunk_range: object\n            corresponding range object for the specified chunker (for\n            DateChunker it is a DateRange object or a DatetimeIndex,\n            as returned by pandas.date_range\n        filter_data: boolean\n            perform chunk level filtering on the data (see filter in _chunker)\n            only applicable when chunk_range is specified\n        kwargs: ?\n            values passed to the serializer. Varies by serializer\n\n        Returns\n        -------\n        DataFrame or Series, or in the case when multiple symbols are given,\n        returns a dict of symbols (symbol -> dataframe/series)\n        \"\"\"\n        if not isinstance(symbol, list):\n            symbol = [symbol]\n\n        sym = self._get_symbol_info(symbol)\n        if not sym:\n            raise NoDataFoundException('No data found for %s' % (symbol))\n\n        spec = {SYMBOL: {'$in': symbol}}\n        chunker = CHUNKER_MAP[sym[0][CHUNKER]]\n        deser = SER_MAP[sym[0][SERIALIZER]].deserialize\n\n        if chunk_range is not None:\n            spec.update(chunker.to_mongo(chunk_range))\n\n        by_start_segment = [(SYMBOL, pymongo.ASCENDING),\n                            (START, pymongo.ASCENDING),\n                            (SEGMENT, pymongo.ASCENDING)]\n        segment_cursor = self._collection.find(spec, sort=by_start_segment)\n\n        chunks = defaultdict(list)\n        for _, segments in groupby(segment_cursor, key=lambda x: (x[START], x[SYMBOL])):\n            segments = list(segments)\n            mdata = self._mdata.find_one({SYMBOL: segments[0][SYMBOL],\n                                          START: segments[0][START],\n                                          END: segments[0][END]})\n\n            # when len(segments) == 1, this is essentially a no-op\n            # otherwise, take all segments and reassemble the data to one chunk\n            chunk_data = b''.join([doc[DATA] for doc in segments])\n            chunks[segments[0][SYMBOL]].append({DATA: chunk_data, METADATA: mdata})\n\n        skip_filter = not filter_data or chunk_range is None\n\n        if len(symbol) > 1:\n            return {sym: deser(chunks[sym], **kwargs) if skip_filter else chunker.filter(deser(chunks[sym], **kwargs), chunk_range) for sym in symbol}\n        else:\n            return deser(chunks[symbol[0]], **kwargs) if skip_filter else chunker.filter(deser(chunks[symbol[0]], **kwargs), chunk_range)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef read_audit_log(self, symbol=None):\n        if symbol:\n            return [x for x in self._audit.find({'symbol': symbol}, {'_id': False})]\n        return [x for x in self._audit.find({}, {'_id': False})]", "response": "Reads the audit log for a specific symbol."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef write(self, symbol, item, metadata=None, chunker=DateChunker(), audit=None, **kwargs):\n        if not isinstance(item, (DataFrame, Series)):\n            raise Exception(\"Can only chunk DataFrames and Series\")\n\n        self._arctic_lib.check_quota()\n\n        previous_shas = []\n        doc = {}\n        meta = {}\n\n        doc[SYMBOL] = symbol\n        doc[LEN] = len(item)\n        doc[SERIALIZER] = self.serializer.TYPE\n        doc[CHUNKER] = chunker.TYPE\n        doc[USERMETA] = metadata\n\n        sym = self._get_symbol_info(symbol)\n        if sym:\n            previous_shas = set([Binary(x[SHA]) for x in self._collection.find({SYMBOL: symbol},\n                                                                               projection={SHA: True, '_id': False},\n                                                                               )])\n        ops = []\n        meta_ops = []\n        chunk_count = 0\n\n        for start, end, chunk_size, record in chunker.to_chunks(item, **kwargs):\n            chunk_count += 1\n            data = self.serializer.serialize(record)\n            doc[CHUNK_SIZE] = chunk_size\n            doc[METADATA] = {'columns': data[METADATA][COLUMNS] if COLUMNS in data[METADATA] else ''}\n            meta = data[METADATA]\n\n            for i in xrange(int(len(data[DATA]) / MAX_CHUNK_SIZE + 1)):\n                chunk = {DATA: Binary(data[DATA][i * MAX_CHUNK_SIZE: (i + 1) * MAX_CHUNK_SIZE])}\n                chunk[SEGMENT] = i\n                chunk[START] = meta[START] = start\n                chunk[END] = meta[END] = end\n                chunk[SYMBOL] = meta[SYMBOL] = symbol\n                dates = [chunker.chunk_to_str(start), chunker.chunk_to_str(end), str(chunk[SEGMENT]).encode('ascii')]\n                chunk[SHA] = self._checksum(dates, chunk[DATA])\n\n                meta_ops.append(pymongo.ReplaceOne({SYMBOL: symbol,\n                                                    START: start,\n                                                    END: end},\n                                                   meta, upsert=True))\n\n                if chunk[SHA] not in previous_shas:\n                    ops.append(pymongo.UpdateOne({SYMBOL: symbol,\n                                                  START: start,\n                                                  END: end,\n                                                  SEGMENT: chunk[SEGMENT]},\n                                                 {'$set': chunk}, upsert=True))\n                else:\n                    # already exists, dont need to update in mongo\n                    previous_shas.remove(chunk[SHA])\n\n        if ops:\n            self._collection.bulk_write(ops, ordered=False)\n        if meta_ops:\n            self._mdata.bulk_write(meta_ops, ordered=False)\n\n        doc[CHUNK_COUNT] = chunk_count\n        doc[APPEND_COUNT] = 0\n\n        if previous_shas:\n            mongo_retry(self._collection.delete_many)({SYMBOL: symbol, SHA: {'$in': list(previous_shas)}})\n\n        mongo_retry(self._symbols.update_one)({SYMBOL: symbol},\n                                              {'$set': doc},\n                                              upsert=True)\n        if audit is not None:\n            audit['symbol'] = symbol\n            audit['action'] = 'write'\n            audit['chunks'] = chunk_count\n            self._audit.insert_one(audit)", "response": "Writes data from item to symbol in the database."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nappending data from item to symbol s data in the database.", "response": "def append(self, symbol, item, upsert=False, metadata=None, audit=None, **kwargs):\n        \"\"\"\n        Appends data from item to symbol's data in the database.\n\n        Is not idempotent\n\n        Parameters\n        ----------\n        symbol: str\n            the symbol for the given item in the DB\n        item: DataFrame or Series\n            the data to append\n        upsert:\n            write data if symbol does not exist\n        metadata: ?\n            optional per symbol metadata\n        audit: dict\n            optional audit information\n        kwargs:\n            passed to write if upsert is true and symbol does not exist\n        \"\"\"\n        sym = self._get_symbol_info(symbol)\n        if not sym:\n            if upsert:\n                return self.write(symbol, item, metadata=metadata, audit=audit, **kwargs)\n            else:\n                raise NoDataFoundException(\"Symbol does not exist.\")\n        if audit is not None:\n            audit['symbol'] = symbol\n            audit['action'] = 'append'\n        self.__update(sym, item, metadata=metadata, combine_method=SER_MAP[sym[SERIALIZER]].combine, audit=audit)"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nupdates the user s data in DB with the data in item.", "response": "def update(self, symbol, item, metadata=None, chunk_range=None, upsert=False, audit=None, **kwargs):\n        \"\"\"\n        Overwrites data in DB with data in item for the given symbol.\n\n        Is idempotent\n\n        Parameters\n        ----------\n        symbol: str\n            the symbol for the given item in the DB\n        item: DataFrame or Series\n            the data to update\n        metadata: ?\n            optional per symbol metadata\n        chunk_range: None, or a range object\n            If a range is specified, it will clear/delete the data within the\n            range and overwrite it with the data in item. This allows the user\n            to update with data that might only be a subset of the\n            original data.\n        upsert: bool\n            if True, will write the data even if the symbol does not exist.\n        audit: dict\n            optional audit information\n        kwargs:\n            optional keyword args passed to write during an upsert. Includes:\n            chunk_size\n            chunker\n        \"\"\"\n        sym = self._get_symbol_info(symbol)\n        if not sym:\n            if upsert:\n                return self.write(symbol, item, metadata=metadata, audit=audit, **kwargs)\n            else:\n                raise NoDataFoundException(\"Symbol does not exist.\")\n        if audit is not None:\n            audit['symbol'] = symbol\n            audit['action'] = 'update'\n        if chunk_range is not None:\n            if len(CHUNKER_MAP[sym[CHUNKER]].filter(item, chunk_range)) == 0:\n                raise Exception('Range must be inclusive of data')\n            self.__update(sym, item, metadata=metadata, combine_method=self.serializer.combine, chunk_range=chunk_range, audit=audit)\n        else:\n            self.__update(sym, item, metadata=metadata, combine_method=lambda old, new: new, chunk_range=chunk_range, audit=audit)"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn information about the symbol in a dictionary.", "response": "def get_info(self, symbol):\n        \"\"\"\n        Returns information about the symbol, in a dictionary\n\n        Parameters\n        ----------\n        symbol: str\n            the symbol for the given item in the DB\n\n        Returns\n        -------\n        dictionary\n        \"\"\"\n        sym = self._get_symbol_info(symbol)\n        if not sym:\n            raise NoDataFoundException(\"Symbol does not exist.\")\n        ret = {}\n        ret['chunk_count'] = sym[CHUNK_COUNT]\n        ret['len'] = sym[LEN]\n        ret['appended_rows'] = sym[APPEND_COUNT]\n        ret['metadata'] = sym[METADATA] if METADATA in sym else None\n        ret['chunker'] = sym[CHUNKER]\n        ret['chunk_size'] = sym[CHUNK_SIZE] if CHUNK_SIZE in sym else 0\n        ret['serializer'] = sym[SERIALIZER]\n        return ret"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nread user defined metadata out for the given symbol.", "response": "def read_metadata(self, symbol):\n        '''\n        Reads user defined metadata out for the given symbol\n\n        Parameters\n        ----------\n        symbol: str\n            symbol for the given item in the DB\n\n        Returns\n        -------\n        ?\n        '''\n        sym = self._get_symbol_info(symbol)\n        if not sym:\n            raise NoDataFoundException(\"Symbol does not exist.\")\n        x = self._symbols.find_one({SYMBOL: symbol})\n        return x[USERMETA] if USERMETA in x else None"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nwriting user defined metadata for the given symbol", "response": "def write_metadata(self, symbol, metadata):\n        '''\n        writes user defined metadata for the given symbol\n\n        Parameters\n        ----------\n        symbol: str\n            symbol for the given item in the DB\n        metadata: ?\n            metadata to write\n        '''\n        sym = self._get_symbol_info(symbol)\n        if not sym:\n            raise NoDataFoundException(\"Symbol does not exist.\")\n\n        sym[USERMETA] = metadata\n        self._symbols.replace_one({SYMBOL: symbol}, sym)"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns a generator of start and end tuples for each chunk in the given symbol", "response": "def get_chunk_ranges(self, symbol, chunk_range=None, reverse=False):\n        \"\"\"\n        Returns a generator of (Start, End) tuples for each chunk in the symbol\n\n        Parameters\n        ----------\n        symbol: str\n            the symbol for the given item in the DB\n        chunk_range: None, or a range object\n            allows you to subset the chunks by range\n        reverse: boolean\n            return the chunk ranges in reverse order\n\n        Returns\n        -------\n        generator\n        \"\"\"\n        sym = self._get_symbol_info(symbol)\n        if not sym:\n            raise NoDataFoundException(\"Symbol does not exist.\")\n        c = CHUNKER_MAP[sym[CHUNKER]]\n\n        # all symbols have a segment 0\n        spec = {SYMBOL: symbol, SEGMENT: 0}\n        if chunk_range is not None:\n            spec.update(CHUNKER_MAP[sym[CHUNKER]].to_mongo(chunk_range))\n\n        for x in self._collection.find(spec,\n                                       projection=[START, END],\n                                       sort=[(START, pymongo.ASCENDING if not reverse else pymongo.DESCENDING)]):\n            yield (c.chunk_to_str(x[START]), c.chunk_to_str(x[END]))"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef iterator(self, symbol, chunk_range=None):\n        sym = self._get_symbol_info(symbol)\n        if not sym:\n            raise NoDataFoundException(\"Symbol does not exist.\")\n\n        c = CHUNKER_MAP[sym[CHUNKER]]\n\n        for chunk in list(self.get_chunk_ranges(symbol, chunk_range=chunk_range)):\n            yield self.read(symbol, chunk_range=c.to_range(chunk[0], chunk[1]))", "response": "Returns an iterator that iterates over the items in the DB in ascending order."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef reverse_iterator(self, symbol, chunk_range=None):\n        sym = self._get_symbol_info(symbol)\n        if not sym:\n            raise NoDataFoundException(\"Symbol does not exist.\")\n\n        c = CHUNKER_MAP[sym[CHUNKER]]\n\n        for chunk in list(self.get_chunk_ranges(symbol, chunk_range=chunk_range, reverse=True)):\n            yield self.read(symbol, chunk_range=c.to_range(chunk[0], chunk[1]))", "response": "Returns an iterator that iterates over the items in descending order."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef mongo_retry(f):\n    log_all_exceptions = 'arctic' in f.__module__ if f.__module__ else False\n\n    @wraps(f)\n    def f_retry(*args, **kwargs):\n        global _retry_count, _in_retry\n        top_level = not _in_retry\n        _in_retry = True\n        try:\n            while True:\n                try:\n                    return f(*args, **kwargs)\n                except (DuplicateKeyError, ServerSelectionTimeoutError) as e:\n                    # Re-raise errors that won't go away.\n                    _handle_error(f, e, _retry_count, **_get_host(args))\n                    raise\n                except (OperationFailure, AutoReconnect) as e:\n                    _retry_count += 1\n                    _handle_error(f, e, _retry_count, **_get_host(args))\n                except Exception as e:\n                    if log_all_exceptions:\n                        _log_exception(f.__name__, e, _retry_count, **_get_host(args))\n                    raise\n        finally:\n            if top_level:\n                _in_retry = False\n                _retry_count = 0\n    return f_retry", "response": "Decorator that retries the given function if it fails."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _start_end(date_range, dts):\n    # FIXME: timezones\n    assert len(dts)\n    _assert_no_timezone(date_range)\n    date_range = to_pandas_closed_closed(date_range, add_tz=False)\n    start = np.datetime64(date_range.start) if date_range.start else dts[0]\n    end = np.datetime64(date_range.end) if date_range.end else dts[-1]\n    return start, end", "response": "Return start and end of the passed\n    in datetimes."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ngenerates index of datetime64 - > item offset.", "response": "def _segment_index(self, recarr, existing_index, start, new_segments):\n        \"\"\"\n        Generate index of datetime64 -> item offset.\n\n        Parameters:\n        -----------\n        new_data: new data being written (or appended)\n        existing_index: index field from the versions document of the previous version\n        start: first (0-based) offset of the new data\n        segments: list of offsets. Each offset is the row index of the\n                  the last row of a particular chunk relative to the start of the _original_ item.\n                  array(new_data) - segments = array(offsets in item)\n\n        Returns:\n        --------\n        Binary(compress(array([(index, datetime)]))\n            Where index is the 0-based index of the datetime in the DataFrame\n        \"\"\"\n        # find the index of the first datetime64 column\n        idx_col = self._datetime64_index(recarr)\n        # if one exists let's create the index on it\n        if idx_col is not None:\n            new_segments = np.array(new_segments, dtype='i8')\n            last_rows = recarr[new_segments - start]\n            # create numpy index\n            index = np.core.records.fromarrays([last_rows[idx_col]] + [new_segments, ], dtype=INDEX_DTYPE)\n            # append to existing index if exists\n            if existing_index:\n                # existing_index_arr is read-only but it's never written to\n                existing_index_arr = np.frombuffer(decompress(existing_index), dtype=INDEX_DTYPE)\n                if start > 0:\n                    existing_index_arr = existing_index_arr[existing_index_arr['index'] < start]\n                index = np.concatenate((existing_index_arr, index))\n            return Binary(compress(index.tostring()))\n        elif existing_index:\n            raise ArcticException(\"Could not find datetime64 index in item but existing data contains one\")\n        return None"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ngives a np. recarray find the first datetime64 column", "response": "def _datetime64_index(self, recarr):\n        \"\"\" Given a np.recarray find the first datetime64 column \"\"\"\n        # TODO: Handle multi-indexes\n        names = recarr.dtype.names\n        for name in names:\n            if recarr[name].dtype == DTN64_DTYPE:\n                return name\n        return None"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ngiving a version return the chunks associated with that version and the date_range.", "response": "def _index_range(self, version, symbol, date_range=None, **kwargs):\n        \"\"\" Given a version, read the segment_index and return the chunks associated\n        with the date_range. As the segment index is (id -> last datetime)\n        we need to take care in choosing the correct chunks. \"\"\"\n        if date_range and 'segment_index' in version:\n            # index is read-only but it's never written to\n            index = np.frombuffer(decompress(version['segment_index']), dtype=INDEX_DTYPE)\n            dtcol = self._datetime64_index(index)\n            if dtcol and len(index):\n                dts = index[dtcol]\n                start, end = _start_end(date_range, dts)\n                if start > dts[-1]:\n                    return -1, -1\n                idxstart = min(np.searchsorted(dts, start), len(dts) - 1)\n                idxend = min(np.searchsorted(dts, end, side='right'), len(dts) - 1)\n                return int(index['index'][idxstart]), int(index['index'][idxend] + 1)\n        return super(PandasStore, self)._index_range(version, symbol, **kwargs)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ngiving a recarr slice out the given artic. date. DateRange", "response": "def _daterange(self, recarr, date_range):\n        \"\"\" Given a recarr, slice out the given artic.date.DateRange if a\n        datetime64 index exists \"\"\"\n        idx = self._datetime64_index(recarr)\n        if idx and len(recarr):\n            dts = recarr[idx]\n            mask = Series(np.zeros(len(dts)), index=dts)\n            start, end = _start_end(date_range, dts)\n            mask[start:end] = 1.0\n            return recarr[mask.values.astype(bool)]\n        return recarr"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nparses out the relevant information in version and returns it to the user in a dictionary", "response": "def get_info(self, version):\n        \"\"\"\n        parses out the relevant information in version\n        and returns it to the user in a dictionary\n        \"\"\"\n        ret = super(PandasStore, self).get_info(version)\n        ret['col_names'] = version['dtype_metadata']\n        ret['handler'] = self.__class__.__name__\n        ret['dtype'] = ast.literal_eval(version['dtype'])\n        return ret"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _update_fw_pointers(collection, symbol, version, previous_version, is_append, shas_to_add=None):\n    version[FW_POINTERS_CONFIG_KEY] = ARCTIC_FORWARD_POINTERS_CFG.name  # get the str as enum is not BSON serializable\n\n    if ARCTIC_FORWARD_POINTERS_CFG is FwPointersCfg.DISABLED:\n        return\n\n    version_shas = set()\n\n    if is_append:\n        # Appends are tricky, as we extract the SHAs from the previous version (assuming it has FW pointers info)\n        prev_fw_cfg = get_fwptr_config(previous_version)\n        if prev_fw_cfg is FwPointersCfg.DISABLED.name:\n            version_shas.update(Binary(sha) for sha in collection.find(\n                {'symbol': symbol,\n                 'parent': version_base_or_id(previous_version),\n                 'segment': {'$lt': previous_version['up_to']}},\n                {'sha': 1}))\n        else:\n            version_shas.update(previous_version[FW_POINTERS_REFS_KEY])\n\n    # It is a write (we always get the all-inclusive set of SHAs), so no need to obtain previous SHAs\n    version_shas.update(shas_to_add)\n\n    # Verify here the number of seen segments vs expected ones\n    if len(version_shas) != version['segment_count']:\n        raise pymongo.errors.OperationFailure(\"Mismatched number of forward pointers to segments for {}: {} != {})\"\n                                              \"Is append: {}. Previous version: {}. \"\n                                              \"Gathered forward pointers segment shas: {}.\".format(\n            symbol, len(version_shas), version['segment_count'], is_append, previous_version['_id'], version_shas))\n\n    version[FW_POINTERS_REFS_KEY] = list(version_shas)", "response": "This function updates the version document with the forward pointers to segments."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _spec_fw_pointers_aware(symbol, version, from_index=None, to_index=None):\n    spec = {'symbol': symbol,\n            'segment': {'$lt': version['up_to'] if to_index is None else to_index}}\n    if from_index is not None:\n        spec['segment']['$gte'] = from_index\n\n    # Version was written with an older arctic, not FW-pointers aware, or written with FW pointers disabled\n    if FW_POINTERS_CONFIG_KEY not in version or version[FW_POINTERS_CONFIG_KEY] == FwPointersCfg.DISABLED.name:\n        spec['parent'] = version_base_or_id(version)\n        return spec\n\n    v_fw_config = FwPointersCfg[version[FW_POINTERS_CONFIG_KEY]]\n\n    # Version was created exclusively with fw pointers\n    if v_fw_config is FwPointersCfg.ENABLED:\n        if from_index is None and to_index is None:\n            del spec['segment']\n        spec['sha'] = {'$in': version[FW_POINTERS_REFS_KEY]}\n        return spec\n\n    # Version was created both with fw and legacy pointers, choose based on module configuration\n    if v_fw_config is FwPointersCfg.HYBRID:\n        if ARCTIC_FORWARD_POINTERS_CFG is FwPointersCfg.DISABLED:\n            spec['parent'] = version_base_or_id(version)\n        else:\n            if from_index is None and to_index is None:\n                del spec['segment']\n            spec['sha'] = {'$in': version[FW_POINTERS_REFS_KEY]}\n        return spec\n\n    # The code below shouldn't really be reached.\n    raise DataIntegrityException(\"Unhandled FW pointers configuration ({}: {}/{}/{})\".format(\n        version.get('symbol'), version.get('_id'), version.get('version'), v_fw_config))", "response": "This method updates the find query filter spec used to read a version."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning the index and the index of the array from which the symbol is from.", "response": "def _index_range(self, version, symbol, from_version=None, **kwargs):\n        \"\"\"\n        Tuple describing range to read from the ndarray - closed:open\n        \"\"\"\n        from_index = None\n        if from_version:\n            from_index = from_version['up_to']\n        return from_index, None"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _do_read(self, collection, version, symbol, index_range=None):\n        from_index = index_range[0] if index_range else None\n        to_index = version['up_to']\n        if index_range and index_range[1] and index_range[1] < version['up_to']:\n            to_index = index_range[1]\n        segment_count = version.get('segment_count') if from_index is None else None\n\n        spec = _spec_fw_pointers_aware(symbol, version, from_index, to_index)\n\n        data = bytearray()\n        i = -1\n        for i, x in enumerate(sorted(collection.find(spec), key=itemgetter('segment'))):\n            data.extend(decompress(x['data']) if x['compressed'] else x['data'])\n\n        # Check that the correct number of segments has been returned\n        if segment_count is not None and i + 1 != segment_count:\n            raise OperationFailure(\"Incorrect number of segments returned for {}:{}.  Expected: {}, but got {}. {}\".format(\n                                   symbol, version['version'], segment_count, i + 1, collection.database.name + '.' + collection.name))\n\n        dtype = self._dtype(version['dtype'], version.get('dtype_metadata', {}))\n        rtn = np.frombuffer(data, dtype=dtype).reshape(version.get('shape', (-1)))\n        return rtn", "response": "Internal method to read the data from the database."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nadd the library with the given date range to the underlying collection of libraries used by this store.", "response": "def add(self, date_range, library_name):\n        \"\"\"\n        Adds the library with the given date range to the underlying collection of libraries used by this store.\n        The underlying libraries should not overlap as the date ranges are assumed to be CLOSED_CLOSED by this function\n        and the rest of the class.\n\n        Arguments:\n\n        date_range: A date range provided on the assumption that it is CLOSED_CLOSED. If for example the underlying\n        libraries were split by year, the start of the date range would be datetime.datetime(year, 1, 1) and the end\n        would be datetime.datetime(year, 12, 31, 23, 59, 59, 999000). The date range must fall on UTC day boundaries,\n        that is the start must be add midnight and the end must be 1 millisecond before midnight.\n\n        library_name: The name of the underlying library. This must be the name of a valid Arctic library\n        \"\"\"\n        # check that the library is valid\n        try:\n            self._arctic_lib.arctic[library_name]\n        except Exception as e:\n            logger.error(\"Could not load library\")\n            raise e\n        assert date_range.start and date_range.end, \"Date range should have start and end properties {}\".format(date_range)\n        start = date_range.start.astimezone(mktz('UTC')) if date_range.start.tzinfo is not None else date_range.start.replace(tzinfo=mktz('UTC'))\n        end = date_range.end.astimezone(mktz('UTC')) if date_range.end.tzinfo is not None else date_range.end.replace(tzinfo=mktz('UTC'))\n        assert start.time() == time.min and end.time() == end_time_min, \"Date range should fall on UTC day boundaries {}\".format(date_range)\n        # check that the date range does not overlap\n        library_metadata = self._get_library_metadata(date_range)\n        if len(library_metadata) > 1 or (len(library_metadata) == 1 and library_metadata[0] != library_name):\n            raise OverlappingDataException(\"\"\"There are libraries that overlap with the date range:\nlibrary: {}\noverlapping libraries: {}\"\"\".format(library_name, [l.library for l in library_metadata]))\n        self._collection.update_one({'library_name': library_name},\n                                    {'$set': {'start': start, 'end': end}}, upsert=True)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef write(self, symbol, data):\n\n        # get the full set of date ranges that we have\n        cursor = self._collection.find()\n        for res in cursor:\n            library = self._arctic_lib.arctic[res['library_name']]\n            dslice = self._slice(data, to_dt(res['start'], mktz('UTC')), to_dt(res['end'], mktz('UTC')))\n            if len(dslice) != 0:\n                library.write(symbol, dslice)", "response": "Writes the data to the underlying low - level library."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nretrieve the library metadata for the given date range.", "response": "def _get_library_metadata(self, date_range):\n        \"\"\"\n        Retrieve the libraries for the given date range, the assumption is that the date ranges do not overlap and\n        they are CLOSED_CLOSED.\n\n        At the moment the date range is mandatory\n        \"\"\"\n        if date_range is None:\n            raise Exception(\"A date range must be provided\")\n        if not (date_range.start and date_range.end):\n            raise Exception(\"The date range {0} must contain a start and end date\".format(date_range))\n\n        start = date_range.start if date_range.start.tzinfo is not None else date_range.start.replace(tzinfo=mktz())\n        end = date_range.end if date_range.end.tzinfo is not None else date_range.end.replace(tzinfo=mktz())\n        query = {'$or': [{'start': {'$lte': start}, 'end': {'$gte': start}},\n                         {'start': {'$gte': start}, 'end': {'$lte': end}},\n                         {'start': {'$lte': end}, 'end': {'$gte': end}}]}\n\n        cursor = self._collection.find(query,\n                                       projection={'library_name': 1, 'start': 1, 'end': 1},\n                                       sort=[('start', pymongo.ASCENDING)])\n\n        results = []\n        for res in cursor:\n            start = res['start']\n            if date_range.start.tzinfo is not None and start.tzinfo is None:\n                start = start.replace(tzinfo=mktz(\"UTC\")).astimezone(tz=date_range.start.tzinfo)\n\n            end = res['end']\n            if date_range.end.tzinfo is not None and end.tzinfo is None:\n                end = end.replace(tzinfo=mktz(\"UTC\")).astimezone(tz=date_range.end.tzinfo)\n\n            results.append(TickStoreLibrary(res['library_name'], DateRange(start, end, CLOSED_CLOSED)))\n        return results"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nwriting data to the version store.", "response": "def write(self, symbol, data, prune_previous_version=True, metadata=None, **kwargs):\n        \"\"\"\n        Records a write request to be actioned on context exit. Takes exactly the same parameters as the regular\n        library write call.\n        \"\"\"\n        if data is not None:\n            # We only write data if existing data is None or the Timeseries data has changed or metadata has changed\n            if self.base_ts.data is None or not are_equals(data, self.base_ts.data) or metadata != self.base_ts.metadata:\n                self._do_write = True\n        self._write = partial(self._version_store.write, symbol, data, prune_previous_version=prune_previous_version,\n                              metadata=metadata, **kwargs)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nconvert a pandas DataFrame to records.", "response": "def _to_records(self, df, string_max_len=None, forced_dtype=None):\n        \"\"\"\n        Similar to DataFrame.to_records()\n        Differences:\n            Attempt type conversion for pandas columns stored as objects (e.g. strings),\n            as we can only store primitives in the ndarray.\n            Use dtype metadata to store column and index names.\n\n        string_max_len: integer - enforces a string size on the dtype, if any\n                                  strings exist in the record\n        \"\"\"\n\n        index_names, ix_vals, metadata = self._index_to_records(df)\n        columns, column_vals, multi_column = self._column_data(df)\n\n        if \"\" in columns:\n            raise ArcticException(\"Cannot use empty string as a column name.\")\n\n        if multi_column is not None:\n            metadata['multi_column'] = multi_column\n        metadata['columns'] = columns\n        names = index_names + columns\n\n        arrays = []\n        for arr, name in zip(ix_vals + column_vals, index_names + columns):\n            arrays.append(_to_primitive(arr, string_max_len,\n                                        forced_dtype=None if forced_dtype is None else forced_dtype[name]))\n\n        if forced_dtype is None:\n            dtype = np.dtype([(str(x), v.dtype) if len(v.shape) == 1 else (str(x), v.dtype, v.shape[1])\n                              for x, v in zip(names, arrays)],\n                             metadata=metadata)\n        else:\n            dtype = forced_dtype\n\n        # The argument names is ignored when dtype is passed\n        rtn = np.rec.fromarrays(arrays, dtype=dtype, names=names)\n        # For some reason the dtype metadata is lost in the line above\n        # and setting rtn.dtype to dtype does not preserve the metadata\n        # see https://github.com/numpy/numpy/issues/6771\n\n        return (rtn, dtype)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef fast_check_serializable(self, df):\n        i_dtype, f_dtypes = df.index.dtype, df.dtypes\n        index_has_object = df.index.dtype is NP_OBJECT_DTYPE\n        fields_with_object = [f for f in df.columns if f_dtypes[f] is NP_OBJECT_DTYPE]\n        if df.empty or (not index_has_object and not fields_with_object):\n            arr, _ = self._to_records(df.iloc[:10])  # only first few rows for performance\n            return arr, {}\n        # If only the Index has Objects, choose a small slice (two columns if possible,\n        # to avoid switching from a DataFrame to a Series)\n        df_objects_only = df[fields_with_object if fields_with_object else df.columns[:2]]\n        # Let any exceptions bubble up from here\n        arr, dtype = self._to_records(df_objects_only)\n        return arr, {f: dtype[f] for f in dtype.names}", "response": "Convert efficiently the frame s object - columns object - index multi - column columns or multi - column columns to records."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\napplies func to each chunk in lib. symbol", "response": "def read_apply(lib, symbol, func, chunk_range=None):\n    \"\"\"\n    Apply `func` to each chunk in lib.symbol\n\n    Parameters\n    ----------\n    lib: arctic library\n    symbol: str\n        the symbol for the given item in the DB\n    chunk_range: None, or a range object\n        allows you to subset the chunks by range\n\n    Returns\n    -------\n    generator\n    \"\"\"\n    for chunk in lib.iterator(symbol, chunk_range=chunk_range):\n        yield func(chunk)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef list_symbols(self, regex=None, as_of=None, **kwargs):\n\n        # Skip aggregation pipeline\n        if not (regex or as_of or kwargs):\n            return self.distinct('symbol')\n\n        # Index-based query part\n        index_query = {}\n        if as_of is not None:\n            index_query['start_time'] = {'$lte': as_of}\n\n        if regex or as_of:\n            # make sure that symbol is present in query even if only as_of is specified to avoid document scans\n            # see 'Pipeline Operators and Indexes' at\n            # https://docs.mongodb.com/manual/core/aggregation-pipeline/#aggregation-pipeline-operators-and-performance\n            index_query['symbol'] = {'$regex': regex or '^'}\n\n        # Document query part\n        data_query = {}\n        if kwargs:\n            for k, v in six.iteritems(kwargs):\n                data_query['metadata.' + k] = v\n\n        # Sort using index, relying on https://docs.mongodb.com/manual/core/aggregation-pipeline-optimization/\n        pipeline = [{'$sort': {'symbol': pymongo.ASCENDING,\n                               'start_time': pymongo.DESCENDING}}]\n\n        # Index-based filter on symbol and start_time\n        if index_query:\n            pipeline.append({'$match': index_query})\n        # Group by 'symbol' and get the latest known data\n        pipeline.append({'$group': {'_id': '$symbol',\n                                    'metadata': {'$first': '$metadata'}}})\n        # Match the data fields\n        if data_query:\n            pipeline.append({'$match': data_query})\n        # Return only 'symbol' field value\n        pipeline.append({'$project': {'_id': 0, 'symbol': '$_id'}})\n\n        return sorted(r['symbol'] for r in self.aggregate(pipeline))", "response": "Returns the list of symbols in this library."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef read_history(self, symbol):\n        find = self.find({'symbol': symbol}, sort=[('start_time', pymongo.ASCENDING)])\n        times = []\n        entries = []\n        for item in find:\n            times.append(item['start_time'])\n            entries.append(item['metadata'])\n        return pd.DataFrame({symbol: entries}, times)", "response": "Return all metadata saved for symbol"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef read(self, symbol, as_of=None):\n        if as_of is not None:\n            res = self.find_one({'symbol': symbol, 'start_time': {'$lte': as_of}},\n                                sort=[('start_time', pymongo.DESCENDING)])\n        else:\n            res = self.find_one({'symbol': symbol}, sort=[('start_time', pymongo.DESCENDING)])\n        return res['metadata'] if res is not None else None", "response": "Reads the current metadata for a given item."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef write_history(self, collection):\n        documents = []\n        for dataframe in collection:\n            if len(dataframe.columns) != 1:\n                raise ValueError('More than one symbol found in a DataFrame')\n            symbol = dataframe.columns[0]\n            times = dataframe.index\n            entries = dataframe[symbol].values\n            if self.has_symbol(symbol):\n                self.purge(symbol)\n            doc = {'symbol': symbol, 'metadata': entries[0], 'start_time': times[0]}\n            for metadata, start_time in zip(entries[1:], times[1:]):\n                if metadata == doc['metadata']:\n                    continue\n                doc['end_time'] = start_time\n                documents.append(doc)\n                doc = {'symbol': symbol, 'metadata': metadata, 'start_time': start_time}\n            documents.append(doc)\n\n        self.insert_many(documents)", "response": "Write all metadata history for symbols in collection."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nappend a new entry to the metadata table for the item.", "response": "def append(self, symbol, metadata, start_time=None):\n        \"\"\"\n        Update metadata entry for `symbol`\n\n        Parameters\n        ----------\n        symbol : `str`\n            symbol name for the item\n        metadata : `dict`\n            to be persisted\n        start_time : `datetime.datetime`\n            when metadata becomes effective\n            Default: datetime.datetime.utcnow()\n        \"\"\"\n        if start_time is None:\n            start_time = dt.utcnow()\n        old_metadata = self.find_one({'symbol': symbol}, sort=[('start_time', pymongo.DESCENDING)])\n        if old_metadata is not None:\n            if old_metadata['start_time'] >= start_time:\n                raise ValueError('start_time={} is earlier than the last metadata @{}'.format(start_time,\n                                                                                              old_metadata['start_time']))\n            if old_metadata['metadata'] == metadata:\n                return old_metadata\n        elif metadata is None:\n            return\n\n        self.find_one_and_update({'symbol': symbol}, {'$set': {'end_time': start_time}},\n                                 sort=[('start_time', pymongo.DESCENDING)])\n        document = {'_id': bson.ObjectId(), 'symbol': symbol, 'metadata': metadata, 'start_time': start_time}\n        mongo_retry(self.insert_one)(document)\n\n        logger.debug('Finished writing metadata for %s', symbol)\n        return document"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ndelete the last metadata of a specific symbol", "response": "def pop(self, symbol):\n        \"\"\"\n        Delete current metadata of `symbol`\n\n        Parameters\n        ----------\n        symbol : `str`\n            symbol name to delete\n\n        Returns\n        -------\n        Deleted metadata\n        \"\"\"\n        last_metadata = self.find_one({'symbol': symbol}, sort=[('start_time', pymongo.DESCENDING)])\n        if last_metadata is None:\n            raise NoDataFoundException('No metadata found for symbol {}'.format(symbol))\n\n        self.find_one_and_delete({'symbol': symbol}, sort=[('start_time', pymongo.DESCENDING)])\n        mongo_retry(self.find_one_and_update)({'symbol': symbol}, {'$unset': {'end_time': ''}},\n                                              sort=[('start_time', pymongo.DESCENDING)])\n\n        return last_metadata"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ndelete all metadata of symbol from the store.", "response": "def purge(self, symbol):\n        \"\"\"\n        Delete all metadata of `symbol`\n\n        Parameters\n        ----------\n        symbol : `str`\n            symbol name to delete\n        \"\"\"\n        logger.warning(\"Deleting entire metadata history for %r from %r\" % (symbol, self._arctic_lib.get_name()))\n        self.delete_many({'symbol': symbol})"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef stats(self):\n        res = {}\n        db = self._collection.database\n        res['dbstats'] = db.command('dbstats')\n        res['data'] = db.command('collstats', self._collection.name)\n        res['totals'] = {'count': res['data']['count'],\n                         'size': res['data']['size']\n                         }\n        return res", "response": "Return database usage statistics. Used by quota."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nstores the given thing in the database.", "response": "def store(self, thing):\n        \"\"\"\n        Simple persistence method\n        \"\"\"\n        to_store = {'field1': thing.field1,\n                    'date_field': thing.date_field,\n                    }\n        to_store['stuff'] = Binary(cPickle.dumps(thing.stuff))\n        # Respect any soft-quota on write - raises if stats().totals.size > quota \n        self._arctic_lib.check_quota()\n        self._collection.insert_one(to_store)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreading the data for the named symbol and returns a BitemporalItem object with the data and metadata elements.", "response": "def read(self, symbol, as_of=None, raw=False, **kwargs):\n        # TODO: shall we block from_version from getting into super.read?\n        \"\"\"Read data for the named symbol. Returns a BitemporalItem object with\n        a data and metdata element (as passed into write).\n\n        Parameters\n        ----------\n        symbol : `str`\n            symbol name for the item\n        as_of : `datetime.datetime`\n            Return the data as it was as_of the point in time.\n        raw : `bool`\n            If True, will return the full bitemporal dataframe (i.e. all versions of the data). This also means as_of is\n            ignored.\n\n        Returns\n        -------\n        BitemporalItem namedtuple which contains a .data and .metadata element\n        \"\"\"\n        item = self._store.read(symbol, **kwargs)\n        last_updated = max(item.data.index.get_level_values(self.observe_column))\n        if raw:\n            return BitemporalItem(symbol=symbol, library=self._store._arctic_lib.get_name(), data=item.data,\n                                  metadata=item.metadata,\n                                  last_updated=last_updated)\n        else:\n            index_names = list(item.data.index.names)\n            index_names.remove(self.observe_column)\n            return BitemporalItem(symbol=symbol, library=self._store._arctic_lib.get_name(),\n                                  data=groupby_asof(item.data, as_of=as_of, dt_col=index_names,\n                                                    asof_col=self.observe_column),\n                                  metadata=item.metadata, last_updated=last_updated)"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nupdates the data for the specified item in this library.", "response": "def update(self, symbol, data, metadata=None, upsert=True, as_of=None, **kwargs):\n        \"\"\" Append 'data' under the specified 'symbol' name to this library.\n\n        Parameters\n        ----------\n        symbol : `str`\n            symbol name for the item\n        data : `pd.DataFrame`\n            to be persisted\n        metadata : `dict`\n            An optional dictionary of metadata to persist along with the symbol. If None and there are existing\n            metadata, current metadata will be maintained\n        upsert : `bool`\n            Write 'data' if no previous version exists.\n        as_of : `datetime.datetime`\n            The \"insert time\". Default to datetime.now()\n        \"\"\"\n        local_tz = mktz()\n        if not as_of:\n            as_of = dt.now()\n        if as_of.tzinfo is None:\n            as_of = as_of.replace(tzinfo=local_tz)\n        data = self._add_observe_dt_index(data, as_of)\n        if upsert and not self._store.has_symbol(symbol):\n            df = data\n        else:\n            existing_item = self._store.read(symbol, **kwargs)\n            if metadata is None:\n                metadata = existing_item.metadata\n            df = existing_item.data.append(data).sort_index(kind='mergesort')\n        self._store.write(symbol, df, metadata=metadata, prune_previous_version=True)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef string_to_daterange(str_range, delimiter='-', as_dates=False, interval=CLOSED_CLOSED):\n    num_dates = str_range.count(delimiter) + 1\n    if num_dates > 2:\n        raise ValueError('Too many dates in input string [%s] with delimiter (%s)' % (str_range, delimiter))\n\n    # Allow the user to use the [date-date), etc. range syntax to specify the interval.\n    range_mode = Ranges.get(str_range[0] + str_range[-1], None)\n    if range_mode:\n        return string_to_daterange(str_range[1:-1], delimiter, as_dates, interval=range_mode)\n\n    if as_dates:\n        parse_dt = lambda s: parse(s).date() if s else None\n    else:\n        parse_dt = lambda s: parse(s) if s else None\n    if num_dates == 2:\n        d = [parse_dt(x) for x in str_range.split(delimiter)]\n        oc = interval\n    else:\n        start = parse_dt(str_range)\n        d = [start, start + datetime.timedelta(1)]\n        oc = CLOSED_OPEN  # Always use closed-open for a single date/datetime.\n    return DateRange(d[0], d[1], oc)", "response": "Convert a string to a DateRange type."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef to_dt(date, default_tz=None):\n    if isinstance(date, (int, long)):\n        return ms_to_datetime(date, default_tz)\n    elif date.tzinfo is None:\n        if default_tz is None:\n            raise ValueError(\"Must specify a TimeZone on incoming data\")\n        return date.replace(tzinfo=default_tz)\n    return date", "response": "Converts a datetime object to a non - naive datetime object."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nconvert a date_range object to a Pandas DateRange slicing is CLOSED - CLOSED inclusive at both ends.", "response": "def to_pandas_closed_closed(date_range, add_tz=True):\n    \"\"\"\n    Pandas DateRange slicing is CLOSED-CLOSED inclusive at both ends.\n\n    Parameters\n    ----------\n    date_range : `DateRange` object\n        converted to CLOSED_CLOSED form for Pandas slicing\n\n    add_tz : `bool`\n        Adds a TimeZone to the daterange start and end if it doesn't\n        have one.\n\n    Returns\n    -------\n    Returns a date_range with start-end suitable for slicing in pandas.\n    \"\"\"\n    if not date_range:\n        return None\n\n    start = date_range.start\n    end = date_range.end\n    if start:\n        start = to_dt(start, mktz()) if add_tz else start\n        if date_range.startopen:\n            start += timedelta(milliseconds=1)\n\n    if end:\n        end = to_dt(end, mktz()) if add_tz else end\n        if date_range.endopen:\n            end -= timedelta(milliseconds=1)\n    return DateRange(start, end)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef ms_to_datetime(ms, tzinfo=None):\n    if not isinstance(ms, (int, long)):\n        raise TypeError('expected integer, not %s' % type(ms))\n\n    if tzinfo is None:\n        tzinfo = mktz()\n\n    return datetime.datetime.fromtimestamp(ms * 1e-3, tzinfo)", "response": "Convert a millisecond time value to an offset - aware Python datetime object."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nconvert a Python datetime object to a millisecond epoch time value.", "response": "def datetime_to_ms(d):\n    \"\"\"Convert a Python datetime object to a millisecond epoch (UTC) time value.\"\"\"\n    try:\n        millisecond = d.microsecond // 1000\n        return calendar.timegm(_add_tzone(d).utctimetuple()) * 1000 + millisecond\n    except AttributeError:\n        raise TypeError('expect Python datetime object, not %s' % type(d))"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef utc_dt_to_local_dt(dtm):\n    utc_zone = mktz(\"UTC\")\n    if dtm.tzinfo is not None and dtm.tzinfo != utc_zone:\n        raise ValueError(\n            \"Expected dtm without tzinfo or with UTC, not %r\" % (\n                dtm.tzinfo\n            )\n        )\n\n    if dtm.tzinfo is None:\n        dtm = dtm.replace(tzinfo=utc_zone)\n    return dtm.astimezone(mktz())", "response": "Convert a UTC datetime to a local datetime in local timezone"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef intersection(self, other):\n        startopen = other.startopen if self.start is None \\\n            else self.startopen if other.start is None \\\n            else other.startopen if self.start < other.start \\\n            else self.startopen if self.start > other.start \\\n            else (self.startopen or other.startopen)\n        endopen = other.endopen if self.end is None \\\n            else self.endopen if other.end is None \\\n            else other.endopen if self.end > other.end \\\n            else self.endopen if self.end < other.end \\\n            else (self.endopen or other.endopen)\n\n        new_start = self.start if other.start is None \\\n            else other.start if self.start is None \\\n            else max(self.start, other.start)\n        new_end = self.end if other.end is None \\\n            else other.end if self.end is None \\\n            else min(self.end, other.end)\n\n        interval = INTERVAL_LOOKUP[(startopen, endopen)]\n\n        return DateRange(new_start, new_end, interval)", "response": "Create a new DateRange representing the maximal range enclosed by this range and other."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef as_dates(self):\n        new_start = self.start.date() if self.start and isinstance(self.start, datetime.datetime) else self.start\n        new_end = self.end.date() if self.end and isinstance(self.end, datetime.datetime) else self.end\n        return DateRange(new_start, new_end, CLOSED_CLOSED)", "response": "Create a new DateRange with the datetimes converted to dates."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nconvert a DateRange into a MongoDb query string.", "response": "def mongo_query(self):\n        \"\"\"\n        Convert a DateRange into a MongoDb query string. FIXME: Mongo can only handle\n        datetimes in queries, so we should make this handle the case where start/end are\n        datetime.date and extend accordingly (being careful about the interval logic).\n        \"\"\"\n        comps = {OPEN_CLOSED: ('t', 'te'), OPEN_OPEN: ('t', 't'),\n                 CLOSED_OPEN: ('te', 't'), CLOSED_CLOSED: ('te', 'te')}\n        query = {}\n        comp = comps[self.interval]\n        if self.start:\n            query['$g' + comp[0]] = self.start\n        if self.end:\n            query['$l' + comp[1]] = self.end\n        return query"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn the upper and lower bounds along with operators that are needed to do an in range test.", "response": "def get_date_bounds(self):\n        \"\"\"\n        Return the upper and lower bounds along\n        with operators that are needed to do an 'in range' test.\n        Useful for SQL commands.\n\n        Returns\n        -------\n        tuple: (`str`, `date`, `str`, `date`)\n                (date_gt, start, date_lt, end)\n        e.g.:\n                ('>=', start_date, '<', end_date)\n        \"\"\"\n        start = end = None\n        date_gt = '>='\n        date_lt = '<='\n        if self:\n            if self.start:\n                start = self.start\n            if self.end:\n                end = self.end\n            if self.startopen:\n                date_gt = '>'\n            if self.endopen:\n                date_lt = '<'\n\n        return date_gt, start, date_lt, end"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nchunk the dataframe/series by dates Parameters ---------- df: pandas dataframe or series chunk_size: str any valid Pandas frequency string func: function func will be applied to each `chunk` generated by the chunker. This function CANNOT modify the date column of the dataframe! Returns ------- generator that produces tuples: (start date, end date, chunk_size, dataframe/series)", "response": "def to_chunks(self, df, chunk_size='D', func=None, **kwargs):\n        \"\"\"\n        chunks the dataframe/series by dates\n\n        Parameters\n        ----------\n        df: pandas dataframe or series\n        chunk_size: str\n            any valid Pandas frequency string\n        func: function\n            func will be applied to each `chunk` generated by the chunker.\n            This function CANNOT modify the date column of the dataframe!\n\n        Returns\n        -------\n        generator that produces tuples: (start date, end date,\n                  chunk_size, dataframe/series)\n        \"\"\"\n        if 'date' in df.index.names:\n            dates = df.index.get_level_values('date')\n            if not df.index.is_monotonic_increasing:\n                df = df.sort_index()\n        elif 'date' in df.columns:\n            dates = pd.DatetimeIndex(df.date)\n            if not dates.is_monotonic_increasing:\n                # providing support for pandas 0.16.2 to 0.20.x\n                # neither sort method exists in both\n                try:\n                    df = df.sort_values('date')\n                except AttributeError:\n                    df = df.sort(columns='date')\n                dates = pd.DatetimeIndex(df.date)\n        else:\n            raise Exception(\"Data must be datetime indexed or have a column named 'date'\")\n\n        period_obj = dates.to_period(chunk_size)\n        period_obj_reduced = period_obj.drop_duplicates()\n        count = 0\n        for _, g in df.groupby(period_obj._data):\n            start = period_obj_reduced[count].start_time.to_pydatetime(warn=False)\n            end = period_obj_reduced[count].end_time.to_pydatetime(warn=False)\n            count += 1\n            if func:\n                yield start, end, chunk_size, func(g)\n            else:\n                yield start, end, chunk_size, g"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef to_mongo(self, range_obj):\n        if isinstance(range_obj, (pd.DatetimeIndex, tuple)):\n            range_obj = DateRange(range_obj[0], range_obj[-1])\n        if range_obj.start and range_obj.end:\n            return {'$and': [{START: {'$lte': range_obj.end}}, {END: {'$gte': range_obj.start}}]}\n        elif range_obj.start:\n            return {END: {'$gte': range_obj.start}}\n        elif range_obj.end:\n            return {START: {'$lte': range_obj.end}}\n        else:\n            return {}", "response": "Converts a date range object to a MongoDB query that filters by the range\n       "}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nfilters the data by the specified date range.", "response": "def filter(self, data, range_obj):\n        \"\"\"\n        ensures data is properly subset to the range in range_obj.\n        (Depending on how the chunking is implemented, it might be possible\n        to specify a chunk range that reads out more than the actual range\n        eg: date range, chunked monthly. read out 2016-01-01 to 2016-01-02.\n        This will read ALL of January 2016 but it should be subset to just\n        the first two days)\n\n        returns\n        -------\n        data, filtered by range_obj\n        \"\"\"\n        if isinstance(range_obj, (pd.DatetimeIndex, tuple)):\n            range_obj = DateRange(range_obj[0], range_obj[-1])\n\n        range_obj = to_pandas_closed_closed(range_obj, add_tz=False)\n        start = range_obj.start\n        end = range_obj.end\n\n        if 'date' in data.index.names:\n            return data[start:end]\n        elif 'date' in data.columns:\n            if start and end:\n                return data[(data.date >= start) & (data.date <= end)]\n            elif start:\n                return data[(data.date >= start)]\n            elif end:\n                return data[(data.date <= end)]\n            else:\n                return data\n        else:\n            return data"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef exclude(self, data, range_obj):\n        if isinstance(range_obj, (pd.DatetimeIndex, tuple)):\n            range_obj = DateRange(range_obj[0], range_obj[-1])\n        if 'date' in data.index.names:\n            return data[(data.index.get_level_values('date') < range_obj.start) | (data.index.get_level_values('date') > range_obj.end)]\n        elif 'date' in data.columns:\n            return data[(data.date < range_obj.start) | (data.date > range_obj.end)]\n        else:\n            return data", "response": "Removes data within the bounds of the range object"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef enable_sharding(arctic, library_name, hashed=True, key='symbol'):\n    c = arctic._conn\n    lib = arctic[library_name]._arctic_lib\n    dbname = lib._db.name\n    library_name = lib.get_top_level_collection().name\n    try:\n        c.admin.command('enablesharding', dbname)\n    except pymongo.errors.OperationFailure as e:\n        if 'already enabled' not in str(e):\n            raise\n    if not hashed:\n        logger.info(\"Range sharding '\" + key + \"' on: \" + dbname + '.' + library_name)\n        c.admin.command('shardCollection', dbname + '.' + library_name, key={key: 1})\n    else:\n        logger.info(\"Hash sharding '\" + key + \"' on: \" + dbname + '.' + library_name)\n        c.admin.command('shardCollection', dbname + '.' + library_name, key={key: 'hashed'})", "response": "Enable sharding on a library."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef insert_one(self, document, **kwargs):\n        self._arctic_lib.check_quota()\n        return self._collection.insert_one(document, **kwargs)", "response": "Insert one document into the collection."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ninsert a list of documents into the collection.", "response": "def insert_many(self, documents, **kwargs):\n        \"\"\"\n        See http://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.insert_many\n        \"\"\"\n        self._arctic_lib.check_quota()\n        return self._collection.insert_many(documents, **kwargs)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef update_one(self, filter, update, **kwargs):\n        self._arctic_lib.check_quota()\n        return self._collection.update_one(filter, update, **kwargs)", "response": "Update one entry in the collection"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef update_many(self, filter, update, **kwargs):\n        self._arctic_lib.check_quota()\n        return self._collection.update_many(filter, update, **kwargs)", "response": "Update many items in the collection"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreplaces one entry in the collection with another.", "response": "def replace_one(self, filter, replacement, **kwargs):\n        \"\"\"\n        See http://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.replace_one\n        \"\"\"\n        self._arctic_lib.check_quota()\n        return self._collection.replace_one(filter, replacement, **kwargs)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nfind one and replace the object with replacement", "response": "def find_one_and_replace(self, filter, replacement, **kwargs):\n        \"\"\"\n        See http://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.find_one_and_replace\n        \"\"\"\n        self._arctic_lib.check_quota()\n        return self._collection.find_one_and_replace(filter, replacement, **kwargs)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef find_one_and_update(self, filter, update, **kwargs):\n        self._arctic_lib.check_quota()\n        return self._collection.find_one_and_update(filter, update, **kwargs)", "response": "Find one and update the object with the filter and kwargs"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nbulk write to the collection.", "response": "def bulk_write(self, requests, **kwargs):\n        \"\"\"\n        See http://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.bulk_write\n\n        Warning: this is wrapped in mongo_retry, and is therefore potentially unsafe if the write you want to execute\n        isn't idempotent.\n        \"\"\"\n        self._arctic_lib.check_quota()\n        return self._collection.bulk_write(requests, **kwargs)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef count(self, filter, **kwargs):\n        return mongo_count(self._collection, filter=filter, **kwargs)", "response": "Count the number of items in the collection."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef mktz(zone=None):\n    if zone is None:\n        zone = tzlocal.get_localzone().zone\n    zone = six.u(zone)\n    tz = dateutil.tz.gettz(zone)\n    if not tz:\n        raise TimezoneError('Timezone \"%s\" can not be read' % (zone))\n    # Stash the zone name as an attribute (as pytz does)\n    if not hasattr(tz, 'zone'):\n        tz.zone = zone\n        for p in dateutil.tz.TZPATHS:\n            if zone.startswith(p):\n                tz.zone = zone[len(p) + 1:]\n                break\n    return tz", "response": "Create a new timezone object based on the zone name."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nensure that symbol(s) have contiguous segment ids Parameters ---------- library: arctic library symbol: None, str, list of str None: all symbols str: single symbol list: list of symbols Returns ------- list of str - Symbols 'fixed'", "response": "def segment_id_repair(library, symbol=None):\n    \"\"\"\n    Ensure that symbol(s) have contiguous segment ids\n\n    Parameters\n    ----------\n    library: arctic library\n    symbol: None, str, list of str\n        None: all symbols\n        str: single symbol\n        list: list of symbols\n\n    Returns\n    -------\n    list of str - Symbols 'fixed'\n    \"\"\"\n    ret = []\n\n    if symbol is None:\n        symbol = library.list_symbols()\n    elif not isinstance(symbol, list):\n        symbol = [symbol]\n\n    by_segment = [(START, pymongo.ASCENDING),\n                  (SEGMENT, pymongo.ASCENDING)]\n\n    for sym in symbol:\n        cursor = library._collection.find({SYMBOL: sym}, sort=by_segment)\n        # group by chunk\n        for _, segments in groupby(cursor, key=lambda x: (x[START], x[SYMBOL])):\n            segments = list(segments)\n            # if the start segment is not 0, we need to fix this symbol\n            if segments[0][SEGMENT] == -1:\n                # since the segment is part of the index, we have to clean up first\n                library._collection.delete_many({SYMBOL: sym, START: segments[0][START]})\n                # map each segment in the interval to the correct segment\n                for index, seg in enumerate(segments):\n                    seg[SEGMENT] = index\n                library._collection.insert_many(segments)\n                ret.append(sym)\n\n    return ret"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef multicolumn_store_with_uncompressed_write(mongo_server):\n    mongo_server.api.drop_database('arctic_test')\n\n    library_name = 'arctic_test.TEST'\n    arctic = m.Arctic(mongo_host=mongo_server.api)\n    arctic.initialize_library(library_name, m.VERSION_STORE, segment='month')\n\n    db = mongo_server.api.arctic_test\n    db.TEST.insert_many([\n        {\n            'parent': [bson.ObjectId('5ad0dc065c911d1188b512d8')],\n            'data': bson.Binary(b'\\x11\\x00\\x00\\x002x\\x01\\x00\\x01\\x00\\x80\\x02\\x00\\x00\\x00\\x00\\x00\\x00\\x00', 0),\n            'symbol': 'pandas',\n            'sha': bson.Binary(b'\\xaa\\\\`\\x0e\\xc2D-\\xc1_\\xf7\\xfd\\x12\\xfa\\xd2\\x17\\x05`\\x00\\x98\\xe2', 0),\n            'compressed': True,\n            '_id': bson.ObjectId('5ad0dc067934ecad404070be'),\n            'segment': 0\n        },\n        {\n            'parent': [bson.ObjectId('5ad0dc065c911d1188b512d8')],\n            'data': bson.Binary(b'y\\x03\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x04\\x00\\x00\\x00\\x00\\x00\\x00\\x00', 0),\n            'symbol': 'pandas',\n            'sha': bson.Binary(b'\\xfe=WQ\\xb5\\xfdL\\xb7\\xcavd\\x85o\\x04]\\x04\\xdb\\xa8]3', 0),\n            'compressed': False,\n            '_id': bson.ObjectId('5ad0dc077934ecad404070bf'),\n            'segment': 1\n        }\n    ])\n    db.TEST.ARCTIC.update_one({\"_id\": \"ARCTIC_META\"}, {\"$set\": {\"_id\": \"ARCTIC_META\", \"TYPE\": \"VersionStore\", \"QUOTA\": 10737418240}})\n    db.TEST.version_nums.insert_one({'symbol': 'pandas', '_id': bson.ObjectId('5ad0dc067934ecad404070bd'), 'version': 2})\n    db.TEST.versions.insert_many([\n        {\n            'append_count': 0,\n            'dtype_metadata': {\n                'index': ['index'],\n                'columns': [\"('a', 'a')\", \"('w', 'v')\"]\n            },\n            'segment_count': 1,\n            'dtype': '[(\\'index\\', \\'S1\\'), (\"(\\'a\\', \\'a\\')\", \\'<i8\\'), (\"(\\'w\\', \\'v\\')\", \\'<i8\\')]',\n            'symbol': 'pandas',\n            'up_to': 1,\n            'metadata': None,\n            'sha': bson.Binary(b'\\xf2\\x15h\\x9d\\x925\\x95\\xa5\\x0e\\x95J\\xc4x\\xfc\\xfc\\xd5\\x80\\xe0\\x1d\\xef', 0),\n            'shape': [-1],\n            'version': 1,\n            'base_sha': bson.Binary(b'\\xf2\\x15h\\x9d\\x925\\x95\\xa5\\x0e\\x95J\\xc4x\\xfc\\xfc\\xd5\\x80\\xe0\\x1d\\xef', 0),\n            '_id': bson.ObjectId('5ad0dc065c911d1188b512d8'),\n            'type': 'pandasdf',\n            'append_size': 0\n        },\n        {\n            'append_count': 1,\n            'dtype_metadata': {\n                'index': ['index'],\n                'columns': [\"('a', 'a')\", \"('w', 'v')\"]\n            },\n            'segment_count': 2,\n            'sha': bson.Binary(b'1\\x83[ZO\\xec\\x080D\\x80f\\xe4@\\xe4\\xd3\\x94yG\\xe2\\x08', 0),\n            'dtype': '[(\\'index\\', \\'S1\\'), (\"(\\'a\\', \\'a\\')\", \\'<i8\\'), (\"(\\'w\\', \\'v\\')\", \\'<i8\\')]',\n            'symbol': 'pandas',\n            'up_to': 2,\n            'metadata': None,\n            'base_version_id': bson.ObjectId('5ad0dc065c911d1188b512d8'),\n            'shape': [-1],\n            'version': 2,\n            'base_sha': bson.Binary(b'\\xf2\\x15h\\x9d\\x925\\x95\\xa5\\x0e\\x95J\\xc4x\\xfc\\xfc\\xd5\\x80\\xe0\\x1d\\xef', 0),\n            '_id': bson.ObjectId('5ad0dc075c911d1188b512d9'),\n            'type': 'pandasdf',\n            'append_size': 17\n        }\n    ])\n\n    return {'symbol': 'pandas', 'store': arctic.get_library('arctic_test.TEST')}", "response": "Stores the multicolumn of the current language in a single database."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef ndarray_store_with_uncompressed_write(mongo_server):\n    mongo_server.api.drop_database('arctic_test')\n\n    library_name = 'arctic_test.TEST'\n    arctic = m.Arctic(mongo_host=mongo_server.api)\n    arctic.initialize_library(library_name, m.VERSION_STORE, segment='month')\n\n    db = mongo_server.api.arctic_test\n    db.TEST.insert_many([\n        {\n            \"_id\": bson.ObjectId(\"5ad0742ca0949de6727cf994\"),\n            \"segment\": 0,\n            \"sha\": bson.Binary(base64.b64decode(\"Fk+quqPVSDfaajYJkOAvnDyXtGQ=\"), 0),\n            \"symbol\": \"MYARR\",\n            \"data\": bson.Binary(base64.b64decode(\"CAAAAIAAAAAAAAAAAA==\"), 0),\n            \"compressed\": True,\n            \"parent\": [bson.ObjectId(\"5ad0742c5c911d4d80ee2ea3\")]\n        },\n        {\n            \"_id\": bson.ObjectId(\"5ad0742ca0949de6727cf995\"),\n            \"sha\": bson.Binary(base64.b64decode(\"eqpp8VOJBttTz0j5H+QGtOQ+r44=\"), 0),\n            \"symbol\": \"MYARR\",\n            \"segment\": 1,\n            \"data\": bson.Binary(base64.b64decode(\"AQAAAAAAAAA=\"), 0),\n            \"compressed\": False,\n            \"parent\": [bson.ObjectId(\"5ad0742c5c911d4d80ee2ea3\")]\n        }\n    ])\n    db.TEST.ARCTIC.update_one({\"_id\": \"ARCTIC_META\"}, {\"$set\": {\"_id\": \"ARCTIC_META\", \"TYPE\": \"VersionStore\", \"QUOTA\": 10737418240}})\n    db.TEST.versions_nums.insert_one({\"_id\": bson.ObjectId(\"5ad0742ca0949de6727cf993\"), \"symbol\": \"MYARR\", \"version\": 2})\n    db.TEST.versions.insert_many([\n        {\n            \"_id\": bson.ObjectId(\"5ad0742c5c911d4d80ee2ea3\"),\n            \"append_count\": 0,\n            \"dtype_metadata\": {},\n            \"segment_count\": 1,\n            \"dtype\": \"[('abc', '<i8')]\",\n            \"symbol\": \"MYARR\",\n            \"up_to\": 1,\n            \"append_size\": 0,\n            \"sha\": bson.Binary(base64.b64decode(\"Bf5AV1MWbxJVWefJrFWGVPEHx+k=\"), 0),\n            \"shape\": [-1],\n            \"version\": 1,\n            \"base_sha\": bson.Binary(base64.b64decode(\"Bf5AV1MWbxJVWefJrFWGVPEHx+k=\"), 0),\n            \"type\": \"ndarray\",\n            \"metadata\": None\n        },\n        {\n            \"_id\": bson.ObjectId(\"5ad0742c5c911d4d80ee2ea4\"),\n            \"append_count\": 1,\n            \"dtype_metadata\": {},\n            \"segment_count\": 2,\n            \"base_version_id\": bson.ObjectId(\"5ad0742c5c911d4d80ee2ea3\"),\n            \"dtype\": \"[('abc', '<i8')]\",\n            \"symbol\": \"MYARR\",\n            \"up_to\": 2,\n            \"append_size\": 8,\n            \"sha\": bson.Binary(base64.b64decode(\"Ax7oBxVFw1/9wKog2gfOLjbOVD8=\"), 0),\n            \"shape\": [-1],\n            \"version\": 2,\n            \"base_sha\": bson.Binary(base64.b64decode(\"Bf5AV1MWbxJVWefJrFWGVPEHx+k=\"), 0),\n            \"type\": \"ndarray\",\n            \"metadata\": None\n        }\n    ])\n\n    return {'symbol': 'MYARR', 'store': arctic.get_library('arctic_test.TEST')}", "response": "Stores the array of objects in the database with uncompressed writes."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef list_symbols(self, all_symbols=False, snapshot=None, regex=None, **kwargs):\n        query = {}\n        if regex is not None:\n            query['symbol'] = {'$regex': regex}\n        if kwargs:\n            for k, v in six.iteritems(kwargs):\n                # TODO: this doesn't work as expected as it ignores the versions with metadata.deleted set\n                #       as a result it will return symbols with matching metadata which have been deleted\n                #       Maybe better add a match step in the pipeline instead of making it part of the query\n                query['metadata.' + k] = v\n        if snapshot is not None:\n            try:\n                query['parent'] = self._snapshots.find_one({'name': snapshot})['_id']\n            except TypeError:\n                raise NoDataFoundException('No snapshot %s in library %s' % (snapshot, self._arctic_lib.get_name()))\n        elif all_symbols:\n            return self._versions.find(query).distinct('symbol')\n\n        # Return just the symbols which aren't deleted in the 'trunk' of this library\n        pipeline = []\n        if query:\n            # Match based on user criteria first\n            pipeline.append({'$match': query})\n        pipeline.extend([\n            # version_custom value is: 2*version + (0 if deleted else 1)\n            # This is used to optimize aggregation query:\n            #  - avoid sorting\n            #  - be able to rely on the latest version (max) for the deleted status\n            #\n            # Be aware of that if you don't use custom sort or if use a sort before $group which utilizes\n            # exactly an existing index, the $group will do best effort to utilize this index:\n            #  - https://jira.mongodb.org/browse/SERVER-4507\n            {'$group': {\n                '_id': '$symbol',\n                'version_custom': {\n                    '$max': {\n                        '$add': [\n                            {'$multiply': ['$version', 2]},\n                            {'$cond': [{'$eq': ['$metadata.deleted', True]}, 1, 0]}\n                        ]\n                    }\n                },\n            }},\n            # Don't include symbols which are part of some snapshot, but really deleted...\n            {'$match': {'version_custom': {'$mod': [2, 0]}}}\n        ])\n\n        # We may hit the group memory limit (100MB), so use allowDiskUse to circumvent this\n        #  - https://docs.mongodb.com/manual/reference/operator/aggregation/group/#group-memory-limit\n        return sorted([x['_id'] for x in self._versions.aggregate(pipeline, allowDiskUse=True)])", "response": "Returns a list of all available symbols in this library."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef has_symbol(self, symbol, as_of=None):\n        try:\n            # Always use the primary for has_symbol, it's safer\n            self._read_metadata(symbol, as_of=as_of, read_preference=ReadPreference.PRIMARY)\n            return True\n        except NoDataFoundException:\n            return False", "response": "Returns True if the symbol exists in this library AND the version number of the version number is greater than or equal to the requested point in time."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef read_audit_log(self, symbol=None, message=None):\n        query = {}\n        if symbol:\n            if isinstance(symbol, six.string_types):\n                query['symbol'] = {'$regex': symbol}\n            else:\n                query['symbol'] = {'$in': list(symbol)}\n\n        if message is not None:\n            query['message'] = message\n\n        def _pop_id(x):\n            x.pop('_id')\n            return x\n\n        return [_pop_id(x) for x in self._audit.find(query, sort=[('_id', -1)])]", "response": "Reads the audit log associated with a given symbol."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning a list of versions for the specified symbol and snapshot.", "response": "def list_versions(self, symbol=None, snapshot=None, latest_only=False):\n        \"\"\"\n        Return a list of versions filtered by the passed in parameters.\n\n        Parameters\n        ----------\n        symbol : `str`\n            Symbol to return versions for.  If None returns versions across all\n            symbols in the library.\n        snapshot : `str`\n            Return the versions contained in the named snapshot\n        latest_only : `bool`\n            Only include the latest version for a specific symbol\n\n        Returns\n        -------\n        List of dictionaries describing the discovered versions in the library\n        \"\"\"\n        if symbol is None:\n            symbols = self.list_symbols(snapshot=snapshot)\n        else:\n            symbols = [symbol]\n\n        query = {}\n\n        if snapshot is not None:\n            try:\n                query['parent'] = self._snapshots.find_one({'name': snapshot})['_id']\n            except TypeError:\n                raise NoDataFoundException('No snapshot %s in library %s' % (snapshot, self._arctic_lib.get_name()))\n\n        versions = []\n        snapshots = {ss.get('_id'): ss.get('name') for ss in self._snapshots.find()}\n        for symbol in symbols:\n            query['symbol'] = symbol\n            seen_symbols = set()\n            for version in self._versions.find(query, projection=['symbol', 'version', 'parent', 'metadata.deleted'], sort=[('version', -1)]):\n                if latest_only and version['symbol'] in seen_symbols:\n                    continue\n                seen_symbols.add(version['symbol'])\n                meta = version.get('metadata')\n                versions.append({'symbol': version['symbol'], 'version': version['version'],\n                                 'deleted': meta.get('deleted', False) if meta else False,\n                                 # We return offset-aware datetimes in Local Time.\n                                 'date': ms_to_datetime(datetime_to_ms(version['_id'].generation_time)),\n                                 'snapshots': [snapshots[s] for s in version.get('parent', []) if s in snapshots]})\n        return versions"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreads the data for the named symbol from the version store.", "response": "def read(self, symbol, as_of=None, date_range=None, from_version=None, allow_secondary=None, **kwargs):\n        \"\"\"\n        Read data for the named symbol.  Returns a VersionedItem object with\n        a data and metdata element (as passed into write).\n\n        Parameters\n        ----------\n        symbol : `str`\n            symbol name for the item\n        as_of : `str` or `int` or `datetime.datetime`\n            Return the data as it was as_of the point in time.\n            `int` : specific version number\n            `str` : snapshot name which contains the version\n            `datetime.datetime` : the version of the data that existed as_of the requested point in time\n        date_range: `arctic.date.DateRange`\n            DateRange to read data for.  Applies to Pandas data, with a DateTime index\n            returns only the part of the data that falls in the DateRange.\n        allow_secondary : `bool` or `None`\n            Override the default behavior for allowing reads from secondary members of a cluster:\n            `None` : use the settings from the top-level `Arctic` object used to query this version store.\n            `True` : allow reads from secondary members\n            `False` : only allow reads from primary members\n\n        Returns\n        -------\n        VersionedItem namedtuple which contains a .data and .metadata element\n        \"\"\"\n        try:\n            read_preference = self._read_preference(allow_secondary)\n            _version = self._read_metadata(symbol, as_of=as_of, read_preference=read_preference)\n            return self._do_read(symbol, _version, from_version,\n                                 date_range=date_range, read_preference=read_preference, **kwargs)\n        except (OperationFailure, AutoReconnect) as e:\n            # Log the exception so we know how often this is happening\n            log_exception('read', e, 1)\n            # If we've failed to read from the secondary, then it's possible the\n            # secondary has lagged.  In this case direct the query to the primary.\n            _version = mongo_retry(self._read_metadata)(symbol, as_of=as_of,\n                                                        read_preference=ReadPreference.PRIMARY)\n            return self._do_read_retry(symbol, _version, from_version,\n                                       date_range=date_range,\n                                       read_preference=ReadPreference.PRIMARY,\n                                       **kwargs)\n        except Exception as e:\n            log_exception('read', e, 1)\n            raise"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_info(self, symbol, as_of=None):\n        version = self._read_metadata(symbol, as_of=as_of, read_preference=None)\n        handler = self._read_handler(version, symbol)\n        if handler and hasattr(handler, 'get_info'):\n            return handler.get_info(version)\n        return {}", "response": "Reads and returns the information about the item with the given symbol."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn the numerical representation of the arctic version used to write the last version for the given symbol.", "response": "def get_arctic_version(self, symbol, as_of=None):\n        \"\"\"\n        Return the numerical representation of the arctic version used to write the last (or as_of) version for\n        the given symbol.\n\n        Parameters\n        ----------\n        symbol : `str`\n            symbol name for the item\n        as_of : `str` or int or `datetime.datetime`\n            Return the data as it was as_of the point in time.\n            `int` : specific version number\n            `str` : snapshot name which contains the version\n            `datetime.datetime` : the version of the data that existed as_of the requested point in time\n\n        Returns\n        -------\n        arctic_version : int\n            The numerical representation of Arctic version, used to create the specified symbol version\n        \"\"\"\n        return self._read_metadata(symbol, as_of=as_of).get('arctic_version', 0)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef read_metadata(self, symbol, as_of=None, allow_secondary=None):\n        _version = self._read_metadata(symbol, as_of=as_of, read_preference=self._read_preference(allow_secondary))\n        return VersionedItem(symbol=symbol, library=self._arctic_lib.get_name(), version=_version['version'],\n                             metadata=_version.pop('metadata', None), data=None,\n                             host=self._arctic_lib.arctic.mongo_host)", "response": "Reads the metadata for a specific item."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef append(self, symbol, data, metadata=None, prune_previous_version=True, upsert=True, **kwargs):\n        self._arctic_lib.check_quota()\n        version = {'_id': bson.ObjectId()}\n        version['arctic_version'] = ARCTIC_VERSION_NUMERICAL\n        version['symbol'] = symbol\n        spec = {'symbol': symbol}\n        previous_version = self._versions.find_one(spec,\n                                                   sort=[('version', pymongo.DESCENDING)])\n\n        if len(data) == 0 and previous_version is not None:\n            return VersionedItem(symbol=symbol, library=self._arctic_lib.get_name(), version=previous_version['version'],\n                                 metadata=version.pop('metadata', None), data=None,\n                                 host=self._arctic_lib.arctic.mongo_host)\n\n        if upsert and previous_version is None:\n            return self.write(symbol=symbol, data=data, prune_previous_version=prune_previous_version, metadata=metadata)\n\n        assert previous_version is not None\n        dirty_append = False\n\n        # Take a version number for this append.\n        # If the version numbers of this and the previous version aren't sequential,\n        # then we've either had a failed append in the past,\n        # we're in the midst of a concurrent update, we've deleted a version in-between\n        # or are somehow 'forking' history\n        next_ver = self._version_nums.find_one_and_update({'symbol': symbol, },\n                                                          {'$inc': {'version': 1}},\n                                                          upsert=False, new=True)['version']\n\n        # This is a very important check, do not remove/modify as it is a guard-dog preventing potential data corruption\n        if next_ver != previous_version['version'] + 1:\n            dirty_append = True\n            logger.debug('''version_nums is out of sync with previous version document.\n            This probably means that either a version document write has previously failed, or the previous version has been deleted.''')\n\n        # if the symbol has previously been deleted then overwrite\n        previous_metadata = previous_version.get('metadata', None)\n        if upsert and previous_metadata is not None and previous_metadata.get('deleted', False) is True:\n            return self.write(symbol=symbol, data=data, prune_previous_version=prune_previous_version,\n                              metadata=metadata)\n\n        handler = self._read_handler(previous_version, symbol)\n\n        if metadata is not None:\n            version['metadata'] = metadata\n        elif 'metadata' in previous_version:\n            version['metadata'] = previous_version['metadata']\n\n        if handler and hasattr(handler, 'append') and callable(handler.append):\n            handler.append(self._arctic_lib, version, symbol, data,\n                           previous_version, dirty_append=dirty_append, **kwargs)\n        else:\n            raise Exception(\"Append not implemented for handler %s\" % handler)\n\n        if prune_previous_version and previous_version:\n            # Does not allow prune to remove the base of the new version\n            self._prune_previous_versions(symbol, keep_version=version.get('base_version_id'),\n                                          new_version_shas=version.get(FW_POINTERS_REFS_KEY))\n\n        # Insert the new version into the version DB\n        version['version'] = next_ver\n        self._insert_version(version)\n\n        return VersionedItem(symbol=symbol, library=self._arctic_lib.get_name(), version=version['version'],\n                             metadata=version.pop('metadata', None), data=None,\n                             host=self._arctic_lib.arctic.mongo_host)", "response": "Append data under the specified symbol name to this library."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef write(self, symbol, data, metadata=None, prune_previous_version=True, **kwargs):\n        self._arctic_lib.check_quota()\n        version = {'_id': bson.ObjectId()}\n        version['arctic_version'] = ARCTIC_VERSION_NUMERICAL\n        version['symbol'] = symbol\n        version['version'] = self._version_nums.find_one_and_update({'symbol': symbol},\n                                                                    {'$inc': {'version': 1}},\n                                                                    upsert=True, new=True)['version']\n        version['metadata'] = metadata\n\n        previous_version = self._versions.find_one({'symbol': symbol, 'version': {'$lt': version['version']}},\n                                                   sort=[('version', pymongo.DESCENDING)])\n\n        handler = self._write_handler(version, symbol, data, **kwargs)\n        handler.write(self._arctic_lib, version, symbol, data, previous_version, **kwargs)\n\n        if prune_previous_version and previous_version:\n            self._prune_previous_versions(symbol, new_version_shas=version.get(FW_POINTERS_REFS_KEY))\n\n        # Insert the new version into the version DB\n        self._insert_version(version)\n\n        logger.debug('Finished writing versions for %s', symbol)\n\n        return VersionedItem(symbol=symbol, library=self._arctic_lib.get_name(), version=version['version'],\n                             metadata=version.pop('metadata', None), data=None,\n                             host=self._arctic_lib.arctic.mongo_host)", "response": "Writes the data under the specified symbol to this library."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef write_metadata(self, symbol, metadata, prune_previous_version=True, **kwargs):\n        # Make a normal write with empty data and supplied metadata if symbol does not exist\n        try:\n            previous_version = self._read_metadata(symbol)\n        except NoDataFoundException:\n            return self.write(symbol, data=None, metadata=metadata,\n                              prune_previous_version=prune_previous_version, **kwargs)\n\n        # Reaching here means that and/or metadata exist and we are set to update the metadata\n        new_version_num = self._version_nums.find_one_and_update({'symbol': symbol},\n                                                                 {'$inc': {'version': 1}},\n                                                                 upsert=True, new=True)['version']\n\n        # Populate the new version entry, preserving existing data, and updating with the supplied metadata\n        version = {k: previous_version[k] for k in previous_version.keys() if k != 'parent'}   # don't copy snapshots\n        version['_id'] = bson.ObjectId()\n        version['version'] = new_version_num\n        version['metadata'] = metadata\n        version['base_version_id'] = previous_version.get('base_version_id', previous_version['_id'])\n\n        return self._add_new_version_using_reference(symbol, version, previous_version, prune_previous_version)", "response": "Writes the metadata for the specified symbol to this library."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef restore_version(self, symbol, as_of, prune_previous_version=True):\n        # TODO: This operation is tricky as it may create history branches and lead to corrupted symbols.\n        #       To avoid this we do concat_rewrite (see Issue #579)\n        #       Investigate how this can be optimized and maintain safety (i.e. avoid read/write with serialization\n        #       and compression costs, but instead:\n        #       clone segments (server-side?) / crate new (base) version / update segments' parent).\n\n        version_to_restore = self._read_metadata(symbol, as_of=as_of)\n\n        # At this point it is guaranteed that the as_of version exists and doesn't have the symbol marked as deleted.\n        # If we try to restore the last version, do nothing (No-Op) and return the associated VesionedItem.\n        if self._last_version_seqnum(symbol) == version_to_restore['version']:\n            return VersionedItem(symbol=symbol, library=self._arctic_lib.get_name(),\n                                 version=version_to_restore['version'],\n                                 host=self._arctic_lib.arctic.mongo_host,\n                                 metadata=version_to_restore.pop('metadata', None), data=None)\n\n        # Read the existing data from as_of\n        item = self.read(symbol, as_of=as_of)\n\n        # Write back, creating a new base version\n        new_item = self.write(symbol,\n                              data=item.data, metadata=item.metadata, prune_previous_version=prune_previous_version)\n        return new_item", "response": "Restores the specified symbol s data and metadata to the state of a given version."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _find_prunable_version_ids(self, symbol, keep_mins):\n        read_preference = ReadPreference.SECONDARY_PREFERRED if keep_mins > 0 else ReadPreference.PRIMARY\n        versions = self._versions.with_options(read_preference=read_preference)\n        query = {'symbol': symbol,\n                 # Not snapshotted\n                 '$or': [{'parent': {'$exists': False}}, {'parent': []}],\n                 # At least 'keep_mins' old\n                 '_id': {'$lt': bson.ObjectId.from_datetime(dt.utcnow()\n                                                            # Add one second as the ObjectId\n                                                            # str has random fuzz\n                                                            + timedelta(seconds=1)\n                                                            - timedelta(minutes=keep_mins)\n                                                            )\n                         }\n                 }\n        cursor = versions.find(query,\n                               # Using version number here instead of _id as there's a very unlikely case\n                               # where the versions are created on different hosts or processes at exactly\n                               # the same time.\n                               sort=[('version', pymongo.DESCENDING)],\n                               # Guarantees at least one version is kept\n                               skip=1,\n                               projection={'_id': 1, FW_POINTERS_REFS_KEY: 1, FW_POINTERS_CONFIG_KEY: 1},\n                               )\n        return {v['_id']: ([bson.binary.Binary(x) for x in v.get(FW_POINTERS_REFS_KEY, [])], get_fwptr_config(v))\n                for v in cursor}", "response": "Find all non - snapshotted versions of a symbol that are older than keep_mins."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _find_base_version_ids(self, symbol, version_ids):\n        cursor = self._versions.find({'symbol': symbol,\n                                      '_id': {'$nin': version_ids},\n                                      'base_version_id': {'$exists': True},\n                                      },\n                                     projection={'base_version_id': 1})\n        return [version[\"base_version_id\"] for version in cursor]", "response": "Find all base_version_ids for a symbol that are not bases of version_ids\n       "}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _prune_previous_versions(self, symbol, keep_mins=120, keep_version=None, new_version_shas=None):\n        new_version_shas = new_version_shas if new_version_shas else []\n        prunable_ids_to_shas = self._find_prunable_version_ids(symbol, keep_mins)\n        prunable_ids = list(prunable_ids_to_shas.keys())\n        if keep_version is not None:\n            try:\n                prunable_ids.remove(keep_version)\n            except ValueError:\n                pass\n        if not prunable_ids:\n            return\n\n        base_version_ids = self._find_base_version_ids(symbol, prunable_ids)\n        version_ids = list(set(prunable_ids) - set(base_version_ids))\n        if not version_ids:\n            return\n\n        # Delete the version documents\n        mongo_retry(self._versions.delete_many)({'_id': {'$in': version_ids}})\n\n        prunable_ids_to_shas = {k: prunable_ids_to_shas[k] for k in version_ids}\n\n        # The new version has not been written yet, so make sure that any SHAs pointed by it are preserved\n        shas_to_delete = [sha for v in prunable_ids_to_shas.values() for sha in v[0] if sha not in new_version_shas]\n\n        # Cleanup any chunks\n        mongo_retry(cleanup)(self._arctic_lib, symbol, version_ids, self._versions,\n                             shas_to_delete=shas_to_delete,\n                             pointers_cfgs=[v[1] for v in prunable_ids_to_shas.values()])", "response": "Prune versions not pointed at by snapshots which are at least keep_mins old."}