diff --git "a/chunk_17.jsonl" "b/chunk_17.jsonl" new file mode 100644--- /dev/null +++ "b/chunk_17.jsonl" @@ -0,0 +1,3000 @@ +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ndetects all available configurations for the given interfaces.", "response": "def detect_available_configs(interfaces=None):\n \"\"\"Detect all configurations/channels that the interfaces could\n currently connect with.\n\n This might be quite time consuming.\n\n Automated configuration detection may not be implemented by\n every interface on every platform. This method will not raise\n an error in that case, but with rather return an empty list\n for that interface.\n\n :param interfaces: either\n - the name of an interface to be searched in as a string,\n - an iterable of interface names to search in, or\n - `None` to search in all known interfaces.\n :rtype: list[dict]\n :return: an iterable of dicts, each suitable for usage in\n the constructor of :class:`can.BusABC`.\n \"\"\"\n\n # Figure out where to search\n if interfaces is None:\n # use an iterator over the keys so we do not have to copy it\n interfaces = BACKENDS.keys()\n elif isinstance(interfaces, basestring):\n interfaces = [interfaces, ]\n # else it is supposed to be an iterable of strings\n\n result = []\n for interface in interfaces:\n\n try:\n bus_class = _get_class_for_interface(interface)\n except ImportError:\n log_autodetect.debug('interface \"%s\" can not be loaded for detection of available configurations', interface)\n continue\n\n # get available channels\n try:\n available = list(bus_class._detect_available_configs())\n except NotImplementedError:\n log_autodetect.debug('interface \"%s\" does not support detection of available configurations', interface)\n else:\n log_autodetect.debug('interface \"%s\" detected %i available configurations', interface, len(available))\n\n # add the interface name to the configs if it is not already present\n for config in available:\n if 'interface' not in config:\n config['interface'] = interface\n\n # append to result\n result += available\n\n return result"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef add_bus(self, bus):\n if self._loop is not None and hasattr(bus, 'fileno') and bus.fileno() >= 0:\n # Use file descriptor to watch for messages\n reader = bus.fileno()\n self._loop.add_reader(reader, self._on_message_available, bus)\n else:\n reader = threading.Thread(target=self._rx_thread, args=(bus,),\n name='can.notifier for bus \"{}\"'.format(bus.channel_info))\n reader.daemon = True\n reader.start()\n self._readers.append(reader)", "response": "Add a bus for notification."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef stop(self, timeout=5):\n self._running = False\n end_time = time.time() + timeout\n for reader in self._readers:\n if isinstance(reader, threading.Thread):\n now = time.time()\n if now < end_time:\n reader.join(end_time - now)\n else:\n # reader is a file descriptor\n self._loop.remove_reader(reader)\n for listener in self.listeners:\n if hasattr(listener, 'stop'):\n listener.stop()", "response": "Stop waiting for new messages arrive in the cache."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nsending a message over the serial device.", "response": "def send(self, msg, timeout=None):\n \"\"\"\n Send a message over the serial device.\n\n :param can.Message msg:\n Message to send.\n\n .. note:: Flags like ``extended_id``, ``is_remote_frame`` and\n ``is_error_frame`` will be ignored.\n\n .. note:: If the timestamp is a float value it will be converted\n to an integer.\n\n :param timeout:\n This parameter will be ignored. The timeout value of the channel is\n used instead.\n\n \"\"\"\n try:\n timestamp = struct.pack('= 0:\n self._ucan.set_tx_timeout(self.channel, int(timeout * 1000))\n\n message = CanMsg(msg.arbitration_id,\n MsgFrameFormat.MSG_FF_STD |\n (MsgFrameFormat.MSG_FF_EXT if msg.is_extended_id else 0) |\n (MsgFrameFormat.MSG_FF_RTR if msg.is_remote_frame else 0),\n msg.data)\n self._ucan.write_can_msg(self.channel, [message])"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef flush_tx_buffer(self):\n log.info('Flushing transmit buffer')\n self._ucan.reset_can(self.channel, ResetFlags.RESET_ONLY_TX_BUFF)", "response": "Flushes the transmit buffer."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef create_filter(extended, from_id, to_id, rtr_only, rtr_too):\n return [{\n \"can_id\": Ucan.calculate_acr(extended, from_id, to_id, rtr_only, rtr_too),\n \"can_mask\": Ucan.calculate_amr(extended, from_id, to_id, rtr_only, rtr_too),\n \"extended\": extended\n }]", "response": "Create a filter for a given set of keys."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef shutdown(self):\n try:\n self._ucan.shutdown()\n except Exception as ex:\n log.error(ex)", "response": "Shuts down all CAN interfaces and hardware interfaces."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef load_file_config(path=None, section=None):\n config = ConfigParser()\n if path is None:\n config.read([os.path.expanduser(path) for path in CONFIG_FILES])\n else:\n config.read(path)\n\n _config = {}\n\n section = section if section is not None else 'default'\n if config.has_section(section):\n if config.has_section('default'):\n _config.update(\n dict((key, val) for key, val in config.items('default')))\n _config.update(dict((key, val) for key, val in config.items(section)))\n\n return _config", "response": "Loads configuration from file with following content ::\n [ default ]"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef load_environment_config():\n mapper = {\n 'interface': 'CAN_INTERFACE',\n 'channel': 'CAN_CHANNEL',\n 'bitrate': 'CAN_BITRATE',\n }\n return dict(\n (key, os.environ.get(val))\n for key, val in mapper.items()\n if val in os.environ\n )", "response": "Loads config dict from environmental variables."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef load_config(path=None, config=None, context=None):\n\n # start with an empty dict to apply filtering to all sources\n given_config = config or {}\n config = {}\n\n # use the given dict for default values\n config_sources = [\n given_config,\n can.rc,\n lambda _context: load_environment_config(), # context is not supported\n lambda _context: load_file_config(path, _context)\n ]\n\n # Slightly complex here to only search for the file config if required\n for cfg in config_sources:\n if callable(cfg):\n cfg = cfg(context)\n # remove legacy operator (and copy to interface if not already present)\n if 'bustype' in cfg:\n if 'interface' not in cfg or not cfg['interface']:\n cfg['interface'] = cfg['bustype']\n del cfg['bustype']\n # copy all new parameters\n for key in cfg:\n if key not in config:\n config[key] = cfg[key]\n\n # substitute None for all values not found\n for key in REQUIRED_KEYS:\n if key not in config:\n config[key] = None\n\n # Handle deprecated socketcan types\n if config['interface'] in ('socketcan_native', 'socketcan_ctypes'):\n # DeprecationWarning in 3.x releases\n # TODO: Remove completely in 4.0\n warnings.warn('{} is deprecated, use socketcan instead'.format(config['interface']), DeprecationWarning)\n config['interface'] = 'socketcan'\n\n if config['interface'] not in VALID_INTERFACES:\n raise NotImplementedError('Invalid CAN Bus Type - {}'.format(config['interface']))\n\n if 'bitrate' in config:\n config['bitrate'] = int(config['bitrate'])\n\n can.log.debug(\"can config: {}\".format(config))\n return config", "response": "Load a configuration file and return a dictionary of configuration details."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef set_logging_level(level_name=None):\n can_logger = logging.getLogger('can')\n\n try:\n can_logger.setLevel(getattr(logging, level_name.upper()))\n except AttributeError:\n can_logger.setLevel(logging.DEBUG)\n log.debug(\"Logging set to {}\".format(level_name))", "response": "Set the logging level for the CAN logger."} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncalculating the DLC from data length.", "response": "def len2dlc(length):\n \"\"\"Calculate the DLC from data length.\n\n :param int length: Length in number of bytes (0-64)\n\n :returns: DLC (0-15)\n :rtype: int\n \"\"\"\n if length <= 8:\n return length\n for dlc, nof_bytes in enumerate(CAN_FD_DLC):\n if nof_bytes >= length:\n return dlc\n return 15"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef channel2int(channel):\n if channel is None:\n return None\n if isinstance(channel, int):\n return channel\n # String and byte objects have a lower() method\n if hasattr(channel, \"lower\"):\n match = re.match(r'.*(\\d+)$', channel)\n if match:\n return int(match.group(1))\n return None", "response": "Try to convert the channel to an integer."} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nblock waiting for a message from the Bus.", "response": "def recv(self, timeout=None):\n \"\"\"Block waiting for a message from the Bus.\n\n :type timeout: float or None\n :param timeout:\n seconds to wait for a message or None to wait indefinitely\n\n :rtype: can.Message or None\n :return:\n None on timeout or a :class:`can.Message` object.\n :raises can.CanError:\n if an error occurred while reading\n \"\"\"\n start = time()\n time_left = timeout\n\n while True:\n\n # try to get a message\n msg, already_filtered = self._recv_internal(timeout=time_left)\n\n # return it, if it matches\n if msg and (already_filtered or self._matches_filters(msg)):\n LOG.log(self.RECV_LOGGING_LEVEL, 'Received: %s', msg)\n return msg\n\n # if not, and timeout is None, try indefinitely\n elif timeout is None:\n continue\n\n # try next one only if there still is time, and with\n # reduced timeout\n else:\n\n time_left = timeout - (time() - start)\n\n if time_left > 0:\n continue\n else:\n return None"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nstarting sending a message at a given period.", "response": "def send_periodic(self, msg, period, duration=None, store_task=True):\n \"\"\"Start sending a message at a given period on this bus.\n\n The task will be active until one of the following conditions are met:\n\n - the (optional) duration expires\n - the Bus instance goes out of scope\n - the Bus instance is shutdown\n - :meth:`BusABC.stop_all_periodic_tasks()` is called\n - the task's :meth:`CyclicTask.stop()` method is called.\n\n :param can.Message msg:\n Message to transmit\n :param float period:\n Period in seconds between each message\n :param float duration:\n The duration to keep sending this message at given rate. If\n no duration is provided, the task will continue indefinitely.\n :param bool store_task:\n If True (the default) the task will be attached to this Bus instance.\n Disable to instead manage tasks manually.\n :return:\n A started task instance. Note the task can be stopped (and depending on\n the backend modified) by calling the :meth:`stop` method.\n :rtype: can.broadcastmanager.CyclicSendTaskABC\n\n .. note::\n\n Note the duration before the message stops being sent may not\n be exactly the same as the duration specified by the user. In\n general the message will be sent at the given rate until at\n least **duration** seconds.\n\n .. note::\n\n For extremely long running Bus instances with many short lived tasks the default\n api with ``store_task==True`` may not be appropriate as the stopped tasks are\n still taking up memory as they are associated with the Bus instance.\n \"\"\"\n task = self._send_periodic_internal(msg, period, duration)\n # we wrap the task's stop method to also remove it from the Bus's list of tasks\n original_stop_method = task.stop\n\n def wrapped_stop_method(remove_task=True):\n if remove_task:\n try:\n self._periodic_tasks.remove(task)\n except ValueError:\n pass\n original_stop_method()\n task.stop = wrapped_stop_method\n\n if store_task:\n self._periodic_tasks.append(task)\n\n return task"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _send_periodic_internal(self, msg, period, duration=None):\n if not hasattr(self, \"_lock_send_periodic\"):\n # Create a send lock for this bus\n self._lock_send_periodic = threading.Lock()\n task = ThreadBasedCyclicSendTask(self, self._lock_send_periodic, msg, period, duration)\n return task", "response": "This method is used to send a message in a thread - based manner."} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nstops sending any messages that were started using bus. send_periodic .", "response": "def stop_all_periodic_tasks(self, remove_tasks=True):\n \"\"\"Stop sending any messages that were started using bus.send_periodic\n\n :param bool remove_tasks:\n Stop tracking the stopped tasks.\n \"\"\"\n for task in self._periodic_tasks:\n task.stop(remove_task=remove_tasks)"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef set_filters(self, filters=None):\n self._filters = filters or None\n self._apply_filters(self._filters)", "response": "Apply filters to all messages received by this Bus."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _matches_filters(self, msg):\n\n # if no filters are set, all messages are matched\n if self._filters is None:\n return True\n\n for _filter in self._filters:\n # check if this filter even applies to the message\n if 'extended' in _filter and \\\n _filter['extended'] != msg.is_extended_id:\n continue\n\n # then check for the mask and id\n can_id = _filter['can_id']\n can_mask = _filter['can_mask']\n\n # basically, we compute\n # `msg.arbitration_id & can_mask == can_id & can_mask`\n # by using the shorter, but equivalent from below:\n if (can_id ^ msg.arbitration_id) & can_mask == 0:\n return True\n\n # nothing matched\n return False", "response": "Checks whether the given message matches at least one of the current filters."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef check_valid_rx_can_msg(result):\n return (result.value == ReturnCode.SUCCESSFUL) or (result.value > ReturnCode.WARNING)", "response": "Checks if function UcanServer. read_can_msg returns a valid CAN message."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nchecking if function : meth:`UcanServer. write_can_msg` successfully wrote CAN messages.", "response": "def check_tx_ok(result):\n \"\"\"\n Checks if function :meth:`UcanServer.write_can_msg` successfully wrote CAN message(s).\n\n While using :meth:`UcanServer.write_can_msg_ex` the number of sent CAN messages can be less than\n the number of CAN messages which should be sent.\n\n :param ReturnCode result: Error code of the function.\n :return: True if CAN message(s) was(were) written successfully, otherwise False.\n :rtype: bool\n\n .. :seealso: :const:`ReturnCode.WARN_TXLIMIT`\n \"\"\"\n return (result.value == ReturnCode.SUCCESSFUL) or (result.value > ReturnCode.WARNING)"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef check_error(result):\n return (result.value != ReturnCode.SUCCESSFUL) and (result.value < ReturnCode.WARNING)", "response": "Checks if any function returns an error from USB - CAN - library."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef check_error_cmd(result):\n return (result.value >= ReturnCode.ERRCMD) and (result.value < ReturnCode.WARNING)", "response": "Checks if any function returns an error from firmware in USB - CANmodul."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ninitializing the hardware with the corresponding serial or device number.", "response": "def init_hardware(self, serial=None, device_number=ANY_MODULE):\n \"\"\"\n Initializes the device with the corresponding serial or device number.\n\n :param int or None serial: Serial number of the USB-CANmodul.\n :param int device_number: Device number (0 \u2013 254, or :const:`ANY_MODULE` for the first device).\n \"\"\"\n if not self._hw_is_initialized:\n # initialize hardware either by device number or serial\n if serial is None:\n UcanInitHardwareEx(byref(self._handle), device_number, self._callback_ref, None)\n else:\n UcanInitHardwareEx2(byref(self._handle), serial, self._callback_ref, None)\n self._hw_is_initialized = True"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef init_can(self, channel=Channel.CHANNEL_CH0, BTR=Baudrate.BAUD_1MBit, baudrate=BaudrateEx.BAUDEX_USE_BTR01,\n AMR=AMR_ALL, ACR=ACR_ALL, mode=Mode.MODE_NORMAL, OCR=OutputControl.OCR_DEFAULT,\n rx_buffer_entries=DEFAULT_BUFFER_ENTRIES, tx_buffer_entries=DEFAULT_BUFFER_ENTRIES):\n \"\"\"\n Initializes a specific CAN channel of a device.\n\n :param int channel: CAN channel to be initialized (:data:`Channel.CHANNEL_CH0` or :data:`Channel.CHANNEL_CH1`).\n :param int BTR:\n Baud rate register BTR0 as high byte, baud rate register BTR1 as low byte (see enum :class:`Baudrate`).\n :param int baudrate: Baud rate register for all systec USB-CANmoduls (see enum :class:`BaudrateEx`).\n :param int AMR: Acceptance filter mask (see method :meth:`set_acceptance`).\n :param int ACR: Acceptance filter code (see method :meth:`set_acceptance`).\n :param int mode: Transmission mode of CAN channel (see enum :class:`Mode`).\n :param int OCR: Output Control Register (see enum :class:`OutputControl`).\n :param int rx_buffer_entries: The number of maximum entries in the receive buffer.\n :param int tx_buffer_entries: The number of maximum entries in the transmit buffer.\n \"\"\"\n if not self._ch_is_initialized.get(channel, False):\n init_param = InitCanParam(mode, BTR, OCR, AMR, ACR, baudrate, rx_buffer_entries, tx_buffer_entries)\n UcanInitCanEx2(self._handle, channel, init_param)\n self._ch_is_initialized[channel] = True", "response": "Initializes a specific CAN channel."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef read_can_msg(self, channel, count):\n c_channel = BYTE(channel)\n c_can_msg = (CanMsg * count)()\n c_count = DWORD(count)\n UcanReadCanMsgEx(self._handle, byref(c_channel), c_can_msg, byref(c_count))\n return c_can_msg[:c_count.value], c_channel.value", "response": "Reads one or more CAN messages from the buffer of the specified CAN channel."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ntransmitting one ore more CAN messages through the specified CAN channel of the device.", "response": "def write_can_msg(self, channel, can_msg):\n \"\"\"\n Transmits one ore more CAN messages through the specified CAN channel of the device.\n\n :param int channel:\n CAN channel, which is to be used (:data:`Channel.CHANNEL_CH0` or :data:`Channel.CHANNEL_CH1`).\n :param list(CanMsg) can_msg: List of CAN message structure (see structure :class:`CanMsg`).\n :return: The number of successfully transmitted CAN messages.\n :rtype: int\n \"\"\"\n c_can_msg = (CanMsg * len(can_msg))(*can_msg)\n c_count = DWORD(len(can_msg))\n UcanWriteCanMsgEx(self._handle, channel, c_can_msg, c_count)\n return c_count"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef set_acceptance(self, channel=Channel.CHANNEL_CH0, AMR=AMR_ALL, ACR=ACR_ALL):\n UcanSetAcceptanceEx(self._handle, channel, AMR, ACR)", "response": "This function is used to change the acceptance filter values for a specific CAN channel on a device."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_status(self, channel=Channel.CHANNEL_CH0):\n status = Status()\n UcanGetStatusEx(self._handle, channel, byref(status))\n return status.can_status, status.usb_status", "response": "Returns the error status of a specific CAN channel."} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nread the message counters of the specified CAN channel.", "response": "def get_msg_count_info(self, channel=Channel.CHANNEL_CH0):\n \"\"\"\n Reads the message counters of the specified CAN channel.\n\n :param int channel:\n CAN channel, which is to be used (:data:`Channel.CHANNEL_CH0` or :data:`Channel.CHANNEL_CH1`).\n :return: Tuple with number of CAN messages sent and received.\n :rtype: tuple(int, int)\n \"\"\"\n msg_count_info = MsgCountInfo()\n UcanGetMsgCountInfoEx(self._handle, channel, byref(msg_count_info))\n return msg_count_info.sent_msg_count, msg_count_info.recv_msg_count"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef reset_can(self, channel=Channel.CHANNEL_CH0, flags=ResetFlags.RESET_ALL):\n UcanResetCanEx(self._handle, channel, flags)", "response": "Reset a CAN channel of a device."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_hardware_info(self):\n hw_info_ex = HardwareInfoEx()\n can_info_ch0, can_info_ch1 = ChannelInfo(), ChannelInfo()\n UcanGetHardwareInfoEx2(self._handle, byref(hw_info_ex), byref(can_info_ch0), byref(can_info_ch1))\n return hw_info_ex, can_info_ch0, can_info_ch1", "response": "Returns the extended hardware information structure of a USB - CAN moduls device."} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ndefines a list of CAN messages for automatic transmission.", "response": "def define_cyclic_can_msg(self, channel, can_msg=None):\n \"\"\"\n Defines a list of CAN messages for automatic transmission.\n\n :param int channel: CAN channel, to be used (:data:`Channel.CHANNEL_CH0` or :data:`Channel.CHANNEL_CH1`).\n :param list(CanMsg) can_msg:\n List of CAN messages (up to 16, see structure :class:`CanMsg`), or None to delete an older list.\n \"\"\"\n if can_msg is not None:\n c_can_msg = (CanMsg * len(can_msg))(*can_msg)\n c_count = DWORD(len(can_msg))\n else:\n c_can_msg = CanMsg()\n c_count = 0\n UcanDefineCyclicCanMsg(self._handle, channel, c_can_msg, c_count)"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef read_cyclic_can_msg(self, channel, count):\n c_channel = BYTE(channel)\n c_can_msg = (CanMsg * count)()\n c_count = DWORD(count)\n UcanReadCyclicCanMsg(self._handle, byref(c_channel), c_can_msg, c_count)\n return c_can_msg[:c_count.value]", "response": "Reads back the list of cyclic CAN messages for automatically sending."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_msg_pending(self, channel, flags):\n count = DWORD(0)\n UcanGetMsgPending(self._handle, channel, flags, byref(count))\n return count.value", "response": "Get the number of pending CAN messages."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_can_error_counter(self, channel):\n tx_error_counter = DWORD(0)\n rx_error_counter = DWORD(0)\n UcanGetCanErrorCounter(self._handle, channel, byref(tx_error_counter), byref(rx_error_counter))\n return tx_error_counter, rx_error_counter", "response": "Reads the current value of the error counters within the CAN controller."} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nset the transmission timeout in seconds for the specified CAN channel.", "response": "def set_tx_timeout(self, channel, timeout):\n \"\"\"\n Sets the transmission timeout.\n\n :param int channel: CAN channel, to be used (:data:`Channel.CHANNEL_CH0` or :data:`Channel.CHANNEL_CH1`).\n :param float timeout: Transmit timeout in seconds (value 0 disables this feature).\n \"\"\"\n UcanSetTxTimeout(self._handle, channel, int(timeout * 1000))"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nshut down all CAN interfaces and or the hardware interfaces.", "response": "def shutdown(self, channel=Channel.CHANNEL_ALL, shutdown_hardware=True):\n \"\"\"\n Shuts down all CAN interfaces and/or the hardware interface.\n\n :param int channel:\n CAN channel, to be used (:data:`Channel.CHANNEL_CH0`, :data:`Channel.CHANNEL_CH1` or\n :data:`Channel.CHANNEL_ALL`)\n :param bool shutdown_hardware: If true then the hardware interface will be closed too.\n \"\"\"\n # shutdown each channel if it's initialized\n for _channel, is_initialized in self._ch_is_initialized.items():\n if is_initialized and (_channel == channel or channel == Channel.CHANNEL_ALL or shutdown_hardware):\n UcanDeinitCanEx(self._handle, _channel)\n self._ch_is_initialized[_channel] = False\n\n # shutdown hardware\n if self._hw_is_initialized and shutdown_hardware:\n UcanDeinitHardware(self._handle)\n self._hw_is_initialized = False\n self._handle = Handle(INVALID_HANDLE)"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nconverting a given CAN status value to a string.", "response": "def get_can_status_message(can_status):\n \"\"\"\n Converts a given CAN status value to the appropriate message string.\n\n :param can_status: CAN status value from method :meth:`get_status` (see enum :class:`CanStatus`)\n :return: Status message string.\n :rtype: str\n \"\"\"\n status_msgs = {\n CanStatus.CANERR_TXMSGLOST: \"Transmit message lost\",\n CanStatus.CANERR_MEMTEST: \"Memory test failed\",\n CanStatus.CANERR_REGTEST: \"Register test failed\",\n CanStatus.CANERR_QXMTFULL: \"Transmit queue is full\",\n CanStatus.CANERR_QOVERRUN: \"Receive queue overrun\",\n CanStatus.CANERR_QRCVEMPTY: \"Receive queue is empty\",\n CanStatus.CANERR_BUSOFF: \"Bus Off\",\n CanStatus.CANERR_BUSHEAVY: \"Error Passive\",\n CanStatus.CANERR_BUSLIGHT: \"Warning Limit\",\n CanStatus.CANERR_OVERRUN: \"Rx-buffer is full\",\n CanStatus.CANERR_XMTFULL: \"Tx-buffer is full\",\n }\n return \"OK\" if can_status == CanStatus.CANERR_OK \\\n else \", \".join(msg for status, msg in status_msgs.items() if can_status & status)"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nconverting a given baud rate value for GW - 01 - GW -002 to a appropriate message string.", "response": "def get_baudrate_message(baudrate):\n \"\"\"\n Converts a given baud rate value for GW-001/GW-002 to the appropriate message string.\n\n :param Baudrate baudrate:\n Bus Timing Registers, BTR0 in high order byte and BTR1 in low order byte\n (see enum :class:`Baudrate`)\n :return: Baud rate message string.\n :rtype: str\n \"\"\"\n baudrate_msgs = {\n Baudrate.BAUD_AUTO: \"auto baudrate\",\n Baudrate.BAUD_10kBit: \"10 kBit/sec\",\n Baudrate.BAUD_20kBit: \"20 kBit/sec\",\n Baudrate.BAUD_50kBit: \"50 kBit/sec\",\n Baudrate.BAUD_100kBit: \"100 kBit/sec\",\n Baudrate.BAUD_125kBit: \"125 kBit/sec\",\n Baudrate.BAUD_250kBit: \"250 kBit/sec\",\n Baudrate.BAUD_500kBit: \"500 kBit/sec\",\n Baudrate.BAUD_800kBit: \"800 kBit/sec\",\n Baudrate.BAUD_1MBit: \"1 MBit/s\",\n Baudrate.BAUD_USE_BTREX: \"BTR Ext is used\",\n }\n return baudrate_msgs.get(baudrate, \"BTR is unknown (user specific)\")"} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nconverts a given baud rate value for systec USB - CANmoduls to a appropriate message string.", "response": "def get_baudrate_ex_message(baudrate_ex):\n \"\"\"\n Converts a given baud rate value for systec USB-CANmoduls to the appropriate message string.\n\n :param BaudrateEx baudrate_ex: Bus Timing Registers (see enum :class:`BaudrateEx`)\n :return: Baud rate message string.\n :rtype: str\n \"\"\"\n baudrate_ex_msgs = {\n Baudrate.BAUDEX_AUTO: \"auto baudrate\",\n Baudrate.BAUDEX_10kBit: \"10 kBit/sec\",\n Baudrate.BAUDEX_SP2_10kBit: \"10 kBit/sec\",\n Baudrate.BAUDEX_20kBit: \"20 kBit/sec\",\n Baudrate.BAUDEX_SP2_20kBit: \"20 kBit/sec\",\n Baudrate.BAUDEX_50kBit: \"50 kBit/sec\",\n Baudrate.BAUDEX_SP2_50kBit: \"50 kBit/sec\",\n Baudrate.BAUDEX_100kBit: \"100 kBit/sec\",\n Baudrate.BAUDEX_SP2_100kBit: \"100 kBit/sec\",\n Baudrate.BAUDEX_125kBit: \"125 kBit/sec\",\n Baudrate.BAUDEX_SP2_125kBit: \"125 kBit/sec\",\n Baudrate.BAUDEX_250kBit: \"250 kBit/sec\",\n Baudrate.BAUDEX_SP2_250kBit: \"250 kBit/sec\",\n Baudrate.BAUDEX_500kBit: \"500 kBit/sec\",\n Baudrate.BAUDEX_SP2_500kBit: \"500 kBit/sec\",\n Baudrate.BAUDEX_800kBit: \"800 kBit/sec\",\n Baudrate.BAUDEX_SP2_800kBit: \"800 kBit/sec\",\n Baudrate.BAUDEX_1MBit: \"1 MBit/s\",\n Baudrate.BAUDEX_SP2_1MBit: \"1 MBit/s\",\n Baudrate.BAUDEX_USE_BTR01: \"BTR0/BTR1 is used\",\n }\n return baudrate_ex_msgs.get(baudrate_ex, \"BTR is unknown (user specific)\")"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ncheck if the version number is equal or higher than a specified value.", "response": "def check_version_is_equal_or_higher(cls, version, cmp_major, cmp_minor):\n \"\"\"\n Checks if the version is equal or higher than a specified value.\n\n :param int version: Version number to be checked.\n :param int cmp_major: Major version to be compared with.\n :param int cmp_minor: Minor version to be compared with.\n :return: True if equal or higher, otherwise False.\n :rtype: bool\n \"\"\"\n return (cls.convert_to_major_ver(version) > cmp_major) or \\\n (cls.convert_to_major_ver(version) == cmp_major and cls.convert_to_minor_ver(version) >= cmp_minor)"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncheck whether the module supports automatically transmission of cyclic CAN messages.", "response": "def check_support_cyclic_msg(cls, hw_info_ex):\n \"\"\"\n Checks whether the module supports automatically transmission of cyclic CAN messages.\n\n :param HardwareInfoEx hw_info_ex:\n Extended hardware information structure (see method :meth:`get_hardware_info`).\n :return: True when the module does support cyclic CAN messages, otherwise False.\n :rtype: bool\n \"\"\"\n return cls.check_is_systec(hw_info_ex) and \\\n cls.check_version_is_equal_or_higher(hw_info_ex.m_dwFwVersionEx, 3, 6)"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nchecking whether the module supports a user I / O port.", "response": "def check_support_user_port(cls, hw_info_ex):\n \"\"\"\n Checks whether the module supports a user I/O port.\n\n :param HardwareInfoEx hw_info_ex:\n Extended hardware information structure (see method :meth:`get_hardware_info`).\n :return: True when the module supports a user I/O port, otherwise False.\n :rtype: bool\n \"\"\"\n return ((hw_info_ex.m_dwProductCode & PRODCODE_MASK_PID) != ProductCode.PRODCODE_PID_BASIC) \\\n and ((hw_info_ex.m_dwProductCode & PRODCODE_MASK_PID) != ProductCode.PRODCODE_PID_RESERVED1) \\\n and cls.check_version_is_equal_or_higher(hw_info_ex.m_dwFwVersionEx, 2, 16)"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef check_support_ucannet(cls, hw_info_ex):\n return cls.check_is_systec(hw_info_ex) and \\\n cls.check_version_is_equal_or_higher(hw_info_ex.m_dwFwVersionEx, 3, 8)", "response": "Checks whether the module supports USB - CANnetwork driver."} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncalculate the AMR value for a given CAN - ID range.", "response": "def calculate_amr(cls, is_extended, from_id, to_id, rtr_only=False, rtr_too=True):\n \"\"\"\n Calculates AMR using CAN-ID range as parameter.\n\n :param bool is_extended: If True parameters from_id and to_id contains 29-bit CAN-ID.\n :param int from_id: First CAN-ID which should be received.\n :param int to_id: Last CAN-ID which should be received.\n :param bool rtr_only: If True only RTR-Messages should be received, and rtr_too will be ignored.\n :param bool rtr_too: If True CAN data frames and RTR-Messages should be received.\n :return: Value for AMR.\n :rtype: int\n \"\"\"\n return (((from_id ^ to_id) << 3) | (0x7 if rtr_too and not rtr_only else 0x3)) if is_extended else \\\n (((from_id ^ to_id) << 21) | (0x1FFFFF if rtr_too and not rtr_only else 0xFFFFF))"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _connect_control(self, event, param, arg):\n log.debug(\"Event: %s, Param: %s\" % (event, param))\n\n if event == CbEvent.EVENT_FATALDISCON:\n self.fatal_disconnect_event(param)\n elif event == CbEvent.EVENT_CONNECT:\n self.connect_event()\n elif event == CbEvent.EVENT_DISCONNECT:\n self.disconnect_event()", "response": "This method is the actual callback function for init_hw_connect_control_ex."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nbeing called if a working event occurred. :param int handle: USB-CAN-Handle returned by the function :meth:`init_hardware`. :param int event: Event type. :param int channel: CAN channel (:data:`Channel.CHANNEL_CH0`, :data:`Channel.CHANNEL_CH1` or :data:`Channel.CHANNEL_ANY`). :param arg: Additional parameter defined with :meth:`init_hardware_ex`.", "response": "def _callback(self, handle, event, channel, arg):\n \"\"\"\n Is called if a working event occurred.\n\n :param int handle: USB-CAN-Handle returned by the function :meth:`init_hardware`.\n :param int event: Event type.\n :param int channel:\n CAN channel (:data:`Channel.CHANNEL_CH0`, :data:`Channel.CHANNEL_CH1` or :data:`Channel.CHANNEL_ANY`).\n :param arg: Additional parameter defined with :meth:`init_hardware_ex`.\n \"\"\"\n log.debug(\"Handle: %s, Event: %s, Channel: %s\" % (handle, event, channel))\n\n if event == CbEvent.EVENT_INITHW:\n self.init_hw_event()\n elif event == CbEvent.EVENT_init_can:\n self.init_can_event(channel)\n elif event == CbEvent.EVENT_RECEIVE:\n self.can_msg_received_event(channel)\n elif event == CbEvent.EVENT_STATUS:\n self.status_event(channel)\n elif event == CbEvent.EVENT_DEINIT_CAN:\n self.deinit_can_event(channel)\n elif event == CbEvent.EVENT_DEINITHW:\n self.deinit_hw_event()"} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ngets colors for a specific locale", "response": "def get(self, palette):\n \"\"\"\n Colors API using schema\n This example is using marshmallow schemas\n \"\"\"\n all_colors = {\n 'cmyk': ['cian', 'magenta', 'yellow', 'black'],\n 'rgb': ['red', 'green', 'blue']\n }\n if palette == 'all':\n result = all_colors\n else:\n result = {palette: all_colors.get(palette)}\n return jsonify(result)"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef drop_id_validate(data, schema):\n jsonschema.validate(data, schema)\n if data.get('_id') is not None:\n del data['_id']", "response": "Custom validation function which drops parameter _id from data"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef special_validate(data, schema):\n jsonschema.validate(data, schema)\n data['special'] = str(data['name'] == 'Garfield').lower()", "response": "Custom validation function which inserts a special flag depending on the cat s name"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef validation_error_try_to_accept(err, data, schema):\n if not isinstance(err, ValidationError):\n abort(Response(err, status=HTTPStatus.BAD_REQUEST))\n\n alernative_schema = dict(schema)\n alernative_schema['properties']['running_time'].update({\n 'description': \"Films's running time\",\n 'type': 'integer',\n 'example': 169\n })\n\n try:\n jsonschema.validate(data, alernative_schema)\n except ValidationError as err:\n abort(Response(str(err), status=400))", "response": "Custom validation error handler which attempts alternative\n validation"} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nconverts Marshmallow schemas to dict definitions", "response": "def convert_schemas(d, definitions=None):\n \"\"\"\n Convert Marshmallow schemas to dict definitions\n\n Also updates the optional definitions argument with any definitions\n entries contained within the schema.\n \"\"\"\n if Schema is None:\n raise RuntimeError('Please install marshmallow and apispec')\n\n if definitions is None:\n definitions = {}\n definitions.update(d.get('definitions', {}))\n\n new = {}\n for k, v in d.items():\n if isinstance(v, dict):\n v = convert_schemas(v, definitions)\n if isinstance(v, (list, tuple)):\n new_v = []\n for item in v:\n if isinstance(item, dict):\n new_v.append(convert_schemas(item, definitions))\n else:\n new_v.append(item)\n v = new_v\n if inspect.isclass(v) and issubclass(v, Schema):\n definitions[v.__name__] = schema2jsonschema(v)\n ref = {\n \"$ref\": \"#/definitions/{0}\".format(v.__name__)\n }\n if k == 'parameters':\n new[k] = schema2parameters(v)\n new[k][0]['schema'] = ref\n else:\n new[k] = ref\n else:\n new[k] = v\n\n # This key is not permitted anywhere except the very top level.\n if 'definitions' in new:\n del new['definitions']\n\n return new"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nconverting the current object to flasgger suitable dict", "response": "def to_flasgger(self, app=None, definitions=None, paths=None):\n \"\"\"\n Converts APISpec dict to flasgger suitable dict\n also adds definitions and paths (optional)\n \"\"\"\n if Schema is None:\n raise RuntimeError('Please install marshmallow and apispec')\n\n return flasgger.utils.apispec_to_template(\n app,\n self,\n definitions=definitions,\n paths=paths\n )"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef dispatch_request(self, *args, **kwargs):\n if self.validation:\n specs = {}\n attrs = flasgger.constants.OPTIONAL_FIELDS + [\n 'parameters', 'definitions', 'responses',\n 'summary', 'description'\n ]\n for attr in attrs:\n specs[attr] = getattr(self, attr)\n definitions = {}\n specs.update(convert_schemas(specs, definitions))\n specs['definitions'] = definitions\n flasgger.utils.validate(\n specs=specs, validation_function=self.validation_function)\n return super(SwaggerView, self).dispatch_request(*args, **kwargs)", "response": "Dispatches the request to the appropriate handler."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef merge_specs(target, source):\n\n for key, value in source.items():\n if isinstance(value, dict):\n node = target.setdefault(key, {})\n merge_specs(node, value)\n elif isinstance(value, list):\n node = target.setdefault(key, [])\n node.extend(value)\n else:\n target[key] = value", "response": "Merge the target dictionary with values from the source recursively."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef swag_from(\n specs=None, filetype=None, endpoint=None, methods=None,\n validation=False, schema_id=None, data=None, definition=None,\n validation_function=None, validation_error_handler=None):\n \"\"\"\n Takes a filename.yml, a dictionary or object and loads swagger specs.\n\n :param specs: a filepath, a dictionary or an object\n :param filetype: yml or yaml (json and py to be implemented)\n :param endpoint: endpoint to build definition name\n :param methods: method to build method based specs\n :param validation: perform validation?\n :param schema_id: Definition id ot name to use for validation\n :param data: data to validate (default is request.json)\n :param definition: alias to schema_id\n :param validation_function:\n custom validation function which takes the positional\n arguments: data to be validated at first and schema to validate\n against at second\n :param validation_error_handler: custom function to handle\n exceptions thrown when validating which takes the exception\n thrown as the first, the data being validated as the second and\n the schema being used to validate as the third argument\n \"\"\"\n\n def resolve_path(function, filepath):\n if not filepath.startswith('/'):\n if not hasattr(function, 'root_path'):\n function.root_path = get_root_path(function)\n res = os.path.join(function.root_path, filepath)\n return res\n return filepath\n\n def set_from_filepath(function):\n final_filepath = resolve_path(function, specs)\n function.swag_type = filetype or specs.split('.')[-1]\n\n if endpoint or methods:\n if not hasattr(function, 'swag_paths'):\n function.swag_paths = {}\n\n if not endpoint and not methods:\n function.swag_path = final_filepath\n elif endpoint and methods:\n for verb in methods:\n key = \"{}_{}\".format(endpoint, verb.lower())\n function.swag_paths[key] = final_filepath\n elif endpoint and not methods:\n function.swag_paths[endpoint] = final_filepath\n elif methods and not endpoint:\n for verb in methods:\n function.swag_paths[verb.lower()] = final_filepath\n\n def set_from_specs_dict(function):\n function.specs_dict = specs\n\n def decorator(function):\n\n if isinstance(specs, string_types):\n set_from_filepath(function)\n # function must have or a single swag_path or a list of them\n swag_path = getattr(function, 'swag_path', None)\n swag_paths = getattr(function, 'swag_paths', None)\n validate_args = {\n 'filepath': swag_path or swag_paths,\n 'root': getattr(function, 'root_path', None)\n }\n if isinstance(specs, dict):\n set_from_specs_dict(function)\n validate_args = {'specs': specs}\n\n @wraps(function)\n def wrapper(*args, **kwargs):\n if validation is True:\n validate(\n data,\n schema_id or definition,\n validation_function=validation_function,\n validation_error_handler=validation_error_handler,\n **validate_args\n )\n return function(*args, **kwargs)\n return wrapper\n\n return decorator", "response": "Creates a new swagger specification from a file."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nconverting an APISpec object in to a flasgger template", "response": "def apispec_to_template(app, spec, definitions=None, paths=None):\n \"\"\"\n Converts apispec object in to flasgger definitions template\n :param app: Current app\n :param spec: apispec.APISpec\n :param definitions: a list of [Schema, ..] or [('Name', Schema), ..]\n :param paths: A list of flask views\n \"\"\"\n definitions = definitions or []\n paths = paths or []\n spec_dict = spec.to_dict()\n\n with app.app_context():\n for definition in definitions:\n if isinstance(definition, (tuple, list)):\n name, schema = definition\n else:\n schema = definition\n name = schema.__name__.replace('Schema', '')\n\n spec.components.schema(name, schema=schema)\n\n for path in paths:\n spec.path(view=path)\n\n ret = ordered_dict_to_dict(spec_dict)\n return ret"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef ordered_dict_to_dict(d):\n ret = {}\n new_d = deepcopy(d)\n for k, v in new_d.items():\n if isinstance(v, OrderedDict):\n v = dict(v)\n if isinstance(v, dict):\n v = ordered_dict_to_dict(v)\n ret[k] = v\n return ret", "response": "Converts inner OrderedDict to bare dict"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef pathify(basenames, examples_dir=\"examples/\"): # pragma: no cover\n example = examples_dir.replace(\"/\", \".\")\n return [example + basename for basename in basenames]", "response": "Convert a list of filenames to a list of filenames."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nget all examples modules from examples_dir", "response": "def get_examples(examples_dir=\"examples/\"): # pragma: no cover\n \"\"\"All example modules\"\"\"\n all_files = os.listdir(examples_dir)\n python_files = [f for f in all_files if is_python_file(f)]\n basenames = [remove_suffix(f) for f in python_files]\n modules = [import_module(module) for module in pathify(basenames)]\n return [\n module for module in modules\n if getattr(module, 'app', None) is not None\n ]"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef get_path_from_doc(full_doc):\n swag_path = full_doc.replace('file:', '').strip()\n swag_type = swag_path.split('.')[-1]\n return swag_path, swag_type", "response": "Get the path and type of the swag from the full_doc."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef load_from_file(swag_path, swag_type='yml', root_path=None):\n if swag_type not in ('yaml', 'yml'):\n raise AttributeError(\"Currently only yaml or yml supported\")\n # TODO: support JSON\n\n try:\n enc = detect_by_bom(swag_path)\n with codecs.open(swag_path, encoding=enc) as yaml_file:\n return yaml_file.read()\n except IOError:\n # not in the same dir, add dirname\n swag_path = os.path.join(\n root_path or os.path.dirname(__file__), swag_path\n )\n try:\n enc = detect_by_bom(swag_path)\n with codecs.open(swag_path, encoding=enc) as yaml_file:\n return yaml_file.read()\n except IOError: # pragma: no cover\n # if package dir\n # see https://github.com/rochacbruno/flasgger/pull/104\n # Still not able to reproduce this case\n # test are in examples/package_example\n # need more detail on how to reproduce IOError here\n swag_path = swag_path.replace(\"/\", os.sep).replace(\"\\\\\", os.sep)\n path = swag_path.replace(\n (root_path or os.path.dirname(__file__)), ''\n ).split(os.sep)[1:]\n site_package = imp.find_module(path[0])[1]\n swag_path = os.path.join(site_package, os.sep.join(path[1:]))\n with open(swag_path) as yaml_file:\n return yaml_file.read()", "response": "Load specs from YAML file"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nparsing the docstring of a method or view.", "response": "def parse_docstring(obj, process_doc, endpoint=None, verb=None):\n \"\"\"\n Gets swag data for method/view docstring\n \"\"\"\n first_line, other_lines, swag = None, None, None\n\n full_doc = None\n swag_path = getattr(obj, 'swag_path', None)\n swag_type = getattr(obj, 'swag_type', 'yml')\n swag_paths = getattr(obj, 'swag_paths', None)\n root_path = get_root_path(obj)\n from_file = False\n\n if swag_path is not None:\n full_doc = load_from_file(swag_path, swag_type)\n from_file = True\n elif swag_paths is not None:\n for key in (\"{}_{}\".format(endpoint, verb), endpoint, verb.lower()):\n if key in swag_paths:\n full_doc = load_from_file(swag_paths[key], swag_type)\n break\n from_file = True\n # TODO: handle multiple root_paths\n # to support `import: ` from multiple places\n else:\n full_doc = inspect.getdoc(obj)\n\n if full_doc:\n\n if full_doc.startswith('file:'):\n if not hasattr(obj, 'root_path'):\n obj.root_path = root_path\n swag_path, swag_type = get_path_from_doc(full_doc)\n doc_filepath = os.path.join(obj.root_path, swag_path)\n full_doc = load_from_file(doc_filepath, swag_type)\n from_file = True\n\n full_doc = parse_imports(full_doc, root_path)\n\n yaml_sep = full_doc.find('---')\n\n if yaml_sep != -1:\n line_feed = full_doc.find('\\n')\n if line_feed != -1:\n first_line = process_doc(full_doc[:line_feed])\n other_lines = process_doc(\n full_doc[line_feed + 1: yaml_sep]\n )\n swag = yaml.load(full_doc[yaml_sep + 4:])\n else:\n if from_file:\n swag = yaml.load(full_doc)\n else:\n first_line = full_doc\n\n return first_line, other_lines, swag"} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ngets file path for object and returns its dirname", "response": "def get_root_path(obj):\n \"\"\"\n Get file path for object and returns its dirname\n \"\"\"\n try:\n filename = os.path.abspath(obj.__globals__['__file__'])\n except (KeyError, AttributeError):\n if getattr(obj, '__wrapped__', None):\n # decorator package has been used in view\n return get_root_path(obj.__wrapped__)\n filename = inspect.getfile(obj)\n return os.path.dirname(filename)"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef parse_definition_docstring(obj, process_doc):\n doc_lines, swag = None, None\n\n full_doc = None\n swag_path = getattr(obj, 'swag_path', None)\n swag_type = getattr(obj, 'swag_type', 'yml')\n\n if swag_path is not None:\n full_doc = load_from_file(swag_path, swag_type)\n else:\n full_doc = inspect.getdoc(obj)\n\n if full_doc:\n\n if full_doc.startswith('file:'):\n if not hasattr(obj, 'root_path'):\n obj.root_path = get_root_path(obj)\n swag_path, swag_type = get_path_from_doc(full_doc)\n doc_filepath = os.path.join(obj.root_path, swag_path)\n full_doc = load_from_file(doc_filepath, swag_type)\n\n yaml_sep = full_doc.find('---')\n if yaml_sep != -1:\n doc_lines = process_doc(\n full_doc[:yaml_sep - 1]\n )\n swag = yaml.load(full_doc[yaml_sep:])\n else:\n doc_lines = process_doc(full_doc)\n\n return doc_lines, swag", "response": "Parses the docstring of a class based definition."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef parse_imports(full_doc, root_path=None):\n regex = re.compile('import: \"(.*)\"')\n import_prop = regex.search(full_doc)\n if import_prop:\n start = import_prop.start()\n spaces_num = start - full_doc.rfind('\\n', 0, start) - 1\n filepath = import_prop.group(1)\n if filepath.startswith('/'):\n imported_doc = load_from_file(filepath)\n else:\n imported_doc = load_from_file(filepath, root_path=root_path)\n indented_imported_doc = imported_doc.replace(\n '\\n', '\\n' + ' ' * spaces_num\n )\n full_doc = regex.sub(indented_imported_doc, full_doc, count=1)\n return parse_imports(full_doc)\n return full_doc", "response": "Parses the full docstring and returns a list of all the imports that are present in the document."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef extract_definitions(alist, level=None, endpoint=None, verb=None,\n prefix_ids=False):\n \"\"\"\n Since we couldn't be bothered to register models elsewhere\n our definitions need to be extracted from the parameters.\n We require an 'id' field for the schema to be correctly\n added to the definitions list.\n \"\"\"\n endpoint = endpoint or request.endpoint.lower()\n verb = verb or request.method.lower()\n endpoint = endpoint.replace('.', '_')\n\n def _extract_array_defs(source):\n \"\"\"\n Extracts definitions identified by `id`\n \"\"\"\n # extract any definitions that are within arrays\n # this occurs recursively\n ret = []\n items = source.get('items')\n if items is not None and 'schema' in items:\n ret += extract_definitions(\n [items], level + 1, endpoint, verb, prefix_ids)\n return ret\n\n # for tracking level of recursion\n if level is None:\n level = 0\n\n defs = list()\n for item in alist:\n if not getattr(item, 'get'):\n raise RuntimeError('definitions must be a list of dicts')\n schema = item.get(\"schema\")\n if schema is not None:\n schema_id = schema.get(\"id\")\n if schema_id is not None:\n # add endpoint_verb to schema id to avoid conflicts\n if prefix_ids:\n schema['id'] = schema_id = \"{}_{}_{}\".format(\n endpoint, verb, schema_id\n )\n # ^ api['SWAGGER']['prefix_ids'] = True\n # ... for backwards compatibility with <= 0.5.14\n\n defs.append(schema)\n ref = {\"$ref\": \"#/definitions/{}\".format(schema_id)}\n # only add the reference as a schema if we are in a\n # response or a parameter i.e. at the top level\n # directly ref if a definition is used within another\n # definition\n if level == 0:\n item['schema'] = ref\n else:\n item.update(ref)\n del item['schema']\n\n # extract any definitions that are within properties\n # this occurs recursively\n properties = schema.get('properties')\n if properties is not None:\n defs += extract_definitions(\n properties.values(), level + 1, endpoint, verb, prefix_ids)\n\n defs += _extract_array_defs(schema)\n\n defs += _extract_array_defs(item)\n\n return defs", "response": "Extract the definitions from a list of dicts."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef is_valid_method_view(endpoint):\n klass = endpoint.__dict__.get('view_class', None)\n try:\n return issubclass(klass, MethodView)\n except TypeError:\n return False", "response": "Return True if obj is a valid MethodView"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_vendor_extension_fields(mapping):\n return {k: v for k, v in mapping.items() if k.startswith('x-')}", "response": "Identify vendor extension fields and extract them into a new dictionary."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef login():\n try:\n username = request.form.get(\"username\")\n password = request.form.get(\"password\")\n\n user = authenticate(username, password)\n if not user:\n raise Exception(\"User not found!\")\n\n resp = jsonify({\"message\": \"User authenticated\"})\n resp.status_code = 200\n\n access_token = jwt.jwt_encode_callback(user)\n\n # add token to response headers - so SwaggerUI can use it\n resp.headers.extend({'jwt-token': access_token})\n\n except Exception as e:\n resp = jsonify({\"message\": \"Bad username and/or password\"})\n resp.status_code = 401\n\n return resp", "response": "User authenticate method.\n ---\n description: Authenticate user with supplied credentials.\n parameters:\n - name: username\n in: formData\n type: string\n required: true\n - name: password\n in: formData\n type: string\n required: true\n responses:\n 200:\n description: User successfully logged in.\n 400:\n description: User login failed."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn the data under the API endpoint.", "response": "def get(self):\n \"\"\"\n The data under /apidocs\n json or Swagger UI\n \"\"\"\n base_endpoint = self.config.get('endpoint', 'flasgger')\n specs = [\n {\n \"url\": url_for(\".\".join((base_endpoint, spec['endpoint']))),\n \"title\": spec.get('title', 'API Spec 1'),\n \"version\": spec.get(\"version\", '0.0.1'),\n \"endpoint\": spec.get('endpoint')\n }\n for spec in self.config.get('specs', [])\n ]\n data = {\n \"specs\": specs,\n \"title\": self.config.get('title', 'Flasgger')\n }\n if request.args.get('json'):\n # calling with ?json returns specs\n return jsonify(data)\n else: # pragma: no cover\n data['flasgger_config'] = self.config\n data['json'] = json\n data['flasgger_version'] = __version__\n data['favicon'] = self.config.get(\n 'favicon',\n url_for('flasgger.static', filename='favicon-32x32.png')\n )\n data['swagger_ui_bundle_js'] = self.config.get(\n 'swagger_ui_bundle_js',\n url_for('flasgger.static', filename='swagger-ui-bundle.js')\n )\n data['swagger_ui_standalone_preset_js'] = self.config.get(\n 'swagger_ui_standalone_preset_js',\n url_for('flasgger.static',\n filename='swagger-ui-standalone-preset.js')\n )\n data['jquery_js'] = self.config.get(\n 'jquery_js',\n url_for('flasgger.static', filename='lib/jquery.min.js')\n )\n data['swagger_ui_css'] = self.config.get(\n 'swagger_ui_css',\n url_for('flasgger.static', filename='swagger-ui.css')\n )\n return render_template(\n 'flasgger/index.html',\n **data\n )"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef init_app(self, app, decorators=None):\n self.decorators = decorators or self.decorators\n self.app = app\n\n self.load_config(app)\n # self.load_apispec(app)\n if self.template_file is not None:\n self.template = self.load_swagger_file(self.template_file)\n self.register_views(app)\n self.add_headers(app)\n\n if self.parse:\n if RequestParser is None:\n raise RuntimeError('Please install flask_restful')\n self.parsers = {}\n self.schemas = {}\n self.format_checker = jsonschema.FormatChecker()\n self.parse_request(app)\n\n self._configured = True\n app.swag = self", "response": "Initialize the app with the given flask app"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns all werkzeug rules that match the given filter.", "response": "def get_url_mappings(self, rule_filter=None):\n \"\"\"\n Returns all werkzeug rules\n \"\"\"\n rule_filter = rule_filter or (lambda rule: True)\n app_rules = [\n rule for rule in current_app.url_map.iter_rules()\n if rule_filter(rule)\n ]\n return app_rules"} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nuse for class based definitions", "response": "def get_def_models(self, definition_filter=None):\n \"\"\"\n Used for class based definitions\n \"\"\"\n model_filter = definition_filter or (lambda tag: True)\n return {\n definition.name: definition.obj\n for definition in self.definition_models\n if model_filter(definition)\n }"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef definition(self, name, tags=None):\n def wrapper(obj):\n self.definition_models.append(SwaggerDefinition(name, obj,\n tags=tags))\n return obj\n return wrapper", "response": "Decorator to add a new definition to the list of related resources."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef add_headers(self, app):\n @app.after_request\n def after_request(response): # noqa\n for header, value in self.config.get('headers'):\n response.headers[header] = value\n return response", "response": "Inject headers after request\n "} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef validate(\n self, schema_id, validation_function=None,\n validation_error_handler=None):\n \"\"\"\n A decorator that is used to validate incoming requests data\n against a schema\n\n swagger = Swagger(app)\n\n @app.route('/pets', methods=['POST'])\n @swagger.validate('Pet')\n @swag_from(\"pet_post_endpoint.yml\")\n def post():\n return db.insert(request.data)\n\n This annotation only works if the endpoint is already swagged,\n i.e. placing @swag_from above @validate or not declaring the\n swagger specifications in the method's docstring *won't work*\n\n Naturally, if you use @app.route annotation it still needs to\n be the outermost annotation\n\n :param schema_id: the id of the schema with which the data will\n be validated\n\n :param validation_function: custom validation function which\n takes the positional arguments: data to be validated at\n first and schema to validate against at second\n\n :param validation_error_handler: custom function to handle\n exceptions thrown when validating which takes the exception\n thrown as the first, the data being validated as the second\n and the schema being used to validate as the third argument\n \"\"\"\n\n if validation_function is None:\n validation_function = self.validation_function\n\n if validation_error_handler is None:\n validation_error_handler = self.validation_error_handler\n\n def decorator(func):\n @wraps(func)\n def wrapper(*args, **kwargs):\n specs = get_schema_specs(schema_id, self)\n validate(\n schema_id=schema_id, specs=specs,\n validation_function=validation_function,\n validation_error_handler=validation_error_handler)\n return func(*args, **kwargs)\n\n return wrapper\n\n return decorator", "response": "A decorator that validates incoming data against a schema"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef put(self, todo_id):\n args = parser.parse_args()\n task = {'task': args['task']}\n TODOS[todo_id] = task\n return task, 201", "response": "This is an example\n ."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef colors(palette):\n all_colors = {\n 'cmyk': ['cian', 'magenta', 'yellow', 'black'],\n 'rgb': ['red', 'green', 'blue']\n }\n if palette == 'all':\n result = all_colors\n else:\n result = {palette: all_colors.get(palette)}\n\n return jsonify(result)", "response": "This endpoint returns a list of colors by palette"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nscaling the image to a given scale.", "response": "def __scale_image(image, scale: float):\n \"\"\"\n Scales the image to a given scale.\n :param image:\n :param scale:\n :return:\n \"\"\"\n height, width, _ = image.shape\n\n width_scaled = int(np.ceil(width * scale))\n height_scaled = int(np.ceil(height * scale))\n\n im_data = cv2.resize(image, (width_scaled, height_scaled), interpolation=cv2.INTER_AREA)\n\n # Normalize the image's pixels\n im_data_normalized = (im_data - 127.5) * 0.0078125\n\n return im_data_normalized"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning the non - maximum suppression of the set of entries in the given boxes.", "response": "def __nms(boxes, threshold, method):\n \"\"\"\n Non Maximum Suppression.\n\n :param boxes: np array with bounding boxes.\n :param threshold:\n :param method: NMS method to apply. Available values ('Min', 'Union')\n :return:\n \"\"\"\n if boxes.size == 0:\n return np.empty((0, 3))\n\n x1 = boxes[:, 0]\n y1 = boxes[:, 1]\n x2 = boxes[:, 2]\n y2 = boxes[:, 3]\n s = boxes[:, 4]\n\n area = (x2 - x1 + 1) * (y2 - y1 + 1)\n sorted_s = np.argsort(s)\n\n pick = np.zeros_like(s, dtype=np.int16)\n counter = 0\n while sorted_s.size > 0:\n i = sorted_s[-1]\n pick[counter] = i\n counter += 1\n idx = sorted_s[0:-1]\n\n xx1 = np.maximum(x1[i], x1[idx])\n yy1 = np.maximum(y1[i], y1[idx])\n xx2 = np.minimum(x2[i], x2[idx])\n yy2 = np.minimum(y2[i], y2[idx])\n\n w = np.maximum(0.0, xx2 - xx1 + 1)\n h = np.maximum(0.0, yy2 - yy1 + 1)\n\n inter = w * h\n\n if method is 'Min':\n o = inter / np.minimum(area[i], area[idx])\n else:\n o = inter / (area[i] + area[idx] - inter)\n\n sorted_s = sorted_s[np.where(o <= threshold)]\n\n pick = pick[0:counter]\n\n return pick"} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ndetects bounding boxes from the specified image.", "response": "def detect_faces(self, img) -> list:\n \"\"\"\n Detects bounding boxes from the specified image.\n :param img: image to process\n :return: list containing all the bounding boxes detected with their keypoints.\n \"\"\"\n if img is None or not hasattr(img, \"shape\"):\n raise InvalidImage(\"Image not valid.\")\n\n height, width, _ = img.shape\n stage_status = StageStatus(width=width, height=height)\n\n m = 12 / self.__min_face_size\n min_layer = np.amin([height, width]) * m\n\n scales = self.__compute_scale_pyramid(m, min_layer)\n\n stages = [self.__stage1, self.__stage2, self.__stage3]\n result = [scales, stage_status]\n\n # We pipe here each of the stages\n for stage in stages:\n result = stage(img, result[0], result[1])\n\n [total_boxes, points] = result\n\n bounding_boxes = []\n\n for bounding_box, keypoints in zip(total_boxes, points.T):\n\n bounding_boxes.append({\n 'box': [int(bounding_box[0]), int(bounding_box[1]),\n int(bounding_box[2]-bounding_box[0]), int(bounding_box[3]-bounding_box[1])],\n 'confidence': bounding_box[-1],\n 'keypoints': {\n 'left_eye': (int(keypoints[0]), int(keypoints[5])),\n 'right_eye': (int(keypoints[1]), int(keypoints[6])),\n 'nose': (int(keypoints[2]), int(keypoints[7])),\n 'mouth_left': (int(keypoints[3]), int(keypoints[8])),\n 'mouth_right': (int(keypoints[4]), int(keypoints[9])),\n }\n }\n )\n\n return bounding_boxes"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef __stage1(self, image, scales: list, stage_status: StageStatus):\n total_boxes = np.empty((0, 9))\n status = stage_status\n\n for scale in scales:\n scaled_image = self.__scale_image(image, scale)\n\n img_x = np.expand_dims(scaled_image, 0)\n img_y = np.transpose(img_x, (0, 2, 1, 3))\n\n out = self.__pnet.feed(img_y)\n\n out0 = np.transpose(out[0], (0, 2, 1, 3))\n out1 = np.transpose(out[1], (0, 2, 1, 3))\n\n boxes, _ = self.__generate_bounding_box(out1[0, :, :, 1].copy(),\n out0[0, :, :, :].copy(), scale, self.__steps_threshold[0])\n\n # inter-scale nms\n pick = self.__nms(boxes.copy(), 0.5, 'Union')\n if boxes.size > 0 and pick.size > 0:\n boxes = boxes[pick, :]\n total_boxes = np.append(total_boxes, boxes, axis=0)\n\n numboxes = total_boxes.shape[0]\n\n if numboxes > 0:\n pick = self.__nms(total_boxes.copy(), 0.7, 'Union')\n total_boxes = total_boxes[pick, :]\n\n regw = total_boxes[:, 2] - total_boxes[:, 0]\n regh = total_boxes[:, 3] - total_boxes[:, 1]\n\n qq1 = total_boxes[:, 0] + total_boxes[:, 5] * regw\n qq2 = total_boxes[:, 1] + total_boxes[:, 6] * regh\n qq3 = total_boxes[:, 2] + total_boxes[:, 7] * regw\n qq4 = total_boxes[:, 3] + total_boxes[:, 8] * regh\n\n total_boxes = np.transpose(np.vstack([qq1, qq2, qq3, qq4, total_boxes[:, 4]]))\n total_boxes = self.__rerec(total_boxes.copy())\n\n total_boxes[:, 0:4] = np.fix(total_boxes[:, 0:4]).astype(np.int32)\n status = StageStatus(self.__pad(total_boxes.copy(), stage_status.width, stage_status.height),\n width=stage_status.width, height=stage_status.height)\n\n return total_boxes, status", "response": "This function is the first stage of the MTCNN."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nsecond stage of the MTCNN.", "response": "def __stage2(self, img, total_boxes, stage_status:StageStatus):\n \"\"\"\n Second stage of the MTCNN.\n :param img:\n :param total_boxes:\n :param stage_status:\n :return:\n \"\"\"\n\n num_boxes = total_boxes.shape[0]\n if num_boxes == 0:\n return total_boxes, stage_status\n\n # second stage\n tempimg = np.zeros(shape=(24, 24, 3, num_boxes))\n\n for k in range(0, num_boxes):\n tmp = np.zeros((int(stage_status.tmph[k]), int(stage_status.tmpw[k]), 3))\n\n tmp[stage_status.dy[k] - 1:stage_status.edy[k], stage_status.dx[k] - 1:stage_status.edx[k], :] = \\\n img[stage_status.y[k] - 1:stage_status.ey[k], stage_status.x[k] - 1:stage_status.ex[k], :]\n\n if tmp.shape[0] > 0 and tmp.shape[1] > 0 or tmp.shape[0] == 0 and tmp.shape[1] == 0:\n tempimg[:, :, :, k] = cv2.resize(tmp, (24, 24), interpolation=cv2.INTER_AREA)\n\n else:\n return np.empty(shape=(0,)), stage_status\n\n\n tempimg = (tempimg - 127.5) * 0.0078125\n tempimg1 = np.transpose(tempimg, (3, 1, 0, 2))\n\n out = self.__rnet.feed(tempimg1)\n\n out0 = np.transpose(out[0])\n out1 = np.transpose(out[1])\n\n score = out1[1, :]\n\n ipass = np.where(score > self.__steps_threshold[1])\n\n total_boxes = np.hstack([total_boxes[ipass[0], 0:4].copy(), np.expand_dims(score[ipass].copy(), 1)])\n\n mv = out0[:, ipass[0]]\n\n if total_boxes.shape[0] > 0:\n pick = self.__nms(total_boxes, 0.7, 'Union')\n total_boxes = total_boxes[pick, :]\n total_boxes = self.__bbreg(total_boxes.copy(), np.transpose(mv[:, pick]))\n total_boxes = self.__rerec(total_boxes.copy())\n\n return total_boxes, stage_status"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nthird stage of the MTCNN. :param img: :param total_boxes: :param stage_status: :return:", "response": "def __stage3(self, img, total_boxes, stage_status: StageStatus):\n \"\"\"\n Third stage of the MTCNN.\n\n :param img:\n :param total_boxes:\n :param stage_status:\n :return:\n \"\"\"\n num_boxes = total_boxes.shape[0]\n if num_boxes == 0:\n return total_boxes, np.empty(shape=(0,))\n\n total_boxes = np.fix(total_boxes).astype(np.int32)\n\n status = StageStatus(self.__pad(total_boxes.copy(), stage_status.width, stage_status.height),\n width=stage_status.width, height=stage_status.height)\n\n tempimg = np.zeros((48, 48, 3, num_boxes))\n\n for k in range(0, num_boxes):\n\n tmp = np.zeros((int(status.tmph[k]), int(status.tmpw[k]), 3))\n\n tmp[status.dy[k] - 1:status.edy[k], status.dx[k] - 1:status.edx[k], :] = \\\n img[status.y[k] - 1:status.ey[k], status.x[k] - 1:status.ex[k], :]\n\n if tmp.shape[0] > 0 and tmp.shape[1] > 0 or tmp.shape[0] == 0 and tmp.shape[1] == 0:\n tempimg[:, :, :, k] = cv2.resize(tmp, (48, 48), interpolation=cv2.INTER_AREA)\n else:\n return np.empty(shape=(0,)), np.empty(shape=(0,))\n\n tempimg = (tempimg - 127.5) * 0.0078125\n tempimg1 = np.transpose(tempimg, (3, 1, 0, 2))\n\n out = self.__onet.feed(tempimg1)\n out0 = np.transpose(out[0])\n out1 = np.transpose(out[1])\n out2 = np.transpose(out[2])\n\n score = out2[1, :]\n\n points = out1\n\n ipass = np.where(score > self.__steps_threshold[2])\n\n points = points[:, ipass[0]]\n\n total_boxes = np.hstack([total_boxes[ipass[0], 0:4].copy(), np.expand_dims(score[ipass].copy(), 1)])\n\n mv = out0[:, ipass[0]]\n\n w = total_boxes[:, 2] - total_boxes[:, 0] + 1\n h = total_boxes[:, 3] - total_boxes[:, 1] + 1\n\n points[0:5, :] = np.tile(w, (5, 1)) * points[0:5, :] + np.tile(total_boxes[:, 0], (5, 1)) - 1\n points[5:10, :] = np.tile(h, (5, 1)) * points[5:10, :] + np.tile(total_boxes[:, 1], (5, 1)) - 1\n\n if total_boxes.shape[0] > 0:\n total_boxes = self.__bbreg(total_boxes.copy(), np.transpose(mv))\n pick = self.__nms(total_boxes.copy(), 0.7, 'Min')\n total_boxes = total_boxes[pick, :]\n points = points[:, pick]\n\n return total_boxes, points"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncreating a TF variable with the given name and shape.", "response": "def __make_var(self, name: str, shape: list):\n \"\"\"\n Creates a tensorflow variable with the given name and shape.\n :param name: name to set for the variable.\n :param shape: list defining the shape of the variable.\n :return: created TF variable.\n \"\"\"\n return tf.get_variable(name, shape, trainable=self.__network.is_trainable())"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef new_feed(self, name: str, layer_shape: tuple):\n\n feed_data = tf.placeholder(tf.float32, layer_shape, 'input')\n self.__network.add_layer(name, layer_output=feed_data)", "response": "Creates a new feed layer."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncreating a convolution layer for the current node.", "response": "def new_conv(self, name: str, kernel_size: tuple, channels_output: int,\n stride_size: tuple, padding: str='SAME',\n group: int=1, biased: bool=True, relu: bool=True, input_layer_name: str=None):\n \"\"\"\n Creates a convolution layer for the network.\n :param name: name for the layer\n :param kernel_size: tuple containing the size of the kernel (Width, Height)\n :param channels_output: \u00bf? Perhaps number of channels in the output? it is used as the bias size.\n :param stride_size: tuple containing the size of the stride (Width, Height)\n :param padding: Type of padding. Available values are: ('SAME', 'VALID')\n :param group: groups for the kernel operation. More info required.\n :param biased: boolean flag to set if biased or not.\n :param relu: boolean flag to set if ReLu should be applied at the end of the layer or not.\n :param input_layer_name: name of the input layer for this layer. If None, it will take the last added layer of\n the network.\n \"\"\"\n\n # Verify that the padding is acceptable\n self.__validate_padding(padding)\n\n input_layer = self.__network.get_layer(input_layer_name)\n\n # Get the number of channels in the input\n channels_input = int(input_layer.get_shape()[-1])\n\n # Verify that the grouping parameter is valid\n self.__validate_grouping(channels_input, channels_output, group)\n\n # Convolution for a given input and kernel\n convolve = lambda input_val, kernel: tf.nn.conv2d(input_val, kernel, [1, stride_size[1], stride_size[0], 1],\n padding=padding)\n\n with tf.variable_scope(name) as scope:\n kernel = self.__make_var('weights', shape=[kernel_size[1], kernel_size[0], channels_input // group, channels_output])\n\n output = convolve(input_layer, kernel)\n\n # Add the biases, if required\n if biased:\n biases = self.__make_var('biases', [channels_output])\n output = tf.nn.bias_add(output, biases)\n\n # Apply ReLU non-linearity, if required\n if relu:\n output = tf.nn.relu(output, name=scope.name)\n\n\n self.__network.add_layer(name, layer_output=output)"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncreating a new prelu layer with the given name and input.", "response": "def new_prelu(self, name: str, input_layer_name: str=None):\n \"\"\"\n Creates a new prelu layer with the given name and input.\n :param name: name for this layer.\n :param input_layer_name: name of the layer that serves as input for this one.\n \"\"\"\n input_layer = self.__network.get_layer(input_layer_name)\n\n with tf.variable_scope(name):\n channels_input = int(input_layer.get_shape()[-1])\n alpha = self.__make_var('alpha', shape=[channels_input])\n output = tf.nn.relu(input_layer) + tf.multiply(alpha, -tf.nn.relu(-input_layer))\n\n self.__network.add_layer(name, layer_output=output)"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncreate a new max pooling layer.", "response": "def new_max_pool(self, name:str, kernel_size: tuple, stride_size: tuple, padding='SAME',\n input_layer_name: str=None):\n \"\"\"\n Creates a new max pooling layer.\n :param name: name for the layer.\n :param kernel_size: tuple containing the size of the kernel (Width, Height)\n :param stride_size: tuple containing the size of the stride (Width, Height)\n :param padding: Type of padding. Available values are: ('SAME', 'VALID')\n :param input_layer_name: name of the input layer for this layer. If None, it will take the last added layer of\n the network.\n \"\"\"\n\n self.__validate_padding(padding)\n\n input_layer = self.__network.get_layer(input_layer_name)\n\n output = tf.nn.max_pool(input_layer,\n ksize=[1, kernel_size[1], kernel_size[0], 1],\n strides=[1, stride_size[1], stride_size[0], 1],\n padding=padding,\n name=name)\n\n self.__network.add_layer(name, layer_output=output)"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef new_fully_connected(self, name: str, output_count: int, relu=True, input_layer_name: str=None):\n\n with tf.variable_scope(name):\n input_layer = self.__network.get_layer(input_layer_name)\n vectorized_input, dimension = self.vectorize_input(input_layer)\n\n weights = self.__make_var('weights', shape=[dimension, output_count])\n biases = self.__make_var('biases', shape=[output_count])\n operation = tf.nn.relu_layer if relu else tf.nn.xw_plus_b\n\n fc = operation(vectorized_input, weights, biases, name=name)\n\n self.__network.add_layer(name, layer_output=fc)", "response": "Creates a new fully connected layer."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncreates a new softmax layer for the current level.", "response": "def new_softmax(self, name, axis, input_layer_name: str=None):\n \"\"\"\n Creates a new softmax layer\n :param name: name to set for the layer\n :param axis:\n :param input_layer_name: name of the input layer for this layer. If None, it will take the last added layer of\n the network.\n \"\"\"\n input_layer = self.__network.get_layer(input_layer_name)\n\n if LooseVersion(tf.__version__) < LooseVersion(\"1.5.0\"):\n max_axis = tf.reduce_max(input_layer, axis, keep_dims=True)\n target_exp = tf.exp(input_layer - max_axis)\n normalize = tf.reduce_sum(target_exp, axis, keep_dims=True)\n else:\n max_axis = tf.reduce_max(input_layer, axis, keepdims=True)\n target_exp = tf.exp(input_layer - max_axis)\n normalize = tf.reduce_sum(target_exp, axis, keepdims=True)\n\n softmax = tf.div(target_exp, normalize, name)\n\n self.__network.add_layer(name, layer_output=softmax)"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nadding a layer to the network.", "response": "def add_layer(self, name: str, layer_output):\n \"\"\"\n Adds a layer to the network.\n :param name: name of the layer to add\n :param layer_output: output layer.\n \"\"\"\n self.__layers[name] = layer_output\n self.__last_layer_name = name"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_layer(self, name: str=None):\n if name is None:\n name = self.__last_layer_name\n\n return self.__layers[name]", "response": "Retrieves the layer by its name."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nsetting the weights values of the network.", "response": "def set_weights(self, weights_values: dict, ignore_missing=False):\n \"\"\"\n Sets the weights values of the network.\n :param weights_values: dictionary with weights for each layer\n \"\"\"\n network_name = self.__class__.__name__.lower()\n\n with tf.variable_scope(network_name):\n for layer_name in weights_values:\n with tf.variable_scope(layer_name, reuse=True):\n for param_name, data in weights_values[layer_name].items():\n try:\n var = tf.get_variable(param_name)\n self._session.run(var.assign(data))\n\n except ValueError:\n if not ignore_missing:\n raise"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef feed(self, image):\n network_name = self.__class__.__name__.lower()\n\n with tf.variable_scope(network_name):\n return self._feed(image)", "response": "Feeds the network with an image\n "} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef generate_pdfa_ps(target_filename, icc='sRGB'):\n if icc == 'sRGB':\n icc_profile = SRGB_ICC_PROFILE\n else:\n raise NotImplementedError(\"Only supporting sRGB\")\n\n # pdfmark must contain the full path to the ICC profile, and pdfmark must be\n # also encoded in ASCII. ocrmypdf can be installed anywhere, including to\n # paths that have a non-ASCII character in the filename. Ghostscript\n # accepts hex-encoded strings and converts them to byte strings, so\n # we encode the path with fsencode() and use the hex representation.\n # UTF-16 not accepted here. (Even though ASCII encodable is the usual case,\n # do this always to avoid making it a rare conditional.)\n bytes_icc_profile = os.fsencode(icc_profile)\n hex_icc_profile = hexlify(bytes_icc_profile)\n icc_profile = '<' + hex_icc_profile.decode('ascii') + '>'\n\n t = Template(pdfa_def_template)\n ps = t.substitute(icc_profile=icc_profile, icc_identifier=icc)\n\n # We should have encoded everything to pure ASCII by this point, and\n # to be safe, only allow ASCII in PostScript\n Path(target_filename).write_text(ps, encoding='ascii')", "response": "Generate a Postscript PDF - A file for Ghostscript PDF - A conversion."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef file_claims_pdfa(filename):\n\n with pikepdf.open(filename) as pdf:\n pdfmeta = pdf.open_metadata()\n if not pdfmeta.pdfa_status:\n return {\n 'pass': False,\n 'output': 'pdf',\n 'conformance': 'No PDF/A metadata in XMP',\n }\n valid_part_conforms = {'1A', '1B', '2A', '2B', '2U', '3A', '3B', '3U'}\n conformance = f'PDF/A-{pdfmeta.pdfa_status}'\n pdfa_dict = {}\n if pdfmeta.pdfa_status in valid_part_conforms:\n pdfa_dict['pass'] = True\n pdfa_dict['output'] = 'pdfa'\n pdfa_dict['conformance'] = conformance\n return pdfa_dict", "response": "Determines if the file claims to be PDF or A compliant."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef name2unicode(name):\n if name in glyphname2unicode:\n return glyphname2unicode[name]\n if name.startswith('g') or name.startswith('a'):\n raise KeyError(name)\n if name.startswith('uni'):\n try:\n return chr(int(name[3:], 16))\n except ValueError: # Not hexadecimal\n raise KeyError(name)\n m = STRIP_NAME.search(name)\n if not m:\n raise KeyError(name)\n return chr(int(m.group(0)))", "response": "Fix pdfminer s name2unicode function"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef is_compatible(self, obj):\n both_unicode_mapped = isinstance(self._text, str) and isinstance(obj._text, str)\n try:\n if both_unicode_mapped:\n return self.rendermode == obj.rendermode\n font0, _ = self._text\n font1, _ = obj._text\n return font0 == font1 and self.rendermode == obj.rendermode\n except (ValueError, AttributeError):\n return False", "response": "Check if characters can be combined into a textline\n "} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef open(cls, path):\n filename = fspath(path)\n with _LeptonicaErrorTrap():\n return cls(lept.pixRead(os.fsencode(filename)))", "response": "Load an image file into a PIX object."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef write_implied_format(self, path, jpeg_quality=0, jpeg_progressive=0):\n filename = fspath(path)\n with _LeptonicaErrorTrap():\n lept.pixWriteImpliedFormat(\n os.fsencode(filename), self._cdata, jpeg_quality, jpeg_progressive\n )", "response": "Write pix to the filename with the extension indicating format."} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncreate a copy of a PIL. Image from this Pix", "response": "def frompil(self, pillow_image):\n \"\"\"Create a copy of a PIL.Image from this Pix\"\"\"\n bio = BytesIO()\n pillow_image.save(bio, format='png', compress_level=1)\n py_buffer = bio.getbuffer()\n c_buffer = ffi.from_buffer(py_buffer)\n with _LeptonicaErrorTrap():\n pix = Pix(lept.pixReadMem(c_buffer, len(c_buffer)))\n return pix"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef topil(self):\n from PIL import Image\n\n # Leptonica manages data in words, so it implicitly does an endian\n # swap. Tell Pillow about this when it reads the data.\n pix = self\n if sys.byteorder == 'little':\n if self.mode == 'RGB':\n raw_mode = 'XBGR'\n elif self.mode == 'RGBA':\n raw_mode = 'ABGR'\n elif self.mode == '1':\n raw_mode = '1;I'\n pix = Pix(lept.pixEndianByteSwapNew(pix._cdata))\n else:\n raw_mode = self.mode\n pix = Pix(lept.pixEndianByteSwapNew(pix._cdata))\n else:\n raw_mode = self.mode # no endian swap needed\n\n size = (pix._cdata.w, pix._cdata.h)\n bytecount = pix._cdata.wpl * 4 * pix._cdata.h\n buf = ffi.buffer(pix._cdata.data, bytecount)\n stride = pix._cdata.wpl * 4\n\n im = Image.frombytes(self.mode, size, buf, 'raw', raw_mode, stride)\n\n return im", "response": "Returns a PIL. Image version of this Pix."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef deskew(self, reduction_factor=0):\n with _LeptonicaErrorTrap():\n return Pix(lept.pixDeskew(self._cdata, reduction_factor))", "response": "Returns the deskewed pix object."} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning the pix object rescaled according to the proportions given.", "response": "def scale(self, scale_xy):\n \"Returns the pix object rescaled according to the proportions given.\"\n with _LeptonicaErrorTrap():\n return Pix(lept.pixScale(self._cdata, scale_xy[0], scale_xy[1]))"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef rotate_orth(self, quads):\n \"Orthographic rotation, quads: 0-3, number of clockwise rotations\"\n with _LeptonicaErrorTrap():\n return Pix(lept.pixRotateOrth(self._cdata, quads))", "response": "Orthographic rotation quads : 0 - 3 number of clockwise rotations"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning a tuple ( deskew angle in degrees confidence value.", "response": "def find_skew(self):\n \"\"\"Returns a tuple (deskew angle in degrees, confidence value).\n\n Returns (None, None) if no angle is available.\n \"\"\"\n with _LeptonicaErrorTrap():\n angle = ffi.new('float *', 0.0)\n confidence = ffi.new('float *', 0.0)\n result = lept.pixFindSkew(self._cdata, angle, confidence)\n if result == 0:\n return (angle[0], confidence[0])\n else:\n return (None, None)"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef remove_colormap(self, removal_type):\n with _LeptonicaErrorTrap():\n return Pix(\n lept.pixRemoveColormapGeneral(self._cdata, removal_type, lept.L_COPY)\n )", "response": "Remove a colormap from the image."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nconverting to PDF data with transcoding", "response": "def generate_pdf_ci_data(self, type_, quality):\n \"Convert to PDF data, with transcoding\"\n p_compdata = ffi.new('L_COMP_DATA **')\n result = lept.pixGenerateCIData(self._cdata, type_, quality, 0, p_compdata)\n if result != 0:\n raise LeptonicaError(\"Generate PDF data failed\")\n return CompressedData(p_compdata[0])"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nopens compressed data without transcoding", "response": "def open(cls, path, jpeg_quality=75):\n \"Open compressed data, without transcoding\"\n filename = fspath(path)\n\n p_compdata = ffi.new('L_COMP_DATA **')\n result = lept.l_generateCIDataForPdf(\n os.fsencode(filename), ffi.NULL, jpeg_quality, p_compdata\n )\n if result != 0:\n raise LeptonicaError(\"CompressedData.open\")\n return CompressedData(p_compdata[0])"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_palette_pdf_string(self):\n \"Returns palette pre-formatted for use in PDF\"\n buflen = len('< ') + len(' rrggbb') * self._cdata.ncolors + len('>')\n buf = ffi.buffer(self._cdata.cmapdatahex, buflen)\n return bytes(buf)", "response": "Returns palette pre - formatted for use in PDF"} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nupdates this page s fonts with a reference to the Glyphless font", "response": "def _update_page_resources(*, page, font, font_key, procset):\n \"\"\"Update this page's fonts with a reference to the Glyphless font\"\"\"\n\n if '/Resources' not in page:\n page['/Resources'] = pikepdf.Dictionary({})\n resources = page['/Resources']\n try:\n fonts = resources['/Font']\n except KeyError:\n fonts = pikepdf.Dictionary({})\n if font_key is not None and font_key not in fonts:\n fonts[font_key] = font\n resources['/Font'] = fonts\n\n # Reassign /ProcSet to one that just lists everything - ProcSet is\n # obsolete and doesn't matter but recommended for old viewer support\n resources['/ProcSet'] = procset"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ninserts the text layer from text page 0 on to pdf_base at page_num.", "response": "def _weave_layers_graft(\n *, pdf_base, page_num, text, font, font_key, procset, rotation, strip_old_text, log\n):\n \"\"\"Insert the text layer from text page 0 on to pdf_base at page_num\"\"\"\n\n log.debug(\"Grafting\")\n if Path(text).stat().st_size == 0:\n return\n\n # This is a pointer indicating a specific page in the base file\n pdf_text = pikepdf.open(text)\n pdf_text_contents = pdf_text.pages[0].Contents.read_bytes()\n\n if not tesseract.has_textonly_pdf():\n # If we don't have textonly_pdf, edit the stream to delete the\n # instruction to draw the image Tesseract generated, which we do not\n # use.\n stream = bytearray(pdf_text_contents)\n pattern = b'/Im1 Do'\n idx = stream.find(pattern)\n stream[idx : (idx + len(pattern))] = b' ' * len(pattern)\n pdf_text_contents = bytes(stream)\n\n base_page = pdf_base.pages.p(page_num)\n\n # The text page always will be oriented up by this stage but the original\n # content may have a rotation applied. Wrap the text stream with a rotation\n # so it will be oriented the same way as the rest of the page content.\n # (Previous versions OCRmyPDF rotated the content layer to match the text.)\n mediabox = [float(pdf_text.pages[0].MediaBox[v]) for v in range(4)]\n wt, ht = mediabox[2] - mediabox[0], mediabox[3] - mediabox[1]\n\n mediabox = [float(base_page.MediaBox[v]) for v in range(4)]\n wp, hp = mediabox[2] - mediabox[0], mediabox[3] - mediabox[1]\n\n translate = pikepdf.PdfMatrix().translated(-wt / 2, -ht / 2)\n untranslate = pikepdf.PdfMatrix().translated(wp / 2, hp / 2)\n # -rotation because the input is a clockwise angle and this formula\n # uses CCW\n rotation = -rotation % 360\n rotate = pikepdf.PdfMatrix().rotated(rotation)\n\n # Because of rounding of DPI, we might get a text layer that is not\n # identically sized to the target page. Scale to adjust. Normally this\n # is within 0.998.\n if rotation in (90, 270):\n wt, ht = ht, wt\n scale_x = wp / wt\n scale_y = hp / ht\n\n log.debug('%r', (scale_x, scale_y))\n scale = pikepdf.PdfMatrix().scaled(scale_x, scale_y)\n\n # Translate the text so it is centered at (0, 0), rotate it there, adjust\n # for a size different between initial and text PDF, then untranslate\n ctm = translate @ rotate @ scale @ untranslate\n\n pdf_text_contents = b'q %s cm\\n' % ctm.encode() + pdf_text_contents + b'\\nQ\\n'\n\n new_text_layer = pikepdf.Stream(pdf_base, pdf_text_contents)\n\n if strip_old_text:\n strip_invisible_text(pdf_base, base_page, log)\n\n base_page.page_contents_add(new_text_layer, prepend=True)\n\n _update_page_resources(\n page=base_page, font=font, font_key=font_key, procset=procset\n )\n pdf_text.close()"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nfinding a font from the filename text into pdf_base", "response": "def _find_font(text, pdf_base):\n \"\"\"Copy a font from the filename text into pdf_base\"\"\"\n\n font, font_key = None, None\n possible_font_names = ('/f-0-0', '/F1')\n try:\n with pikepdf.open(text) as pdf_text:\n try:\n pdf_text_fonts = pdf_text.pages[0].Resources.get('/Font', {})\n except (AttributeError, IndexError, KeyError):\n return None, None\n for f in possible_font_names:\n pdf_text_font = pdf_text_fonts.get(f, None)\n if pdf_text_font is not None:\n font_key = f\n break\n if pdf_text_font:\n font = pdf_base.copy_foreign(pdf_text_font)\n return font, font_key\n except (FileNotFoundError, pikepdf.PdfError):\n # PdfError occurs if a 0-length file is written e.g. due to OCR timeout\n return None, None"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ntraverse the table of contents calling visitor_fn at each node.", "response": "def _traverse_toc(pdf_base, visitor_fn, log):\n \"\"\"\n Walk the table of contents, calling visitor_fn() at each node\n\n The /Outlines data structure is a messy data structure, but rather than\n navigating hierarchically we just track unique nodes. Enqueue nodes when\n we find them, and never visit them again. set() is awesome. We look for\n the two types of object in the table of contents that can be page bookmarks\n and update the page entry.\n\n \"\"\"\n\n visited = set()\n queue = set()\n link_keys = ('/Parent', '/First', '/Last', '/Prev', '/Next')\n\n if not '/Outlines' in pdf_base.root:\n return\n\n queue.add(pdf_base.root.Outlines.objgen)\n while queue:\n objgen = queue.pop()\n visited.add(objgen)\n node = pdf_base.get_object(objgen)\n log.debug('fix toc: exploring outline entries at %r', objgen)\n\n # Enumerate other nodes we could visit from here\n for key in link_keys:\n if key not in node:\n continue\n item = node[key]\n if not item.is_indirect:\n # Direct references are not allowed here, but it's not clear\n # what we should do if we find any. Removing them is an option:\n # node[key] = pdf_base.make_indirect(None)\n continue\n objgen = item.objgen\n if objgen not in visited:\n queue.add(objgen)\n\n if visitor_fn:\n visitor_fn(pdf_base, node, log)"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _fix_toc(pdf_base, pageref_remap, log):\n\n if not pageref_remap:\n return\n\n def remap_dest(dest_node):\n \"\"\"\n Inner helper function: change the objgen for any page from the old we\n invalidated to its new one.\n \"\"\"\n try:\n pageref = dest_node[0]\n if pageref['/Type'] == '/Page' and pageref.objgen in pageref_remap:\n new_objgen = pageref_remap[pageref.objgen]\n dest_node[0] = pdf_base.get_object(new_objgen)\n except (IndexError, TypeError) as e:\n log.warning(\"This file may contain invalid table of contents entries\")\n log.debug(e)\n\n def visit_remap_dest(pdf_base, node, log):\n \"\"\"\n Visitor function to fix ToC entries\n\n Test for the two types of references to pages that can occur in ToCs.\n Both types have the same final format (an indirect reference to the\n target page).\n \"\"\"\n if '/Dest' in node:\n # /Dest reference to another page (old method)\n remap_dest(node['/Dest'])\n elif '/A' in node:\n # /A (action) command set to \"GoTo\" (newer method)\n if '/S' in node['/A'] and node['/A']['/S'] == '/GoTo':\n remap_dest(node['/A']['/D'])\n\n _traverse_toc(pdf_base, visit_remap_dest, log)", "response": "Fix the table of contents entries by re - assigning the new objgen number to the new one."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef weave_layers(infiles, output_file, log, context):\n\n def input_sorter(key):\n try:\n return page_number(key)\n except ValueError:\n return -1\n\n flat_inputs = sorted(flatten_groups(infiles), key=input_sorter)\n groups = groupby(flat_inputs, key=input_sorter)\n\n # Extract first item\n _, basegroup = next(groups)\n base = list(basegroup)[0]\n path_base = Path(base).resolve()\n pdf_base = pikepdf.open(path_base)\n font, font_key, procset = None, None, None\n pdfinfo = context.get_pdfinfo()\n pagerefs = {}\n\n procset = pdf_base.make_indirect(\n pikepdf.Object.parse(b'[ /PDF /Text /ImageB /ImageC /ImageI ]')\n )\n\n replacements = 0\n\n # Iterate rest\n for page_num, layers in groups:\n layers = list(layers)\n log.debug(page_num)\n log.debug(layers)\n\n text = next((ii for ii in layers if ii.endswith('.text.pdf')), None)\n image = next((ii for ii in layers if ii.endswith('.image-layer.pdf')), None)\n\n if text and not font:\n font, font_key = _find_font(text, pdf_base)\n\n replacing = False\n content_rotation = pdfinfo[page_num - 1].rotation\n\n path_image = Path(image).resolve() if image else None\n if path_image is not None and path_image != path_base:\n # We are replacing the old page with a rasterized PDF of the new\n # page\n log.debug(\"Replace\")\n old_objgen = pdf_base.pages[page_num - 1].objgen\n\n with pikepdf.open(image) as pdf_image:\n replacements += 1\n image_page = pdf_image.pages[0]\n pdf_base.pages[page_num - 1] = image_page\n\n # We're adding a new page, which will get a new objgen number pair,\n # so we need to update any references to it. qpdf did not like\n # my attempt to update the old object in place, but that is an\n # option to consider\n pagerefs[old_objgen] = pdf_base.pages[page_num - 1].objgen\n replacing = True\n\n autorotate_correction = context.get_rotation(page_num - 1)\n if replacing:\n content_rotation = autorotate_correction\n text_rotation = autorotate_correction\n text_misaligned = (text_rotation - content_rotation) % 360\n log.debug(\n '%r',\n [text_rotation, autorotate_correction, text_misaligned, content_rotation],\n )\n\n if text and font:\n # Graft the text layer onto this page, whether new or old\n strip_old = context.get_options().redo_ocr\n _weave_layers_graft(\n pdf_base=pdf_base,\n page_num=page_num,\n text=text,\n font=font,\n font_key=font_key,\n rotation=text_misaligned,\n procset=procset,\n strip_old_text=strip_old,\n log=log,\n )\n\n # Correct the rotation if applicable\n pdf_base.pages[page_num - 1].Rotate = (\n content_rotation - autorotate_correction\n ) % 360\n\n if replacements % MAX_REPLACE_PAGES == 0:\n # Periodically save and reload the Pdf object. This will keep a\n # lid on our memory usage for very large files. Attach the font to\n # page 1 even if page 1 doesn't use it, so we have a way to get it\n # back.\n # TODO refactor this to outside the loop\n page0 = pdf_base.pages[0]\n _update_page_resources(\n page=page0, font=font, font_key=font_key, procset=procset\n )\n interim = output_file + f'_working{page_num}.pdf'\n pdf_base.save(interim)\n pdf_base.close()\n\n pdf_base = pikepdf.open(interim)\n procset = pdf_base.pages[0].Resources.ProcSet\n font, font_key = None, None # Reacquire this information\n\n _fix_toc(pdf_base, pagerefs, log)\n pdf_base.save(output_file)\n pdf_base.close()", "response": "This function will take a list of infiles and output a file that contains text layer and image layer changes and apply them to the baseline file."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef numeric(basetype, min_=None, max_=None):\n min_ = basetype(min_) if min_ is not None else None\n max_ = basetype(max_) if max_ is not None else None\n\n def _numeric(string):\n value = basetype(string)\n if min_ is not None and value < min_ or max_ is not None and value > max_:\n msg = \"%r not in valid range %r\" % (string, (min_, max_))\n raise argparse.ArgumentTypeError(msg)\n return value\n\n _numeric.__name__ = basetype.__name__\n return _numeric", "response": "Validator for numeric params"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreplace the elaborate ruffus stack trace with a user friendly description of the error message that occurred.", "response": "def do_ruffus_exception(ruffus_five_tuple, options, log):\n \"\"\"Replace the elaborate ruffus stack trace with a user friendly\n description of the error message that occurred.\"\"\"\n exit_code = None\n\n _task_name, _job_name, exc_name, exc_value, exc_stack = ruffus_five_tuple\n\n if isinstance(exc_name, type):\n # ruffus is full of mystery... sometimes (probably when the process\n # group leader is killed) exc_name is the class object of the exception,\n # rather than a str. So reach into the object and get its name.\n exc_name = exc_name.__name__\n\n if exc_name.startswith('ocrmypdf.exceptions.'):\n base_exc_name = exc_name.replace('ocrmypdf.exceptions.', '')\n exc_class = getattr(ocrmypdf_exceptions, base_exc_name)\n exit_code = getattr(exc_class, 'exit_code', ExitCode.other_error)\n try:\n if isinstance(exc_value, exc_class):\n exc_msg = str(exc_value)\n elif isinstance(exc_value, str):\n exc_msg = exc_value\n else:\n exc_msg = str(exc_class())\n except Exception:\n exc_msg = \"Unknown\"\n\n if exc_name in ('builtins.SystemExit', 'SystemExit'):\n match = re.search(r\"\\.(.+?)\\)\", exc_value)\n exit_code_name = match.groups()[0]\n exit_code = getattr(ExitCode, exit_code_name, 'other_error')\n elif exc_name == 'ruffus.ruffus_exceptions.MissingInputFileError':\n log.error(cleanup_ruffus_error_message(exc_value))\n exit_code = ExitCode.input_file\n elif exc_name in ('builtins.KeyboardInterrupt', 'KeyboardInterrupt'):\n # We have to print in this case because the log daemon might be toast\n print(\"Interrupted by user\", file=sys.stderr)\n exit_code = ExitCode.ctrl_c\n elif exc_name == 'subprocess.CalledProcessError':\n # It's up to the subprocess handler to report something useful\n msg = \"Error occurred while running this command:\"\n log.error(msg + '\\n' + exc_value)\n exit_code = ExitCode.child_process_error\n elif exc_name.startswith('ocrmypdf.exceptions.'):\n if exc_msg:\n log.error(exc_msg)\n elif exc_name == 'PIL.Image.DecompressionBombError':\n msg = cleanup_ruffus_error_message(exc_value)\n msg += (\n \"\\nUse the --max-image-mpixels argument to set increase the \"\n \"maximum number of megapixels to accept.\"\n )\n log.error(msg)\n exit_code = ExitCode.input_file\n\n if exit_code is not None:\n return exit_code\n\n if not options.verbose:\n log.error(exc_stack)\n return ExitCode.other_error"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef traverse_ruffus_exception(exceptions, options, log):\n\n exit_codes = []\n for exc in exceptions:\n exit_code = do_ruffus_exception(exc, options, log)\n exit_codes.append(exit_code)\n\n return exit_codes[0]", "response": "Traverse a RethrownJobError and output the exceptions\n "} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncheck if the streams are closed.", "response": "def check_closed_streams(options):\n \"\"\"Work around Python issue with multiprocessing forking on closed streams\n\n https://bugs.python.org/issue28326\n\n Attempting to a fork/exec a new Python process when any of std{in,out,err}\n are closed or not flushable for some reason may raise an exception.\n Fix this by opening devnull if the handle seems to be closed. Do this\n globally to avoid tracking places all places that fork.\n\n Seems to be specific to multiprocessing.Process not all Python process\n forkers.\n\n The error actually occurs when the stream object is not flushable,\n but replacing an open stream object that is not flushable with\n /dev/null is a bad idea since it will create a silent failure. Replacing\n a closed handle with /dev/null seems safe.\n\n \"\"\"\n\n if sys.version_info[0:3] >= (3, 6, 4):\n return True # Issued fixed in Python 3.6.4+\n\n if sys.stderr is None:\n sys.stderr = open(os.devnull, 'w')\n\n if sys.stdin is None:\n if options.input_file == '-':\n print(\"Trying to read from stdin but stdin seems closed\", file=sys.stderr)\n return False\n sys.stdin = open(os.devnull, 'r')\n\n if sys.stdout is None:\n if options.output_file == '-':\n # Can't replace stdout if the user is piping\n # If this case can even happen, it must be some kind of weird\n # stream.\n print(\n textwrap.dedent(\n \"\"\"\\\n Output was set to stdout '-' but the stream attached to\n stdout does not support the flush() system call. This\n will fail.\"\"\"\n ),\n file=sys.stderr,\n )\n return False\n sys.stdout = open(os.devnull, 'w')\n\n return True"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _pdf_guess_version(input_file, search_window=1024):\n\n with open(input_file, 'rb') as f:\n signature = f.read(search_window)\n m = re.search(br'%PDF-(\\d\\.\\d)', signature)\n if m:\n return m.group(1)\n return ''", "response": "Try to find version signature at start of file."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_pageinfo(input_file, context):\n \"Get zero-based page info implied by filename, e.g. 000002.pdf -> 1\"\n pageno = page_number(input_file) - 1\n pageinfo = context.get_pdfinfo()[pageno]\n return pageinfo", "response": "Get zero - based page info implied by filename e. g. 000002. pdf -> 1"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nget the DPI when nonsquare DPI is tolerable", "response": "def get_page_dpi(pageinfo, options):\n \"Get the DPI when nonsquare DPI is tolerable\"\n xres = max(\n pageinfo.xres or VECTOR_PAGE_DPI,\n options.oversample or 0,\n VECTOR_PAGE_DPI if pageinfo.has_vector else 0,\n )\n yres = max(\n pageinfo.yres or VECTOR_PAGE_DPI,\n options.oversample or 0,\n VECTOR_PAGE_DPI if pageinfo.has_vector else 0,\n )\n return (float(xres), float(yres))"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_page_square_dpi(pageinfo, options):\n \"Get the DPI when we require xres == yres, scaled to physical units\"\n xres = pageinfo.xres or 0\n yres = pageinfo.yres or 0\n userunit = pageinfo.userunit or 1\n return float(\n max(\n (xres * userunit) or VECTOR_PAGE_DPI,\n (yres * userunit) or VECTOR_PAGE_DPI,\n VECTOR_PAGE_DPI if pageinfo.has_vector else 0,\n options.oversample or 0,\n )\n )", "response": "Get the DPI when we require xres == yres scaled to physical units"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ngets the DPI when we require xres == yres in Postscript units", "response": "def get_canvas_square_dpi(pageinfo, options):\n \"\"\"Get the DPI when we require xres == yres, in Postscript units\"\"\"\n return float(\n max(\n (pageinfo.xres) or VECTOR_PAGE_DPI,\n (pageinfo.yres) or VECTOR_PAGE_DPI,\n VECTOR_PAGE_DPI if pageinfo.has_vector else 0,\n options.oversample or 0,\n )\n )"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nwork out orientation correct for each page. We ask Ghostscript to draw a preview page, which will rasterize with the current /Rotate applied, and then ask Tesseract which way the page is oriented. If the value of /Rotate is correct (e.g., a user already manually fixed rotation), then Tesseract will say the page is pointing up and the correction is zero. Otherwise, the orientation found by Tesseract represents the clockwise rotation, or the counterclockwise correction to rotation. When we draw the real page for OCR, we rotate it by the CCW correction, which points it (hopefully) upright. _weave.py takes care of the orienting the image and text layers.", "response": "def orient_page(infiles, output_file, log, context):\n \"\"\"\n Work out orientation correct for each page.\n\n We ask Ghostscript to draw a preview page, which will rasterize with the\n current /Rotate applied, and then ask Tesseract which way the page is\n oriented. If the value of /Rotate is correct (e.g., a user already\n manually fixed rotation), then Tesseract will say the page is pointing\n up and the correction is zero. Otherwise, the orientation found by\n Tesseract represents the clockwise rotation, or the counterclockwise\n correction to rotation.\n\n When we draw the real page for OCR, we rotate it by the CCW correction,\n which points it (hopefully) upright. _weave.py takes care of the orienting\n the image and text layers.\n\n \"\"\"\n\n options = context.get_options()\n page_pdf = next(ii for ii in infiles if ii.endswith('.page.pdf'))\n\n if not options.rotate_pages:\n re_symlink(page_pdf, output_file, log)\n return\n preview = next(ii for ii in infiles if ii.endswith('.preview.jpg'))\n\n orient_conf = tesseract.get_orientation(\n preview,\n engine_mode=options.tesseract_oem,\n timeout=options.tesseract_timeout,\n log=log,\n )\n\n direction = {0: '\u21e7', 90: '\u21e8', 180: '\u21e9', 270: '\u21e6'}\n\n pageno = page_number(page_pdf) - 1\n pdfinfo = context.get_pdfinfo()\n existing_rotation = pdfinfo[pageno].rotation\n\n correction = orient_conf.angle % 360\n\n apply_correction = False\n action = ''\n if orient_conf.confidence >= options.rotate_pages_threshold:\n if correction != 0:\n apply_correction = True\n action = ' - will rotate'\n else:\n action = ' - rotation appears correct'\n else:\n if correction != 0:\n action = ' - confidence too low to rotate'\n else:\n action = ' - no change'\n\n facing = ''\n if existing_rotation != 0:\n facing = 'with existing rotation {}, '.format(\n direction.get(existing_rotation, '?')\n )\n facing += 'page is facing {}'.format(direction.get(orient_conf.angle, '?'))\n\n log.info(\n '{pagenum:4d}: {facing}, confidence {conf:.2f}{action}'.format(\n pagenum=page_number(preview),\n facing=facing,\n conf=orient_conf.confidence,\n action=action,\n )\n )\n\n re_symlink(page_pdf, output_file, log)\n if apply_correction:\n context.set_rotation(pageno, correction)"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef select_ocr_image(infiles, output_file, log, context):\n\n image = infiles[0]\n options = context.get_options()\n pageinfo = get_pageinfo(image, context)\n\n with Image.open(image) as im:\n from PIL import ImageColor\n from PIL import ImageDraw\n\n white = ImageColor.getcolor('#ffffff', im.mode)\n # pink = ImageColor.getcolor('#ff0080', im.mode)\n draw = ImageDraw.ImageDraw(im)\n\n xres, yres = im.info['dpi']\n log.debug('resolution %r %r', xres, yres)\n\n if not options.force_ocr:\n # Do not mask text areas when forcing OCR, because we need to OCR\n # all text areas\n mask = None # Exclude both visible and invisible text from OCR\n if options.redo_ocr:\n mask = True # Mask visible text, but not invisible text\n\n for textarea in pageinfo.get_textareas(visible=mask, corrupt=None):\n # Calculate resolution based on the image size and page dimensions\n # without regard whatever resolution is in pageinfo (may differ or\n # be None)\n bbox = [float(v) for v in textarea]\n xscale, yscale = float(xres) / 72.0, float(yres) / 72.0\n pixcoords = [\n bbox[0] * xscale,\n im.height - bbox[3] * yscale,\n bbox[2] * xscale,\n im.height - bbox[1] * yscale,\n ]\n pixcoords = [int(round(c)) for c in pixcoords]\n log.debug('blanking %r', pixcoords)\n draw.rectangle(pixcoords, fill=white)\n # draw.rectangle(pixcoords, outline=pink)\n\n if options.mask_barcodes or options.threshold:\n pix = leptonica.Pix.frompil(im)\n if options.threshold:\n pix = pix.masked_threshold_on_background_norm()\n if options.mask_barcodes:\n barcodes = pix.locate_barcodes()\n for barcode in barcodes:\n decoded, rect = barcode\n log.info('masking barcode %s %r', decoded, rect)\n draw.rectangle(rect, fill=white)\n im = pix.topil()\n\n del draw\n # Pillow requires integer DPI\n dpi = round(xres), round(yres)\n im.save(output_file, dpi=dpi)", "response": "Select the image we send for OCR."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef select_visible_page_image(infiles, output_file, log, context):\n\n options = context.get_options()\n if options.clean_final:\n image_suffix = '.pp-clean.png'\n elif options.deskew:\n image_suffix = '.pp-deskew.png'\n elif options.remove_background:\n image_suffix = '.pp-background.png'\n else:\n image_suffix = '.page.png'\n image = next(ii for ii in infiles if ii.endswith(image_suffix))\n\n pageinfo = get_pageinfo(image, context)\n if pageinfo.images and all(im.enc == 'jpeg' for im in pageinfo.images):\n log.debug(f'{page_number(image):4d}: JPEG input -> JPEG output')\n # If all images were JPEGs originally, produce a JPEG as output\n with Image.open(image) as im:\n # At this point the image should be a .png, but deskew, unpaper\n # might have removed the DPI information. In this case, fall back to\n # square DPI used to rasterize. When the preview image was\n # rasterized, it was also converted to square resolution, which is\n # what we want to give tesseract, so keep it square.\n fallback_dpi = get_page_square_dpi(pageinfo, options)\n dpi = im.info.get('dpi', (fallback_dpi, fallback_dpi))\n\n # Pillow requires integer DPI\n dpi = round(dpi[0]), round(dpi[1])\n im.save(output_file, format='JPEG', dpi=dpi)\n else:\n re_symlink(image, output_file, log)", "response": "Selects a whole page image that we can show the user."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _normalize_stack(graphobjs):\n for operands, operator in graphobjs:\n operator = str(operator)\n if re.match(r'Q*q+$', operator): # Zero or more Q, one or more q\n for char in operator: # Split into individual\n yield ([], char) # Yield individual\n else:\n yield (operands, operator)", "response": "Convert runs of qQ s in the stack into single graphobjs"} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ninterprets the PDF content stream and return a list of the PDF graphics stack.", "response": "def _interpret_contents(contentstream, initial_shorthand=UNIT_SQUARE):\n \"\"\"Interpret the PDF content stream.\n\n The stack represents the state of the PDF graphics stack. We are only\n interested in the current transformation matrix (CTM) so we only track\n this object; a full implementation would need to track many other items.\n\n The CTM is initialized to the mapping from user space to device space.\n PDF units are 1/72\". In a PDF viewer or printer this matrix is initialized\n to the transformation to device space. For example if set to\n (1/72, 0, 0, 1/72, 0, 0) then all units would be calculated in inches.\n\n Images are always considered to be (0, 0) -> (1, 1). Before drawing an\n image there should be a 'cm' that sets up an image coordinate system\n where drawing from (0, 0) -> (1, 1) will draw on the desired area of the\n page.\n\n PDF units suit our needs so we initialize ctm to the identity matrix.\n\n According to the PDF specification, the maximum stack depth is 32. Other\n viewers tolerate some amount beyond this. We issue a warning if the\n stack depth exceeds the spec limit and set a hard limit beyond this to\n bound our memory requirements. If the stack underflows behavior is\n undefined in the spec, but we just pretend nothing happened and leave the\n CTM unchanged.\n \"\"\"\n\n stack = []\n ctm = PdfMatrix(initial_shorthand)\n xobject_settings = []\n inline_images = []\n found_vector = False\n vector_ops = set('S s f F f* B B* b b*'.split())\n image_ops = set('BI ID EI q Q Do cm'.split())\n operator_whitelist = ' '.join(vector_ops | image_ops)\n\n for n, graphobj in enumerate(\n _normalize_stack(\n pikepdf.parse_content_stream(contentstream, operator_whitelist)\n )\n ):\n operands, operator = graphobj\n if operator == 'q':\n stack.append(ctm)\n if len(stack) > 32: # See docstring\n if len(stack) > 128:\n raise RuntimeError(\n \"PDF graphics stack overflowed hard limit, operator %i\" % n\n )\n warn(\"PDF graphics stack overflowed spec limit\")\n elif operator == 'Q':\n try:\n ctm = stack.pop()\n except IndexError:\n # Keeping the ctm the same seems to be the only sensible thing\n # to do. Just pretend nothing happened, keep calm and carry on.\n warn(\"PDF graphics stack underflowed - PDF may be malformed\")\n elif operator == 'cm':\n ctm = PdfMatrix(operands) @ ctm\n elif operator == 'Do':\n image_name = operands[0]\n settings = XobjectSettings(\n name=image_name, shorthand=ctm.shorthand, stack_depth=len(stack)\n )\n xobject_settings.append(settings)\n elif operator == 'INLINE IMAGE': # BI/ID/EI are grouped into this\n iimage = operands[0]\n inline = InlineSettings(\n iimage=iimage, shorthand=ctm.shorthand, stack_depth=len(stack)\n )\n inline_images.append(inline)\n elif operator in vector_ops:\n found_vector = True\n\n return ContentsInfo(\n xobject_settings=xobject_settings,\n inline_images=inline_images,\n found_vector=found_vector,\n )"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ngiving the transformation matrix and image size, find the image DPI. PDFs do not include image resolution information within image data. Instead, the PDF page content stream describes the location where the image will be rasterized, and the effective resolution is the ratio of the pixel size to raster target size. Normally a scanned PDF has the paper size set appropriately but this is not guaranteed. The most common case is a cropped image will change the page size (/CropBox) without altering the page content stream. That means it is not sufficient to assume that the image fills the page, even though that is the most common case. A PDF image may be scaled (always), cropped, translated, rotated in place to an arbitrary angle (rarely) and skewed. Only equal area mappings can be expressed, that is, it is not necessary to consider distortions where the effective DPI varies with position. To determine the image scale, transform an offset axis vector v0 (0, 0), width-axis vector v0 (1, 0), height-axis vector vh (0, 1) with the matrix, which gives the dimensions of the image in PDF units. From there we can compare to actual image dimensions. PDF uses row vector * matrix_tranposed unlike the traditional matrix * column vector. The offset, width and height vectors can be combined in a matrix and multiplied by the transform matrix. Then we want to calculated magnitude(width_vector - offset_vector) and magnitude(height_vector - offset_vector) When the above is worked out algebraically, the effect of translation cancels out, and the vector magnitudes become functions of the nonzero transformation matrix indices. The results of the derivation are used in this code. pdfimages -list does calculate the DPI in some way that is not completely naive, but it does not get the DPI of rotated images right, so cannot be used anymore to validate this. Photoshop works, or using Acrobat to rotate the image back to normal. It does not matter if the image is partially cropped, or even out of the /MediaBox.", "response": "def _get_dpi(ctm_shorthand, image_size):\n \"\"\"Given the transformation matrix and image size, find the image DPI.\n\n PDFs do not include image resolution information within image data.\n Instead, the PDF page content stream describes the location where the\n image will be rasterized, and the effective resolution is the ratio of the\n pixel size to raster target size.\n\n Normally a scanned PDF has the paper size set appropriately but this is\n not guaranteed. The most common case is a cropped image will change the\n page size (/CropBox) without altering the page content stream. That means\n it is not sufficient to assume that the image fills the page, even though\n that is the most common case.\n\n A PDF image may be scaled (always), cropped, translated, rotated in place\n to an arbitrary angle (rarely) and skewed. Only equal area mappings can\n be expressed, that is, it is not necessary to consider distortions where\n the effective DPI varies with position.\n\n To determine the image scale, transform an offset axis vector v0 (0, 0),\n width-axis vector v0 (1, 0), height-axis vector vh (0, 1) with the matrix,\n which gives the dimensions of the image in PDF units. From there we can\n compare to actual image dimensions. PDF uses\n row vector * matrix_tranposed unlike the traditional\n matrix * column vector.\n\n The offset, width and height vectors can be combined in a matrix and\n multiplied by the transform matrix. Then we want to calculated\n magnitude(width_vector - offset_vector)\n and\n magnitude(height_vector - offset_vector)\n\n When the above is worked out algebraically, the effect of translation\n cancels out, and the vector magnitudes become functions of the nonzero\n transformation matrix indices. The results of the derivation are used\n in this code.\n\n pdfimages -list does calculate the DPI in some way that is not completely\n naive, but it does not get the DPI of rotated images right, so cannot be\n used anymore to validate this. Photoshop works, or using Acrobat to\n rotate the image back to normal.\n\n It does not matter if the image is partially cropped, or even out of the\n /MediaBox.\n\n \"\"\"\n\n a, b, c, d, _, _ = ctm_shorthand\n\n # Calculate the width and height of the image in PDF units\n image_drawn_width = hypot(a, b)\n image_drawn_height = hypot(c, d)\n\n # The scale of the image is pixels per unit of default user space (1/72\")\n scale_w = image_size[0] / image_drawn_width\n scale_h = image_size[1] / image_drawn_height\n\n # DPI = scale * 72\n dpi_w = scale_w * 72.0\n dpi_h = scale_h * 72.0\n\n return dpi_w, dpi_h"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nfinding inline images in the contentstream", "response": "def _find_inline_images(contentsinfo):\n \"Find inline images in the contentstream\"\n\n for n, inline in enumerate(contentsinfo.inline_images):\n yield ImageInfo(\n name='inline-%02d' % n, shorthand=inline.shorthand, inline=inline\n )"} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nsearches for all XObject - based images in the container.", "response": "def _image_xobjects(container):\n \"\"\"Search for all XObject-based images in the container\n\n Usually the container is a page, but it could also be a Form XObject\n that contains images. Filter out the Form XObjects which are dealt with\n elsewhere.\n\n Generate a sequence of tuples (image, xobj container), where container,\n where xobj is the name of the object and image is the object itself,\n since the object does not know its own name.\n\n \"\"\"\n\n if '/Resources' not in container:\n return\n resources = container['/Resources']\n if '/XObject' not in resources:\n return\n xobjs = resources['/XObject'].as_dict()\n for xobj in xobjs:\n candidate = xobjs[xobj]\n if not '/Subtype' in candidate:\n continue\n if candidate['/Subtype'] == '/Image':\n pdfimage = candidate\n yield (pdfimage, xobj)"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nfinding images stored in the container that are not part of a regular page.", "response": "def _find_regular_images(container, contentsinfo):\n \"\"\"Find images stored in the container's /Resources /XObject\n\n Usually the container is a page, but it could also be a Form XObject\n that contains images.\n\n Generates images with their DPI at time of drawing.\n \"\"\"\n\n for pdfimage, xobj in _image_xobjects(container):\n\n # For each image that is drawn on this, check if we drawing the\n # current image - yes this is O(n^2), but n == 1 almost always\n for draw in contentsinfo.xobject_settings:\n if draw.name != xobj:\n continue\n\n if draw.stack_depth == 0 and _is_unit_square(draw.shorthand):\n # At least one PDF in the wild (and test suite) draws an image\n # when the graphics stack depth is 0, meaning that the image\n # gets drawn into a square of 1x1 PDF units (or 1/72\",\n # or 0.35 mm). The equivalent DPI will be >100,000. Exclude\n # these from our DPI calculation for the page.\n continue\n\n yield ImageInfo(name=draw.name, pdfimage=pdfimage, shorthand=draw.shorthand)"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nfinding any images that are in Form XObjects in the container.", "response": "def _find_form_xobject_images(pdf, container, contentsinfo):\n \"\"\"Find any images that are in Form XObjects in the container\n\n The container may be a page, or a parent Form XObject.\n\n \"\"\"\n if '/Resources' not in container:\n return\n resources = container['/Resources']\n if '/XObject' not in resources:\n return\n xobjs = resources['/XObject'].as_dict()\n for xobj in xobjs:\n candidate = xobjs[xobj]\n if candidate['/Subtype'] != '/Form':\n continue\n\n form_xobject = candidate\n for settings in contentsinfo.xobject_settings:\n if settings.name != xobj:\n continue\n\n # Find images once for each time this Form XObject is drawn.\n # This could be optimized to cache the multiple drawing events\n # but in practice both Form XObjects and multiple drawing of the\n # same object are both very rare.\n ctm_shorthand = settings.shorthand\n yield from _process_content_streams(\n pdf=pdf, container=form_xobject, shorthand=ctm_shorthand\n )"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nyielding all content streams that are available in the container.", "response": "def _process_content_streams(*, pdf, container, shorthand=None):\n \"\"\"Find all individual instances of images drawn in the container\n\n Usually the container is a page, but it may also be a Form XObject.\n\n On a typical page images are stored inline or as regular images\n in an XObject.\n\n Form XObjects may include inline images, XObject images,\n and recursively, other Form XObjects; and also vector graphic objects.\n\n Every instance of an image being drawn somewhere is flattened and\n treated as a unique image, since if the same image is drawn multiple times\n on one page it may be drawn at differing resolutions, and our objective\n is to find the resolution at which the page can be rastered without\n downsampling.\n\n \"\"\"\n\n if container.get('/Type') == '/Page' and '/Contents' in container:\n initial_shorthand = shorthand or UNIT_SQUARE\n elif container.get('/Type') == '/XObject' and container['/Subtype'] == '/Form':\n # Set the CTM to the state it was when the \"Do\" operator was\n # encountered that is drawing this instance of the Form XObject\n ctm = PdfMatrix(shorthand) if shorthand else PdfMatrix.identity()\n\n # A Form XObject may provide its own matrix to map form space into\n # user space. Get this if one exists\n form_shorthand = container.get('/Matrix', PdfMatrix.identity())\n form_matrix = PdfMatrix(form_shorthand)\n\n # Concatenate form matrix with CTM to ensure CTM is correct for\n # drawing this instance of the XObject\n ctm = form_matrix @ ctm\n initial_shorthand = ctm.shorthand\n else:\n return\n\n contentsinfo = _interpret_contents(container, initial_shorthand)\n\n if contentsinfo.found_vector:\n yield VectorInfo()\n yield from _find_inline_images(contentsinfo)\n yield from _find_regular_images(container, contentsinfo)\n yield from _find_form_xobject_images(pdf, container, contentsinfo)"} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns True if the page has text.", "response": "def _page_has_text(text_blocks, page_width, page_height):\n \"\"\"Smarter text detection that ignores text in margins\"\"\"\n\n pw, ph = float(page_width), float(page_height)\n\n margin_ratio = 0.125\n interior_bbox = (\n margin_ratio * pw, # left\n (1 - margin_ratio) * ph, # top\n (1 - margin_ratio) * pw, # right\n margin_ratio * ph, # bottom (first quadrant: bottom < top)\n )\n\n def rects_intersect(a, b):\n \"\"\"\n Where (a,b) are 4-tuple rects (left-0, top-1, right-2, bottom-3)\n https://stackoverflow.com/questions/306316/determine-if-two-rectangles-overlap-each-other\n Formula assumes all boxes are in first quadrant\n \"\"\"\n return a[0] < b[2] and a[2] > b[0] and a[1] > b[3] and a[3] < b[1]\n\n has_text = False\n for bbox in text_blocks:\n if rects_intersect(bbox, interior_bbox):\n has_text = True\n break\n return has_text"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nextract only limited content from text boxes.", "response": "def simplify_textboxes(miner, textbox_getter):\n \"\"\"Extract only limited content from text boxes\n\n We do this to save memory and ensure that our objects are pickleable.\n \"\"\"\n for box in textbox_getter(miner):\n first_line = box._objs[0]\n first_char = first_line._objs[0]\n\n visible = first_char.rendermode != 3\n corrupt = first_char.get_text() == '\\ufffd'\n yield TextboxInfo(box.bbox, visible, corrupt)"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef page_get_textblocks(infile, pageno, xmltext, height):\n\n root = xmltext\n if not hasattr(xmltext, 'findall'):\n return []\n\n def blocks():\n for span in root.findall('.//span'):\n bbox_str = span.attrib['bbox']\n font_size = span.attrib['size']\n pts = [int(pt) for pt in bbox_str.split()]\n pts[1] = pts[1] - int(float(font_size) + 0.5)\n bbox_topdown = tuple(pts)\n bb = bbox_topdown\n bbox_bottomup = (bb[0], height - bb[3], bb[2], height - bb[1])\n yield bbox_bottomup\n\n def joined_blocks():\n prev = None\n for bbox in blocks():\n if prev is None:\n prev = bbox\n if bbox[1] == prev[1] and bbox[3] == prev[3]:\n gap = prev[2] - bbox[0]\n height = abs(bbox[3] - bbox[1])\n if gap < height:\n # Join boxes\n prev = (prev[0], prev[1], bbox[2], bbox[3])\n continue\n # yield previously joined bboxes and start anew\n yield prev\n prev = bbox\n if prev is not None:\n yield prev\n\n return [block for block in joined_blocks()]", "response": "Get text blocks out of Ghostscript txtwrite xml"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef has_textonly_pdf():\n args_tess = ['tesseract', '--print-parameters', 'pdf']\n params = ''\n try:\n params = check_output(args_tess, universal_newlines=True, stderr=STDOUT)\n except CalledProcessError as e:\n print(\"Could not --print-parameters from tesseract\", file=sys.stderr)\n raise MissingDependencyError from e\n if 'textonly_pdf' in params:\n return True\n return False", "response": "Does Tesseract have textonly_pdf capability?"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ngenerate a. hocr file that reports no text detected on a page that is the same size as the input image.", "response": "def _generate_null_hocr(output_hocr, output_sidecar, image):\n \"\"\"Produce a .hocr file that reports no text detected on a page that is\n the same size as the input image.\"\"\"\n from PIL import Image\n\n im = Image.open(image)\n w, h = im.size\n\n with open(output_hocr, 'w', encoding=\"utf-8\") as f:\n f.write(HOCR_TEMPLATE.format(w, h))\n with open(output_sidecar, 'w', encoding='utf-8') as f:\n f.write('[skipped page]')"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef generate_pdf(\n *,\n input_image,\n skip_pdf=None,\n output_pdf,\n output_text,\n language: list,\n engine_mode,\n text_only: bool,\n tessconfig: list,\n timeout: float,\n pagesegmode: int,\n user_words,\n user_patterns,\n log,\n):\n '''Use Tesseract to render a PDF.\n\n input_image -- image to analyze\n skip_pdf -- if we time out, use this file as output\n output_pdf -- file to generate\n output_text -- OCR text file\n language -- list of languages to consider\n engine_mode -- engine mode argument for tess v4\n text_only -- enable tesseract text only mode?\n tessconfig -- tesseract configuration\n timeout -- timeout (seconds)\n log -- logger object\n '''\n\n args_tesseract = tess_base_args(language, engine_mode)\n\n if pagesegmode is not None:\n args_tesseract.extend(['--psm', str(pagesegmode)])\n\n if text_only and has_textonly_pdf():\n args_tesseract.extend(['-c', 'textonly_pdf=1'])\n\n if user_words:\n args_tesseract.extend(['--user-words', user_words])\n\n if user_patterns:\n args_tesseract.extend(['--user-patterns', user_patterns])\n\n prefix = os.path.splitext(output_pdf)[0] # Tesseract appends suffixes\n\n # Reminder: test suite tesseract spoofers might break after any changes\n # to the number of order parameters here\n\n args_tesseract.extend([input_image, prefix, 'pdf', 'txt'] + tessconfig)\n\n try:\n log.debug(args_tesseract)\n stdout = check_output(args_tesseract, stderr=STDOUT, timeout=timeout)\n if os.path.exists(prefix + '.txt'):\n shutil.move(prefix + '.txt', output_text)\n except TimeoutExpired:\n page_timedout(log, input_image, timeout)\n use_skip_page(text_only, skip_pdf, output_pdf, output_text)\n except CalledProcessError as e:\n tesseract_log_output(log, e.output, input_image)\n if b'Image too large' in e.output:\n use_skip_page(text_only, skip_pdf, output_pdf, output_text)\n return\n raise e from e\n else:\n tesseract_log_output(log, stdout, input_image)", "response": "Generate a PDF file for the current language."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef extract_images(pike, root, log, options, extract_fn):\n\n include_xrefs = set()\n exclude_xrefs = set()\n pageno_for_xref = {}\n errors = 0\n for pageno, page in enumerate(pike.pages):\n try:\n xobjs = page.Resources.XObject\n except AttributeError:\n continue\n for _imname, image in dict(xobjs).items():\n if image.objgen[1] != 0:\n continue # Ignore images in an incremental PDF\n xref = image.objgen[0]\n if hasattr(image, 'SMask'):\n # Ignore soft masks\n smask_xref = image.SMask.objgen[0]\n exclude_xrefs.add(smask_xref)\n include_xrefs.add(xref)\n if xref not in pageno_for_xref:\n pageno_for_xref[xref] = pageno\n\n working_xrefs = include_xrefs - exclude_xrefs\n for xref in working_xrefs:\n image = pike.get_object((xref, 0))\n try:\n result = extract_fn(\n pike=pike, root=root, log=log, image=image, xref=xref, options=options\n )\n except Exception as e:\n log.debug(\"Image xref %s, error %s\", xref, repr(e))\n errors += 1\n else:\n if result:\n _, ext = result\n yield pageno_for_xref[xref], xref, ext", "response": "Extract images from a PDF file."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nextract any > 2bpp images we can improve", "response": "def extract_images_generic(pike, root, log, options):\n \"\"\"Extract any >=2bpp image we think we can improve\"\"\"\n\n jpegs = []\n pngs = []\n for _, xref, ext in extract_images(pike, root, log, options, extract_image_generic):\n log.debug('xref = %s ext = %s', xref, ext)\n if ext == '.png':\n pngs.append(xref)\n elif ext == '.jpg':\n jpegs.append(xref)\n log.debug(\"Optimizable images: JPEGs: %s PNGs: %s\", len(jpegs), len(pngs))\n return jpegs, pngs"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nextracting any bitonal image that we can improve as JBIG2", "response": "def extract_images_jbig2(pike, root, log, options):\n \"\"\"Extract any bitonal image that we think we can improve as JBIG2\"\"\"\n\n jbig2_groups = defaultdict(list)\n for pageno, xref, ext in extract_images(\n pike, root, log, options, extract_image_jbig2\n ):\n group = pageno // options.jbig2_page_group_size\n jbig2_groups[group].append((xref, ext))\n\n # Elide empty groups\n jbig2_groups = {\n group: xrefs for group, xrefs in jbig2_groups.items() if len(xrefs) > 0\n }\n log.debug(\"Optimizable images: JBIG2 groups: %s\", (len(jbig2_groups),))\n return jbig2_groups"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _produce_jbig2_images(jbig2_groups, root, log, options):\n\n def jbig2_group_futures(executor, root, groups):\n for group, xref_exts in groups.items():\n prefix = f'group{group:08d}'\n future = executor.submit(\n jbig2enc.convert_group,\n cwd=fspath(root),\n infiles=(img_name(root, xref, ext) for xref, ext in xref_exts),\n out_prefix=prefix,\n )\n yield future\n\n def jbig2_single_futures(executor, root, groups):\n for group, xref_exts in groups.items():\n prefix = f'group{group:08d}'\n # Second loop is to ensure multiple images per page are unpacked\n for n, xref_ext in enumerate(xref_exts):\n xref, ext = xref_ext\n future = executor.submit(\n jbig2enc.convert_single,\n cwd=fspath(root),\n infile=img_name(root, xref, ext),\n outfile=root / f'{prefix}.{n:04d}',\n )\n yield future\n\n if options.jbig2_page_group_size > 1:\n jbig2_futures = jbig2_group_futures\n else:\n jbig2_futures = jbig2_single_futures\n\n with concurrent.futures.ThreadPoolExecutor(max_workers=options.jobs) as executor:\n futures = jbig2_futures(executor, root, jbig2_groups)\n for future in concurrent.futures.as_completed(futures):\n proc = future.result()\n log.debug(proc.stderr.decode())", "response": "Produce JBIG2 images from the given JBIG2 groups."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nconvert images to JBIG2 and insert into PDF.", "response": "def convert_to_jbig2(pike, jbig2_groups, root, log, options):\n \"\"\"Convert images to JBIG2 and insert into PDF.\n\n When the JBIG2 page group size is > 1 we do several JBIG2 images at once\n and build a symbol dictionary that will span several pages. Each JBIG2\n image must reference to its symbol dictionary. If too many pages shared the\n same dictionary JBIG2 encoding becomes more expensive and less efficient.\n The default value of 10 was determined through testing. Currently this\n must be lossy encoding since jbig2enc does not support refinement coding.\n\n When the JBIG2 symbolic coder is not used, each JBIG2 stands on its own\n and needs no dictionary. Currently this must be lossless JBIG2.\n \"\"\"\n\n _produce_jbig2_images(jbig2_groups, root, log, options)\n\n for group, xref_exts in jbig2_groups.items():\n prefix = f'group{group:08d}'\n jbig2_symfile = root / (prefix + '.sym')\n if jbig2_symfile.exists():\n jbig2_globals_data = jbig2_symfile.read_bytes()\n jbig2_globals = pikepdf.Stream(pike, jbig2_globals_data)\n jbig2_globals_dict = Dictionary(JBIG2Globals=jbig2_globals)\n elif options.jbig2_page_group_size == 1:\n jbig2_globals_dict = None\n else:\n raise FileNotFoundError(jbig2_symfile)\n\n for n, xref_ext in enumerate(xref_exts):\n xref, _ = xref_ext\n jbig2_im_file = root / (prefix + f'.{n:04d}')\n jbig2_im_data = jbig2_im_file.read_bytes()\n im_obj = pike.get_object(xref, 0)\n im_obj.write(\n jbig2_im_data, filter=Name.JBIG2Decode, decode_parms=jbig2_globals_dict\n )"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_version(program, *, version_arg='--version', regex=r'(\\d+(\\.\\d+)*)'):\n \"Get the version of the specified program\"\n args_prog = [program, version_arg]\n try:\n proc = run(\n args_prog,\n close_fds=True,\n universal_newlines=True,\n stdout=PIPE,\n stderr=STDOUT,\n check=True,\n )\n output = proc.stdout\n except FileNotFoundError as e:\n raise MissingDependencyError(\n f\"Could not find program '{program}' on the PATH\"\n ) from e\n except CalledProcessError as e:\n if e.returncode != 0:\n raise MissingDependencyError(\n f\"Ran program '{program}' but it exited with an error:\\n{e.output}\"\n ) from e\n raise MissingDependencyError(\n f\"Could not find program '{program}' on the PATH\"\n ) from e\n try:\n version = re.match(regex, output.strip()).group(1)\n except AttributeError as e:\n raise MissingDependencyError(\n f\"The program '{program}' did not report its version. \"\n f\"Message was:\\n{output}\"\n )\n\n return version", "response": "Get the version of the specified program"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _get_element_text(self, element):\n text = ''\n if element.text is not None:\n text += element.text\n for child in element.getchildren():\n text += self._get_element_text(child)\n if element.tail is not None:\n text += element.tail\n return text", "response": "Return the textual content of the element and its children."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef element_coordinates(cls, element):\n out = (0, 0, 0, 0)\n if 'title' in element.attrib:\n matches = cls.box_pattern.search(element.attrib['title'])\n if matches:\n coords = matches.group(1).split()\n out = Rect._make(int(coords[n]) for n in range(4))\n return out", "response": "Returns the coordinates of the bounding box around\n an element"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef baseline(cls, element):\n if 'title' in element.attrib:\n matches = cls.baseline_pattern.search(element.attrib['title'])\n if matches:\n return float(matches.group(1)), int(matches.group(2))\n return (0, 0)", "response": "Returns a tuple containing the baseline slope and intercept."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef pt_from_pixel(self, pxl):\n return Rect._make((c / self.dpi * inch) for c in pxl)", "response": "Returns the quantity in PDF units given quantity in pixels"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef to_pdf(\n self,\n outFileName,\n imageFileName=None,\n showBoundingboxes=False,\n fontname=\"Helvetica\",\n invisibleText=False,\n interwordSpaces=False,\n ):\n \"\"\"\n Creates a PDF file with an image superimposed on top of the text.\n Text is positioned according to the bounding box of the lines in\n the hOCR file.\n The image need not be identical to the image used to create the hOCR\n file.\n It can have a lower resolution, different color mode, etc.\n \"\"\"\n # create the PDF file\n # page size in points (1/72 in.)\n pdf = Canvas(outFileName, pagesize=(self.width, self.height), pageCompression=1)\n\n # draw bounding box for each paragraph\n # light blue for bounding box of paragraph\n pdf.setStrokeColorRGB(0, 1, 1)\n # light blue for bounding box of paragraph\n pdf.setFillColorRGB(0, 1, 1)\n pdf.setLineWidth(0) # no line for bounding box\n for elem in self.hocr.findall(\".//%sp[@class='%s']\" % (self.xmlns, \"ocr_par\")):\n\n elemtxt = self._get_element_text(elem).rstrip()\n if len(elemtxt) == 0:\n continue\n\n pxl_coords = self.element_coordinates(elem)\n pt = self.pt_from_pixel(pxl_coords)\n\n # draw the bbox border\n if showBoundingboxes:\n pdf.rect(\n pt.x1, self.height - pt.y2, pt.x2 - pt.x1, pt.y2 - pt.y1, fill=1\n )\n\n found_lines = False\n for line in self.hocr.findall(\n \".//%sspan[@class='%s']\" % (self.xmlns, \"ocr_line\")\n ):\n found_lines = True\n self._do_line(\n pdf,\n line,\n \"ocrx_word\",\n fontname,\n invisibleText,\n interwordSpaces,\n showBoundingboxes,\n )\n\n if not found_lines:\n # Tesseract did not report any lines (just words)\n root = self.hocr.find(\".//%sdiv[@class='%s']\" % (self.xmlns, \"ocr_page\"))\n self._do_line(\n pdf,\n root,\n \"ocrx_word\",\n fontname,\n invisibleText,\n interwordSpaces,\n showBoundingboxes,\n )\n # put the image on the page, scaled to fill the page\n if imageFileName is not None:\n pdf.drawImage(imageFileName, 0, 0, width=self.width, height=self.height)\n\n # finish up the page and save it\n pdf.showPage()\n pdf.save()", "response": "Creates a PDF file containing the contents of the HOC."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef extract_text(input_file, pageno=1):\n\n if pageno is not None:\n pages = ['-dFirstPage=%i' % pageno, '-dLastPage=%i' % pageno]\n else:\n pages = []\n\n args_gs = (\n [\n 'gs',\n '-dQUIET',\n '-dSAFER',\n '-dBATCH',\n '-dNOPAUSE',\n '-sDEVICE=txtwrite',\n '-dTextFormat=0',\n ]\n + pages\n + ['-o', '-', fspath(input_file)]\n )\n\n p = run(args_gs, stdout=PIPE, stderr=PIPE)\n if p.returncode != 0:\n raise SubprocessOutputError(\n 'Ghostscript text extraction failed\\n%s\\n%s\\n%s'\n % (input_file, p.stdout.decode(), p.stderr.decode())\n )\n\n return p.stdout", "response": "Use the txtwrite device to get text layout information out\n "} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef rasterize_pdf(\n input_file,\n output_file,\n xres,\n yres,\n raster_device,\n log,\n pageno=1,\n page_dpi=None,\n rotation=None,\n filter_vector=False,\n):\n \"\"\"Rasterize one page of a PDF at resolution (xres, yres) in canvas units.\n\n The image is sized to match the integer pixels dimensions implied by\n (xres, yres) even if those numbers are noninteger. The image's DPI will\n be overridden with the values in page_dpi.\n\n :param input_file: pathlike\n :param output_file: pathlike\n :param xres: resolution at which to rasterize page\n :param yres:\n :param raster_device:\n :param log:\n :param pageno: page number to rasterize (beginning at page 1)\n :param page_dpi: resolution tuple (x, y) overriding output image DPI\n :param rotation: 0, 90, 180, 270: clockwise angle to rotate page\n :param filter_vector: if True, remove vector graphics objects\n :return:\n \"\"\"\n res = xres, yres\n int_res = round(xres), round(yres)\n if not page_dpi:\n page_dpi = res\n\n with NamedTemporaryFile(delete=True) as tmp:\n args_gs = (\n [\n 'gs',\n '-dQUIET',\n '-dSAFER',\n '-dBATCH',\n '-dNOPAUSE',\n f'-sDEVICE={raster_device}',\n f'-dFirstPage={pageno}',\n f'-dLastPage={pageno}',\n f'-r{str(int_res[0])}x{str(int_res[1])}',\n ]\n + (['-dFILTERVECTOR'] if filter_vector else [])\n + [\n '-o',\n tmp.name,\n '-dAutoRotatePages=/None', # Probably has no effect on raster\n '-f',\n fspath(input_file),\n ]\n )\n\n log.debug(args_gs)\n p = run(args_gs, stdout=PIPE, stderr=STDOUT, universal_newlines=True)\n if _gs_error_reported(p.stdout):\n log.error(p.stdout)\n else:\n log.debug(p.stdout)\n\n if p.returncode != 0:\n log.error('Ghostscript rasterizing failed')\n raise SubprocessOutputError()\n\n # Ghostscript only accepts integers for output resolution\n # if the resolution happens to be fractional, then the discrepancy\n # would change the size of the output page, especially if the DPI\n # is quite low. Resize the image to the expected size\n\n tmp.seek(0)\n with Image.open(tmp) as im:\n expected_size = (\n round(im.size[0] / int_res[0] * res[0]),\n round(im.size[1] / int_res[1] * res[1]),\n )\n if expected_size != im.size or page_dpi != (xres, yres):\n log.debug(\n f\"Ghostscript: resize output image {im.size} -> {expected_size}\"\n )\n im = im.resize(expected_size)\n\n if rotation is not None:\n log.debug(\"Rotating output by %i\", rotation)\n # rotation is a clockwise angle and Image.ROTATE_* is\n # counterclockwise so this cancels out the rotation\n if rotation == 90:\n im = im.transpose(Image.ROTATE_90)\n elif rotation == 180:\n im = im.transpose(Image.ROTATE_180)\n elif rotation == 270:\n im = im.transpose(Image.ROTATE_270)\n if rotation % 180 == 90:\n page_dpi = page_dpi[1], page_dpi[0]\n im.save(fspath(output_file), dpi=page_dpi)", "response": "Rasterize one page of a PDF file."} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ngenerates a PDF file for a single PDF.", "response": "def generate_pdfa(\n pdf_pages,\n output_file,\n compression,\n log,\n threads=1,\n pdf_version='1.5',\n pdfa_part='2',\n):\n \"\"\"Generate a PDF/A.\n\n The pdf_pages, a list files, will be merged into output_file. One or more\n PDF files may be merged. One of the files in this list must be a pdfmark\n file that provides Ghostscript with details on how to perform the PDF/A\n conversion. By default with we pick PDF/A-2b, but this works for 1 or 3.\n\n compression can be 'jpeg', 'lossless', or an empty string. In 'jpeg',\n Ghostscript is instructed to convert color and grayscale images to DCT\n (JPEG encoding). In 'lossless' Ghostscript is told to convert images to\n Flate (lossless/PNG). If the parameter is omitted Ghostscript is left to\n make its own decisions about how to encode images; it appears to use a\n heuristic to decide how to encode images. As of Ghostscript 9.25, we\n support passthrough JPEG which allows Ghostscript to avoid transcoding\n images entirely. (The feature was added in 9.23 but broken, and the 9.24\n release of Ghostscript had regressions, so we don't support it until 9.25.)\n \"\"\"\n compression_args = []\n if compression == 'jpeg':\n compression_args = [\n \"-dAutoFilterColorImages=false\",\n \"-dColorImageFilter=/DCTEncode\",\n \"-dAutoFilterGrayImages=false\",\n \"-dGrayImageFilter=/DCTEncode\",\n ]\n elif compression == 'lossless':\n compression_args = [\n \"-dAutoFilterColorImages=false\",\n \"-dColorImageFilter=/FlateEncode\",\n \"-dAutoFilterGrayImages=false\",\n \"-dGrayImageFilter=/FlateEncode\",\n ]\n else:\n compression_args = [\n \"-dAutoFilterColorImages=true\",\n \"-dAutoFilterGrayImages=true\",\n ]\n\n # Older versions of Ghostscript expect a leading slash in\n # sColorConversionStrategy, newer ones should not have it. See Ghostscript\n # git commit fe1c025d.\n strategy = 'RGB' if version() >= '9.19' else '/RGB'\n\n if version() == '9.23':\n # 9.23: new feature JPEG passthrough is broken in some cases, best to\n # disable it always\n # https://bugs.ghostscript.com/show_bug.cgi?id=699216\n compression_args.append('-dPassThroughJPEGImages=false')\n\n with NamedTemporaryFile(delete=True) as gs_pdf:\n # nb no need to specify ProcessColorModel when ColorConversionStrategy\n # is set; see:\n # https://bugs.ghostscript.com/show_bug.cgi?id=699392\n args_gs = (\n [\n \"gs\",\n \"-dQUIET\",\n \"-dBATCH\",\n \"-dNOPAUSE\",\n \"-dCompatibilityLevel=\" + str(pdf_version),\n \"-dNumRenderingThreads=\" + str(threads),\n \"-sDEVICE=pdfwrite\",\n \"-dAutoRotatePages=/None\",\n \"-sColorConversionStrategy=\" + strategy,\n ]\n + compression_args\n + [\n \"-dJPEGQ=95\",\n \"-dPDFA=\" + pdfa_part,\n \"-dPDFACompatibilityPolicy=1\",\n \"-sOutputFile=\" + gs_pdf.name,\n ]\n )\n args_gs.extend(fspath(s) for s in pdf_pages) # Stringify Path objs\n log.debug(args_gs)\n p = run(args_gs, stdout=PIPE, stderr=STDOUT, universal_newlines=True)\n\n if _gs_error_reported(p.stdout):\n log.error(p.stdout)\n elif 'overprint mode not set' in p.stdout:\n # Unless someone is going to print PDF/A documents on a\n # magical sRGB printer I can't see the removal of overprinting\n # being a problem....\n log.debug(\n \"Ghostscript had to remove PDF 'overprinting' from the \"\n \"input file to complete PDF/A conversion. \"\n )\n else:\n log.debug(p.stdout)\n\n if p.returncode == 0:\n # Ghostscript does not change return code when it fails to create\n # PDF/A - check PDF/A status elsewhere\n copy(gs_pdf.name, fspath(output_file))\n else:\n log.error('Ghostscript PDF/A rendering failed')\n raise SubprocessOutputError()"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nconvert a zero consonant to a non - zero word word", "response": "def convert_zero_consonant(pinyin):\n \"\"\"\u96f6\u58f0\u6bcd\u8f6c\u6362\uff0c\u8fd8\u539f\u539f\u59cb\u7684\u97f5\u6bcd\n\n i\u884c\u7684\u97f5\u6bcd\uff0c\u524d\u9762\u6ca1\u6709\u58f0\u6bcd\u7684\u65f6\u5019\uff0c\u5199\u6210yi(\u8863)\uff0cya(\u5440)\uff0cye(\u8036)\uff0cyao(\u8170)\uff0c\n you(\u5fe7)\uff0cyan(\u70df)\uff0cyin(\u56e0)\uff0cyang(\u592e)\uff0cying(\u82f1)\uff0cyong(\u96cd)\u3002\n\n u\u884c\u7684\u97f5\u6bcd\uff0c\u524d\u9762\u6ca1\u6709\u58f0\u6bcd\u7684\u65f6\u5019\uff0c\u5199\u6210wu(\u4e4c)\uff0cwa(\u86d9)\uff0cwo(\u7a9d)\uff0cwai(\u6b6a)\uff0c\n wei(\u5a01)\uff0cwan(\u5f2f)\uff0cwen(\u6e29)\uff0cwang(\u6c6a)\uff0cweng(\u7fc1)\u3002\n\n \u00fc\u884c\u7684\u97f5\u6bcd\uff0c\u524d\u9762\u6ca1\u6709\u58f0\u6bcd\u7684\u65f6\u5019\uff0c\u5199\u6210yu(\u8fc2)\uff0cyue(\u7ea6)\uff0cyuan(\u51a4)\uff0c\n yun(\u6655)\uff1b\u00fc\u4e0a\u4e24\u70b9\u7701\u7565\u3002\n \"\"\"\n # y: yu -> v, yi -> i, y -> i\n if pinyin.startswith('y'):\n # \u53bb\u9664 y \u540e\u7684\u62fc\u97f3\n no_y_py = pinyin[1:]\n first_char = no_y_py[0] if len(no_y_py) > 0 else None\n\n # yu -> \u00fc: yue -> \u00fce\n if first_char in U_TONES:\n pinyin = UV_MAP[first_char] + pinyin[2:]\n # yi -> i: yi -> i\n elif first_char in I_TONES:\n pinyin = no_y_py\n # y -> i: ya -> ia\n else:\n pinyin = 'i' + no_y_py\n return pinyin\n\n # w: wu -> u, w -> u\n if pinyin.startswith('w'):\n # \u53bb\u9664 w \u540e\u7684\u62fc\u97f3\n no_w_py = pinyin[1:]\n first_char = no_w_py[0] if len(no_w_py) > 0 else None\n\n # wu -> u: wu -> u\n if first_char in U_TONES:\n pinyin = pinyin[1:]\n # w -> u: wa -> ua\n else:\n pinyin = 'u' + pinyin[1:]\n return pinyin\n\n return pinyin"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nconverts a Pinyin file to UV.", "response": "def convert_uv(pinyin):\n \"\"\"\u00fc \u8f6c\u6362\uff0c\u8fd8\u539f\u539f\u59cb\u7684\u97f5\u6bcd\n\n \u00fc\u884c\u7684\u97f5\u8ddf\u58f0\u6bcdj\uff0cq\uff0cx\u62fc\u7684\u65f6\u5019\uff0c\u5199\u6210ju(\u5c45)\uff0cqu(\u533a)\uff0cxu(\u865a)\uff0c\n \u00fc\u4e0a\u4e24\u70b9\u4e5f\u7701\u7565\uff1b\u4f46\u662f\u8ddf\u58f0\u6bcdn\uff0cl\u62fc\u7684\u65f6\u5019\uff0c\u4ecd\u7136\u5199\u6210n\u00fc(\u5973)\uff0cl\u00fc(\u5415)\u3002\n \"\"\"\n return UV_RE.sub(\n lambda m: ''.join((m.group(1), UV_MAP[m.group(2)], m.group(3))),\n pinyin)"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef convert_iou(pinyin):\n return IU_RE.sub(lambda m: m.group(1) + IU_MAP[m.group(2)], pinyin)", "response": "converts iou to uni"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef convert_uei(pinyin):\n return UI_RE.sub(lambda m: m.group(1) + UI_MAP[m.group(2)], pinyin)", "response": "convert uti to uui"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef convert_uen(pinyin):\n return UN_RE.sub(lambda m: m.group(1) + UN_MAP[m.group(2)], pinyin)", "response": "convert uen to un"} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nconverts finals to a single word", "response": "def convert_finals(pinyin):\n \"\"\"\u8fd8\u539f\u539f\u59cb\u7684\u97f5\u6bcd\"\"\"\n pinyin = convert_zero_consonant(pinyin)\n pinyin = convert_uv(pinyin)\n pinyin = convert_iou(pinyin)\n pinyin = convert_uei(pinyin)\n pinyin = convert_uen(pinyin)\n return pinyin"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef simple_seg(hans):\n assert not isinstance(hans, bytes_type), \\\n 'must be unicode string or [unicode, ...] list'\n\n if isinstance(hans, text_type):\n return _seg(hans)\n else:\n hans = list(hans)\n if len(hans) == 1:\n return simple_seg(hans[0])\n return list(chain(*[simple_seg(x) for x in hans]))", "response": "Simple segmenter for a list of strings or list of strings"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef load_single_dict(pinyin_dict, style='default'):\n if style == 'tone2':\n for k, v in pinyin_dict.items():\n v = _replace_tone2_style_dict_to_default(v)\n PINYIN_DICT[k] = v\n else:\n PINYIN_DICT.update(pinyin_dict)\n\n mmseg.retrain(mmseg.seg)", "response": "Load a single dict into the Pinyin dictionary."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef load_phrases_dict(phrases_dict, style='default'):\n if style == 'tone2':\n for k, value in phrases_dict.items():\n v = [\n list(map(_replace_tone2_style_dict_to_default, pys))\n for pys in value\n ]\n PHRASES_DICT[k] = v\n else:\n PHRASES_DICT.update(phrases_dict)\n\n mmseg.retrain(mmseg.seg)", "response": "Load the phrases_dict into the internal dictionary of the class."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nconverting a Pinyin style to a fixed style.", "response": "def to_fixed(pinyin, style, strict=True):\n \"\"\"\u6839\u636e\u62fc\u97f3\u98ce\u683c\u683c\u5f0f\u5316\u5e26\u58f0\u8c03\u7684\u62fc\u97f3.\n\n :param pinyin: \u5355\u4e2a\u62fc\u97f3\n :param style: \u62fc\u97f3\u98ce\u683c\n :param strict: \u662f\u5426\u4e25\u683c\u9075\u7167\u300a\u6c49\u8bed\u62fc\u97f3\u65b9\u6848\u300b\u6765\u5904\u7406\u58f0\u6bcd\u548c\u97f5\u6bcd\n :return: \u6839\u636e\u62fc\u97f3\u98ce\u683c\u683c\u5f0f\u5316\u540e\u7684\u62fc\u97f3\u5b57\u7b26\u4e32\n :rtype: unicode\n \"\"\"\n return convert_style(pinyin, style=style, strict=strict, default=pinyin)"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nhandle a Nopinyin character.", "response": "def _handle_nopinyin_char(chars, errors='default'):\n \"\"\"\u5904\u7406\u6ca1\u6709\u62fc\u97f3\u7684\u5b57\u7b26\"\"\"\n if callable_check(errors):\n return errors(chars)\n\n if errors == 'default':\n return chars\n elif errors == 'ignore':\n return None\n elif errors == 'replace':\n if len(chars) > 1:\n return ''.join(text_type('%x' % ord(x)) for x in chars)\n else:\n return text_type('%x' % ord(chars))"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef single_pinyin(han, style, heteronym, errors='default', strict=True):\n num = ord(han)\n # \u5904\u7406\u6ca1\u6709\u62fc\u97f3\u7684\u5b57\u7b26\n if num not in PINYIN_DICT:\n return handle_nopinyin(han, errors=errors, heteronym=heteronym)\n\n pys = PINYIN_DICT[num].split(',') # \u5b57\u7684\u62fc\u97f3\u5217\u8868\n if not heteronym:\n return [[_to_fixed(pys[0], style, strict=strict)]]\n\n # \u8f93\u51fa\u591a\u97f3\u5b57\u7684\u591a\u4e2a\u8bfb\u97f3\n # \u4e34\u65f6\u5b58\u50a8\u5df2\u5b58\u5728\u7684\u62fc\u97f3\uff0c\u907f\u514d\u591a\u97f3\u5b57\u62fc\u97f3\u8f6c\u6362\u4e3a\u975e\u97f3\u6807\u98ce\u683c\u51fa\u73b0\u91cd\u590d\u3002\n # TODO: change to use set\n # TODO: add test for cache\n py_cached = {}\n pinyins = []\n for i in pys:\n py = _to_fixed(i, style, strict=strict)\n if py in py_cached:\n continue\n py_cached[py] = py\n pinyins.append(py)\n return [pinyins]", "response": "single_pinyin \u5355\u5b57\u62fc\u97f3\u8f6c\u6362 \u5904\u7406 \u5355\u5b57\u62fc\u97f3\u8f6c\u6362 \u5904\u7406 \u5355\u5b57\u62fc\u97f3\u8f6c\u6362 \u5904\u7406 \u5355\u5b57\u62fc\u97f3\ufffd"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef phrase_pinyin(phrase, style, heteronym, errors='default', strict=True):\n py = []\n if phrase in PHRASES_DICT:\n py = deepcopy(PHRASES_DICT[phrase])\n for idx, item in enumerate(py):\n if heteronym:\n py[idx] = _remove_dup_items([\n _to_fixed(x, style=style, strict=strict) for x in item])\n else:\n py[idx] = [_to_fixed(item[0], style=style, strict=strict)]\n else:\n for i in phrase:\n single = single_pinyin(i, style=style, heteronym=heteronym,\n errors=errors, strict=strict)\n if single:\n py.extend(single)\n return py", "response": "\u8bcd\u8bed\u62fc\u97f3\u8f6c\u6362.\n\n :param phrase: \u8bcd\u8bed\n :param errors: \u6307\u5b9a\u5982\u4f55\u5904\u7406\u6ca1\u6709\u62fc\u97f3\u7684\u5b57\u7b26\n :param strict: \u662f\u5426\u4e25\u683c\u9075\u7167\u300a\u6c49\u8bed\u62fc\u97f3\u65b9\u6848\u300b\u6765\u5904\u7406\u58f0\u6bcd\u548c\u97f5\u6bcd\n :return: \u62fc\u97f3\u5217\u8868\n :rtype: list"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef pinyin(hans, style=Style.TONE, heteronym=False,\n errors='default', strict=True):\n \"\"\"\u5c06\u6c49\u5b57\u8f6c\u6362\u4e3a\u62fc\u97f3.\n\n :param hans: \u6c49\u5b57\u5b57\u7b26\u4e32( ``'\u4f60\u597d\u5417'`` )\u6216\u5217\u8868( ``['\u4f60\u597d', '\u5417']`` ).\n \u53ef\u4ee5\u4f7f\u7528\u81ea\u5df1\u559c\u7231\u7684\u5206\u8bcd\u6a21\u5757\u5bf9\u5b57\u7b26\u4e32\u8fdb\u884c\u5206\u8bcd\u5904\u7406,\n \u53ea\u9700\u5c06\u7ecf\u8fc7\u5206\u8bcd\u5904\u7406\u7684\u5b57\u7b26\u4e32\u5217\u8868\u4f20\u8fdb\u6765\u5c31\u53ef\u4ee5\u4e86\u3002\n :type hans: unicode \u5b57\u7b26\u4e32\u6216\u5b57\u7b26\u4e32\u5217\u8868\n :param style: \u6307\u5b9a\u62fc\u97f3\u98ce\u683c\uff0c\u9ed8\u8ba4\u662f :py:attr:`~pypinyin.Style.TONE` \u98ce\u683c\u3002\n \u66f4\u591a\u62fc\u97f3\u98ce\u683c\u8be6\u89c1 :class:`~pypinyin.Style`\n :param errors: \u6307\u5b9a\u5982\u4f55\u5904\u7406\u6ca1\u6709\u62fc\u97f3\u7684\u5b57\u7b26\u3002\u8be6\u89c1 :ref:`handle_no_pinyin`\n\n * ``'default'``: \u4fdd\u7559\u539f\u59cb\u5b57\u7b26\n * ``'ignore'``: \u5ffd\u7565\u8be5\u5b57\u7b26\n * ``'replace'``: \u66ff\u6362\u4e3a\u53bb\u6389 ``\\\\u`` \u7684 unicode \u7f16\u7801\u5b57\u7b26\u4e32\n (``'\\\\u90aa'`` => ``'90aa'``)\n * callable \u5bf9\u8c61: \u56de\u8c03\u51fd\u6570\u4e4b\u7c7b\u7684\u53ef\u8c03\u7528\u5bf9\u8c61\u3002\n\n :param heteronym: \u662f\u5426\u542f\u7528\u591a\u97f3\u5b57\n :param strict: \u662f\u5426\u4e25\u683c\u9075\u7167\u300a\u6c49\u8bed\u62fc\u97f3\u65b9\u6848\u300b\u6765\u5904\u7406\u58f0\u6bcd\u548c\u97f5\u6bcd\uff0c\u8be6\u89c1 :ref:`strict`\n :return: \u62fc\u97f3\u5217\u8868\n :rtype: list\n\n :raise AssertionError: \u5f53\u4f20\u5165\u7684\u5b57\u7b26\u4e32\u4e0d\u662f unicode \u5b57\u7b26\u65f6\u4f1a\u629b\u51fa\u8fd9\u4e2a\u5f02\u5e38\n\n Usage::\n\n >>> from pypinyin import pinyin, Style\n >>> import pypinyin\n >>> pinyin('\u4e2d\u5fc3')\n [['zh\u014dng'], ['x\u012bn']]\n >>> pinyin('\u4e2d\u5fc3', heteronym=True) # \u542f\u7528\u591a\u97f3\u5b57\u6a21\u5f0f\n [['zh\u014dng', 'zh\u00f2ng'], ['x\u012bn']]\n >>> pinyin('\u4e2d\u5fc3', style=Style.FIRST_LETTER) # \u8bbe\u7f6e\u62fc\u97f3\u98ce\u683c\n [['z'], ['x']]\n >>> pinyin('\u4e2d\u5fc3', style=Style.TONE2)\n [['zho1ng'], ['xi1n']]\n >>> pinyin('\u4e2d\u5fc3', style=Style.CYRILLIC)\n [['\u0447\u0436\u0443\u043d1'], ['\u0441\u0438\u043d\u044c1']]\n \"\"\"\n # \u5bf9\u5b57\u7b26\u4e32\u8fdb\u884c\u5206\u8bcd\u5904\u7406\n if isinstance(hans, text_type):\n han_list = seg(hans)\n else:\n han_list = chain(*(seg(x) for x in hans))\n pys = []\n for words in han_list:\n pys.extend(_pinyin(words, style, heteronym, errors, strict=strict))\n return pys", "response": "\u5c06\u6c49\u5b57\u8f6c\u6362\u4e3a\u62fc\u97f3.\n\n :param hans: \u6c49\u5b57\u5b57\u7b26\u4e32( ``'\u4f60\u597d\u5417'`` )\u6216\u5217\u8868( ``['\u4f60\u597d', '\u5417']`` ).\n \u53ef\u4ee5\u4f7f\u7528\u81ea\u5df1\u559c\u7231\u7684\u5206\u8bcd\u6a21\u5757\u5bf9\u5b57\u7b26\u4e32\u8fdb\u884c\u5206\u8bcd\u5904\u7406,\n \u53ea\u9700\u5c06\u7ecf\u8fc7\u5206\u8bcd\u5904\u7406\u7684\u5b57\u7b26\u4e32\u5217\u8868\u4f20\u8fdb\u6765\u5c31\u53ef\u4ee5\u4e86\u3002\n :type hans: unicode \u5b57\u7b26\u4e32\u6216\u5b57\u7b26\u4e32\u5217\u8868\n :param style: \u6307\u5b9a\u62fc\u97f3\u98ce\u683c\uff0c\u9ed8\u8ba4\u662f :py:attr:`~pypinyin.Style.TONE` \u98ce\u683c\u3002\n \u66f4\u591a\u62fc\u97f3\u98ce\u683c\u8be6\u89c1 :class:`~pypinyin.Style`\n :param errors: \u6307\u5b9a\u5982\u4f55\u5904\u7406\u6ca1\u6709\u62fc\u97f3\u7684\u5b57\u7b26\u3002\u8be6\u89c1 :ref:`handle_no_pinyin`\n\n * ``'default'``: \u4fdd\u7559\u539f\u59cb\u5b57\u7b26\n * ``'ignore'``: \u5ffd\u7565\u8be5\u5b57\u7b26\n * ``'replace'``: \u66ff\u6362\u4e3a\u53bb\u6389 ``\\\\u`` \u7684 unicode \u7f16\u7801\u5b57\u7b26\u4e32\n (``'\\\\u90aa'`` => ``'90aa'``)\n * callable \u5bf9\u8c61: \u56de\u8c03\u51fd\u6570\u4e4b\u7c7b\u7684\u53ef\u8c03\u7528\u5bf9\u8c61\u3002\n\n :param heteronym: \u662f\u5426\u542f\u7528\u591a\u97f3\u5b57\n :param strict: \u662f\u5426\u4e25\u683c\u9075\u7167\u300a\u6c49\u8bed\u62fc\u97f3\u65b9\u6848\u300b\u6765\u5904\u7406\u58f0\u6bcd\u548c\u97f5\u6bcd\uff0c\u8be6\u89c1 :ref:`strict`\n :return: \u62fc\u97f3\u5217\u8868\n :rtype: list\n\n :raise AssertionError: \u5f53\u4f20\u5165\u7684\u5b57\u7b26\u4e32\u4e0d\u662f unicode \u5b57\u7b26\u65f6\u4f1a\u629b\u51fa\u8fd9\u4e2a\u5f02\u5e38\n\n Usage::\n\n >>> from pypinyin import pinyin, Style\n >>> import pypinyin\n >>> pinyin('\u4e2d\u5fc3')\n [['zh\u014dng'], ['x\u012bn']]\n >>> pinyin('\u4e2d\u5fc3', heteronym=True) # \u542f\u7528\u591a\u97f3\u5b57\u6a21\u5f0f\n [['zh\u014dng', 'zh\u00f2ng'], ['x\u012bn']]\n >>> pinyin('\u4e2d\u5fc3', style=Style.FIRST_LETTER) # \u8bbe\u7f6e\u62fc\u97f3\u98ce\u683c\n [['z'], ['x']]\n >>> pinyin('\u4e2d\u5fc3', style=Style.TONE2)\n [['zho1ng'], ['xi1n']]\n >>> pinyin('\u4e2d\u5fc3', style=Style.CYRILLIC)\n [['\u0447\u0436\u0443\u043d1'], ['\u0441\u0438\u043d\u044c1']]"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a slug for the given list of hans.", "response": "def slug(hans, style=Style.NORMAL, heteronym=False, separator='-',\n errors='default', strict=True):\n \"\"\"\u751f\u6210 slug \u5b57\u7b26\u4e32.\n\n :param hans: \u6c49\u5b57\n :type hans: unicode or list\n :param style: \u6307\u5b9a\u62fc\u97f3\u98ce\u683c\uff0c\u9ed8\u8ba4\u662f :py:attr:`~pypinyin.Style.NORMAL` \u98ce\u683c\u3002\n \u66f4\u591a\u62fc\u97f3\u98ce\u683c\u8be6\u89c1 :class:`~pypinyin.Style`\n :param heteronym: \u662f\u5426\u542f\u7528\u591a\u97f3\u5b57\n :param separstor: \u4e24\u4e2a\u62fc\u97f3\u95f4\u7684\u5206\u9694\u7b26/\u8fde\u63a5\u7b26\n :param errors: \u6307\u5b9a\u5982\u4f55\u5904\u7406\u6ca1\u6709\u62fc\u97f3\u7684\u5b57\u7b26\uff0c\u8be6\u60c5\u8bf7\u53c2\u8003\n :py:func:`~pypinyin.pinyin`\n :param strict: \u662f\u5426\u4e25\u683c\u9075\u7167\u300a\u6c49\u8bed\u62fc\u97f3\u65b9\u6848\u300b\u6765\u5904\u7406\u58f0\u6bcd\u548c\u97f5\u6bcd\uff0c\u8be6\u89c1 :ref:`strict`\n :return: slug \u5b57\u7b26\u4e32.\n\n :raise AssertionError: \u5f53\u4f20\u5165\u7684\u5b57\u7b26\u4e32\u4e0d\u662f unicode \u5b57\u7b26\u65f6\u4f1a\u629b\u51fa\u8fd9\u4e2a\u5f02\u5e38\n\n ::\n\n >>> import pypinyin\n >>> from pypinyin import Style\n >>> pypinyin.slug('\u4e2d\u56fd\u4eba')\n 'zhong-guo-ren'\n >>> pypinyin.slug('\u4e2d\u56fd\u4eba', separator=' ')\n 'zhong guo ren'\n >>> pypinyin.slug('\u4e2d\u56fd\u4eba', style=Style.FIRST_LETTER)\n 'z-g-r'\n >>> pypinyin.slug('\u4e2d\u56fd\u4eba', style=Style.CYRILLIC)\n '\u0447\u0436\u0443\u043d1-\u0433\u043e2-\u0436\u044d\u043d\u044c2'\n \"\"\"\n return separator.join(chain(*pinyin(hans, style=style, heteronym=heteronym,\n errors=errors, strict=strict)\n ))"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef lazy_pinyin(hans, style=Style.NORMAL, errors='default', strict=True):\n return list(chain(*pinyin(hans, style=style, heteronym=False,\n errors=errors, strict=strict)))", "response": "\u4e0d\u5305\u542b\u591a\u97f3\u5b57\u7684\u62fc\u97f3\u5217\u8868.\n\n \u4e0e :py:func:`~pypinyin.pinyin` \u7684\u533a\u522b\u662f\u8fd4\u56de\u7684\u62fc\u97f3\u662f\u4e2a\u5b57\u7b26\u4e32\uff0c\n \u5e76\u4e14\u6bcf\u4e2a\u5b57\u53ea\u5305\u542b\u4e00\u4e2a\u8bfb\u97f3.\n\n :param hans: \u6c49\u5b57\n :type hans: unicode or list\n :param style: \u6307\u5b9a\u62fc\u97f3\u98ce\u683c\uff0c\u9ed8\u8ba4\u662f :py:attr:`~pypinyin.Style.NORMAL` \u98ce\u683c\u3002\n \u66f4\u591a\u62fc\u97f3\u98ce\u683c\u8be6\u89c1 :class:`~pypinyin.Style`\u3002\n :param errors: \u6307\u5b9a\u5982\u4f55\u5904\u7406\u6ca1\u6709\u62fc\u97f3\u7684\u5b57\u7b26\uff0c\u8be6\u60c5\u8bf7\u53c2\u8003\n :py:func:`~pypinyin.pinyin`\n :param strict: \u662f\u5426\u4e25\u683c\u9075\u7167\u300a\u6c49\u8bed\u62fc\u97f3\u65b9\u6848\u300b\u6765\u5904\u7406\u58f0\u6bcd\u548c\u97f5\u6bcd\uff0c\u8be6\u89c1 :ref:`strict`\n :return: \u62fc\u97f3\u5217\u8868(e.g. ``['zhong', 'guo', 'ren']``)\n :rtype: list\n\n :raise AssertionError: \u5f53\u4f20\u5165\u7684\u5b57\u7b26\u4e32\u4e0d\u662f unicode \u5b57\u7b26\u65f6\u4f1a\u629b\u51fa\u8fd9\u4e2a\u5f02\u5e38\n\n Usage::\n\n >>> from pypinyin import lazy_pinyin, Style\n >>> import pypinyin\n >>> lazy_pinyin('\u4e2d\u5fc3')\n ['zhong', 'xin']\n >>> lazy_pinyin('\u4e2d\u5fc3', style=Style.TONE)\n ['zh\u014dng', 'x\u012bn']\n >>> lazy_pinyin('\u4e2d\u5fc3', style=Style.FIRST_LETTER)\n ['z', 'x']\n >>> lazy_pinyin('\u4e2d\u5fc3', style=Style.TONE2)\n ['zho1ng', 'xi1n']\n >>> lazy_pinyin('\u4e2d\u5fc3', style=Style.CYRILLIC)\n ['\u0447\u0436\u0443\u043d1', '\u0441\u0438\u043d\u044c1']"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef convert(pinyin, style, strict, default=None, **kwargs):\n if style in _registry:\n return _registry[style](pinyin, strict=strict, **kwargs)\n return default", "response": "Convert a Pinyin string to a Pinyin string."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef register(style, func=None):\n if func is not None:\n _registry[style] = func\n return\n\n def decorator(func):\n _registry[style] = func\n\n @wraps(func)\n def wrapper(pinyin, **kwargs):\n return func(pinyin, **kwargs)\n\n return wrapper\n return decorator", "response": "register a function to be used in a specific style"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef get_initials(pinyin, strict):\n if strict:\n _initials = _INITIALS\n else:\n _initials = _INITIALS_NOT_STRICT\n\n for i in _initials:\n if pinyin.startswith(i):\n return i\n return ''", "response": "Get initials from a Pinyin string."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_finals(pinyin, strict):\n if strict:\n pinyin = convert_finals(pinyin)\n\n initials = get_initials(pinyin, strict=strict) or ''\n # \u6ca1\u6709\u58f0\u6bcd\uff0c\u6574\u4e2a\u90fd\u662f\u97f5\u6bcd\n if not initials:\n return pinyin\n # \u6309\u58f0\u6bcd\u5206\u5272\uff0c\u5269\u4e0b\u7684\u5c31\u662f\u97f5\u6bcd\n return ''.join(pinyin.split(initials, 1))", "response": "get_finals - Get the finals of a Pinyin file"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreplace the symbol to number", "response": "def replace_symbol_to_number(pinyin):\n \"\"\"\u628a\u58f0\u8c03\u66ff\u6362\u4e3a\u6570\u5b57\"\"\"\n def _replace(match):\n symbol = match.group(0) # \u5e26\u58f0\u8c03\u7684\u5b57\u7b26\n # \u8fd4\u56de\u4f7f\u7528\u6570\u5b57\u6807\u8bc6\u58f0\u8c03\u7684\u5b57\u7b26\n return PHONETIC_SYMBOL_DICT[symbol]\n\n # \u66ff\u6362\u62fc\u97f3\u4e2d\u7684\u5e26\u58f0\u8c03\u5b57\u7b26\n return RE_PHONETIC_SYMBOL.sub(_replace, pinyin)"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreplacing symbol to no symbol.", "response": "def replace_symbol_to_no_symbol(pinyin):\n \"\"\"\u628a\u5e26\u58f0\u8c03\u5b57\u7b26\u66ff\u6362\u4e3a\u6ca1\u6709\u58f0\u8c03\u7684\u5b57\u7b26\"\"\"\n def _replace(match):\n symbol = match.group(0) # \u5e26\u58f0\u8c03\u7684\u5b57\u7b26\n # \u53bb\u6389\u58f0\u8c03: a1 -> a\n return RE_NUMBER.sub(r'', PHONETIC_SYMBOL_DICT[symbol])\n\n # \u66ff\u6362\u62fc\u97f3\u4e2d\u7684\u5e26\u58f0\u8c03\u5b57\u7b26\n return RE_PHONETIC_SYMBOL.sub(_replace, pinyin)"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef cut(self, text):\n remain = text\n while remain:\n matched = ''\n # \u4e00\u6b21\u52a0\u4e00\u4e2a\u5b57\u7684\u5339\u914d\n for index in range(len(remain)):\n word = remain[:index + 1]\n if word in self._prefix_set:\n matched = word\n else:\n # \u524d\u9762\u7684\u5b57\u7b26\u4e32\u662f\u4e2a\u8bcd\u8bed\n if matched:\n yield matched\n matched = ''\n remain = remain[index:]\n else: # \u524d\u9762\u4e3a\u7a7a\n yield word\n remain = remain[index + 1:]\n # \u6709\u7ed3\u679c\u4e86\uff0c\u5269\u4f59\u7684\u91cd\u65b0\u5f00\u59cb\u5339\u914d\n break\n else: # \u6574\u4e2a\u6587\u672c\u5c31\u662f\u4e00\u4e2a\u8bcd\u8bed\n yield remain\n break", "response": "yields all the words in the given text"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef train(self, word_s):\n for word in word_s:\n # \u628a\u8bcd\u8bed\u7684\u6bcf\u4e2a\u524d\u7f00\u66f4\u65b0\u5230 prefix_set \u4e2d\n for index in range(len(word)):\n self._set.add(word[:index + 1])", "response": "train set of all the key - value pairs"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns the type of a character.", "response": "def get_char_type(ch):\n \"\"\"\n 0, \u6c49\u5b57\n 1, \u82f1\u6587\u5b57\u6bcd\n 2. \u6570\u5b57\n 3. \u5176\u4ed6\n \"\"\"\n if re.match(en_p, ch):\n return 1\n elif re.match(\"\\d+\", ch):\n return 2\n elif re.match(re_han, ch):\n return 3\n else:\n return 4"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef to_latex(circuit, settings=None):\n if settings is None:\n settings = get_default_settings()\n text = header(settings)\n text += body(circuit, settings)\n text += footer()\n return text", "response": "Translates a given pyquil Program to a LaTeX document."} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning the body of the TikZ format of the given circuit.", "response": "def body(circuit, settings):\n \"\"\"\n Return the body of the Latex document, including the entire circuit in\n TikZ format.\n\n :param Program circuit: The circuit to be drawn, represented as a pyquil program.\n :param dict settings:\n\n :return: Latex string to draw the entire circuit.\n :rtype: string\n \"\"\"\n qubit_instruction_mapping = {}\n\n # Allocate each qubit.\n for inst in circuit:\n if isinstance(inst, Measurement):\n inst.qubits = [inst.qubit]\n inst.name = \"MEASURE\"\n else:\n qubits = inst.qubits\n for qubit in qubits:\n qubit_instruction_mapping[qubit.index] = []\n for k, v in list(qubit_instruction_mapping.items()):\n v.append(command(ALLOCATE, [k], [], [k], k))\n\n for inst in circuit:\n qubits = [qubit.index for qubit in inst.qubits]\n gate = inst.name\n # If this is a single qubit instruction.\n if len(qubits) == 1:\n for qubit in qubits:\n qubit_instruction_mapping[qubit].append(command(gate, [qubit], [], [qubit], qubit))\n\n # If this is a many-qubit operation.\n else:\n # All qubits explicitly involved in the gate.\n explicit_lines = [qubit for qubit in copy(qubits)]\n # All lines to consider that will run through this circuit element.\n all_lines = list(range(min(explicit_lines), max(explicit_lines) + 1))\n # Include all lines that are in-use and in the range of lines used in this instruction.\n for line in all_lines:\n if line not in qubit_instruction_mapping.keys() and line in all_lines:\n all_lines.remove(line)\n for i, qubit in enumerate(all_lines):\n if gate == CZ:\n ctrl_lines = list(explicit_lines)\n ctrl_lines.remove(qubits[-1])\n qubit_instruction_mapping[qubit].append(command(Z, list(all_lines), list(ctrl_lines),\n qubits[-1:], None))\n elif gate == CNOT:\n ctrl_lines = list(explicit_lines)\n ctrl_lines.remove(qubits[-1])\n qubit_instruction_mapping[qubit].append(command(X, list(all_lines), list(ctrl_lines),\n qubits[-1:], None))\n else:\n qubit_instruction_mapping[qubit].append(command(gate, list(all_lines), [],\n list(explicit_lines), None))\n\n # Zero index, and remove gaps in spacing.\n relabeled_circuit = {}\n # Store a mapping so we can relabel command labels.\n index_map = {}\n for i, key in enumerate(sorted(qubit_instruction_mapping.keys())):\n relabeled_circuit[i] = qubit_instruction_mapping[key]\n index_map[key] = i\n\n for line in list(relabeled_circuit.values()):\n for cmd in line:\n for i, qubit in enumerate(cmd.lines):\n cmd.lines[i] = index_map[qubit]\n for i, qubit in enumerate(cmd.ctrl_lines):\n cmd.ctrl_lines[i] = index_map[qubit]\n for i, qubit in enumerate(cmd.target_lines):\n cmd.target_lines[i] = index_map[qubit]\n code_generator = CircuitTikzGenerator(settings)\n return code_generator.generate_circuit(relabeled_circuit)"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ngenerating the TikZ code for a given line of the circuit.", "response": "def to_tikz(self, line, circuit, end=None):\n \"\"\"\n Generate the TikZ code for one line of the circuit up to a certain\n gate.\n\n It modifies the circuit to include only the gates which have not been\n drawn. It automatically switches to other lines if the gates on those\n lines have to be drawn earlier.\n\n :param int line: Line to generate the TikZ code for.\n :param Program circuit: The circuit to draw.\n :param int end: Gate index to stop at (for recursion).\n :return:TikZ code representing the current qubit line and, if it was\n necessary to draw other lines, those lines as well.\n :rtype: string\n \"\"\"\n if end is None:\n end = len(circuit[line])\n tikz_code = []\n cmds = circuit[line]\n for i in range(0, end):\n gate = cmds[i].gate\n lines = cmds[i].lines\n ctrl_lines = cmds[i].ctrl_lines\n all_lines = list(lines)\n all_lines.remove(line)\n for l in all_lines:\n gate_idx = 0\n while not (circuit[l][gate_idx] == cmds[i]):\n gate_idx += 1\n tikz_code.append(self.to_tikz(l, circuit, gate_idx))\n\n # We remove the first element of the current line, since we have processed it.\n circuit[l] = circuit[l][1:]\n all_lines = cmds[i].target_lines + cmds[i].ctrl_lines\n pos = max([self.pos[l] for l in range(min(all_lines),\n max(all_lines) + 1)])\n for l in range(min(all_lines), max(all_lines) + 1):\n self.pos[l] = pos + self._gate_pre_offset(gate)\n\n connections = \"\"\n for l in all_lines:\n connections += self._line(self.op_count[l] - 1, self.op_count[l], line=l)\n add_str = \"\"\n if gate == X:\n # draw NOT-gate with controls\n add_str = self._x_gate(cmds[i].target_lines, ctrl_lines)\n # and make the target qubit quantum if one of the controls is\n if not self.is_quantum[lines[0]]:\n if sum([self.is_quantum[i] for i in ctrl_lines]) > 0:\n self.is_quantum[lines[0]] = True\n elif gate == Z and len(ctrl_lines) > 0:\n add_str = self._cz_gate(lines + ctrl_lines)\n elif gate == SWAP:\n add_str = self._swap_gate(lines, ctrl_lines)\n elif gate == MEASURE:\n # draw measurement gate\n for l in lines:\n op = self._op(l)\n width = self._gate_width(MEASURE)\n height = self._gate_height(MEASURE)\n shift0 = .07 * height\n shift1 = .36 * height\n shift2 = .1 * width\n add_str += (\"\\n\\\\node[measure,edgestyle] ({op}) at ({pos},-{line}) {{}};\\n\"\n \"\\\\draw[edgestyle] ([yshift=-{shift1}cm,xshift={shift2}cm]{op}.west)\"\n \" to [out=60,in=180] ([yshift={shift0}cm]{op}.\"\n \"center) to [out=0, in=120] ([yshift=-{shift1}\"\n \"cm,xshift=-{shift2}cm]{op}.east);\\n\"\n \"\\\\draw[edgestyle] ([yshift=-{shift1}cm]{op}.\"\n \"center) to ([yshift=-{shift2}cm,xshift=-{shift1}cm]{op}.north east);\").format(\n op=op, pos=self.pos[l], line=l, shift0=shift0, shift1=shift1, shift2=shift2)\n self.op_count[l] += 1\n self.pos[l] += (self._gate_width(gate) + self._gate_offset(gate))\n self.is_quantum[l] = False\n elif gate == ALLOCATE:\n # Draw 'begin line'.\n add_str = \"\\n\\\\node[none] ({}) at ({},-{}) {{$\\\\Ket{{0}}{}$}};\"\n id_str = \"\"\n if self.settings['gates']['AllocateQubitGate']['draw_id']:\n id_str = \"^{{\\\\textcolor{{red}}{{{}}}}}\".format(cmds[i].id)\n add_str = add_str.format(self._op(line), self.pos[line], line,\n id_str)\n self.op_count[line] += 1\n self.pos[line] += (self._gate_offset(gate) + self._gate_width(gate))\n self.is_quantum[line] = self.settings['lines']['init_quantum']\n else:\n # Regular gate must draw the lines it does not act upon.\n # If it spans multiple qubits.\n add_str = self._regular_gate(gate, lines, ctrl_lines, cmds[i].target_lines)\n for l in lines:\n self.is_quantum[l] = True\n tikz_code.append(add_str)\n if not gate == ALLOCATE:\n tikz_code.append(connections)\n circuit[line] = circuit[line][end:]\n return \"\".join(tikz_code)"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning the string representation of the LaTeX gate.", "response": "def _gate_name(self, gate):\n \"\"\"\n Return the string representation of the gate.\n\n Tries to use gate.tex_str and, if that is not available, uses str(gate)\n instead.\n\n :param string gate: Gate object of which to get the name / LaTeX representation.\n :return: LaTeX gate name.\n :rtype: string\n \"\"\"\n try:\n name = gate.tex_str()\n except AttributeError:\n name = str(gate)\n return name"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns the TikZ code for a SWAP gate.", "response": "def _swap_gate(self, lines, ctrl_lines):\n \"\"\"\n Return the TikZ code for a Swap-gate.\n\n :param lines: List of length 2 denoting the target qubit of the Swap gate.\n :type: list[int]\n :param ctrl_lines: List of qubit lines which act as controls.\n :type: list[int]\n \"\"\"\n delta_pos = self._gate_offset(SWAP)\n gate_width = self._gate_width(SWAP)\n lines.sort()\n\n gate_str = \"\"\n for line in lines:\n op = self._op(line)\n w = \"{}cm\".format(.5 * gate_width)\n s1 = \"[xshift=-{w},yshift=-{w}]{op}.center\".format(w=w, op=op)\n s2 = \"[xshift={w},yshift={w}]{op}.center\".format(w=w, op=op)\n s3 = \"[xshift=-{w},yshift={w}]{op}.center\".format(w=w, op=op)\n s4 = \"[xshift={w},yshift=-{w}]{op}.center\".format(w=w, op=op)\n swap_style = \"swapstyle,edgestyle\"\n if self.settings['gate_shadow']:\n swap_style += \",shadowed\"\n gate_str += (\"\\n\\\\node[swapstyle] ({op}) at ({pos},-{line}) {{}};\"\n \"\\n\\\\draw[{swap_style}] ({s1})--({s2});\\n\"\n \"\\\\draw[{swap_style}] ({s3})--({s4});\"\n ).format(op=op, s1=s1, s2=s2, s3=s3, s4=s4,\n line=line, pos=self.pos[line],\n swap_style=swap_style)\n gate_str += self._line(lines[0], lines[1])\n all_lines = ctrl_lines + lines\n new_pos = self.pos[lines[0]] + delta_pos + gate_width\n for i in all_lines:\n self.op_count[i] += 1\n for i in range(min(all_lines), max(all_lines) + 1):\n self.pos[i] = new_pos\n return gate_str"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning the TikZ code for a NOT gate.", "response": "def _x_gate(self, lines, ctrl_lines):\n \"\"\"\n Return the TikZ code for a NOT-gate.\n\n :param lines: List of length 1 denoting the target qubit of the NOT / X gate.\n :type: list[int]\n :param ctrl_lines: List of qubit lines which act as controls.\n :type: list[int]\n \"\"\"\n line = lines[0]\n delta_pos = self._gate_offset('X')\n gate_width = self._gate_width('X')\n op = self._op(line)\n gate_str = (\"\\n\\\\node[xstyle] ({op}) at ({pos},-{line}) {{}};\\n\\\\draw\"\n \"[edgestyle] ({op}.north)--({op}.south);\\n\\\\draw\"\n \"[edgestyle] ({op}.west)--({op}.east);\").format(op=op, line=line, pos=self.pos[line])\n if len(ctrl_lines) > 0:\n for ctrl in ctrl_lines:\n gate_str += self._phase(ctrl, self.pos[line])\n gate_str += self._line(ctrl, line)\n\n all_lines = ctrl_lines + [line]\n new_pos = self.pos[line] + delta_pos + gate_width\n for i in all_lines:\n self.op_count[i] += 1\n for i in range(min(all_lines), max(all_lines) + 1):\n self.pos[i] = new_pos\n return gate_str"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _cz_gate(self, lines):\n line = lines[0]\n delta_pos = self._gate_offset(Z)\n gate_width = self._gate_width(Z)\n gate_str = self._phase(line, self.pos[line])\n\n for ctrl in lines[1:]:\n gate_str += self._phase(ctrl, self.pos[line])\n gate_str += self._line(ctrl, line)\n\n new_pos = self.pos[line] + delta_pos + gate_width\n for i in lines:\n self.op_count[i] += 1\n for i in range(min(lines), max(lines) + 1):\n self.pos[i] = new_pos\n return gate_str", "response": "Return the TikZ code for an n - controlled Z - gate."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _gate_width(self, gate):\n try:\n gates = self.settings['gates']\n gate_width = gates[gate.__class__.__name__]['width']\n except KeyError:\n gate_width = .5\n return gate_width", "response": "Return the width of the given gate."} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn the offset to use before placing this gate.", "response": "def _gate_pre_offset(self, gate):\n \"\"\"\n Return the offset to use before placing this gate.\n\n :param string gate: The name of the gate whose pre-offset is desired.\n :return: Offset to use before the gate.\n :rtype: float\n \"\"\"\n try:\n gates = self.settings['gates']\n delta_pos = gates[gate.__class__.__name__]['pre_offset']\n except KeyError:\n delta_pos = self._gate_offset(gate)\n return delta_pos"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _gate_height(self, gate):\n try:\n height = self.settings['gates'][gate.__class__.__name__]['height']\n except KeyError:\n height = .5\n return height", "response": "Return the height to use for this gate."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _phase(self, line, pos):\n phase_str = \"\\n\\\\node[phase] ({}) at ({},-{}) {{}};\"\n return phase_str.format(self._op(line), pos, line)", "response": "Returns a string representing a phase of the control circle at a given position."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning the name of the gate that is placing a line.", "response": "def _op(self, line, op=None, offset=0):\n \"\"\"\n Returns the gate name for placing a gate on a line.\n\n :param int line: Line number.\n :param int op: Operation number or, by default, uses the current op count.\n :return: Gate name.\n :rtype: string\n \"\"\"\n if op is None:\n op = self.op_count[line]\n return \"line{}_gate{}\".format(line, op + offset)"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn the string representation of the line between two most recent gates.", "response": "def _line(self, p1, p2, line=None):\n \"\"\"\n Connects p1 and p2, where p1 and p2 are either to qubit line indices,\n in which case the two most recent gates are connected, or two gate\n indices, in which case line denotes the line number and the two gates\n are connected on the given line.\n\n :param int p1: Index of the first object to connect.\n :param int p2: Index of the second object to connect.\n :param int line: Line index - if provided, p1 and p2 are gate indices.\n :return: Latex code to draw this / these line(s).\n :rtype: string\n \"\"\"\n dbl_classical = self.settings['lines']['double_classical']\n\n if line is None:\n quantum = not dbl_classical or self.is_quantum[p1]\n op1, op2 = self._op(p1), self._op(p2)\n loc1, loc2 = 'north', 'south'\n shift = \"xshift={}cm\"\n else:\n quantum = not dbl_classical or self.is_quantum[line]\n op1, op2 = self._op(line, p1), self._op(line, p2)\n loc1, loc2 = 'west', 'east'\n shift = \"yshift={}cm\"\n\n if quantum:\n return \"\\n\\\\draw ({}) edge[edgestyle] ({});\".format(op1, op2)\n else:\n if p2 > p1:\n loc1, loc2 = loc2, loc1\n edge_str = \"\\n\\\\draw ([{shift}]{op1}.{loc1}) edge[edgestyle] ([{shift}]{op2}.{loc2});\"\n line_sep = self.settings['lines']['double_lines_sep']\n shift1 = shift.format(line_sep / 2.)\n shift2 = shift.format(-line_sep / 2.)\n edges_str = edge_str.format(shift=shift1, op1=op1, op2=op2, loc1=loc1, loc2=loc2)\n edges_str += edge_str.format(shift=shift2, op1=op1, op2=op2, loc1=loc1, loc2=loc2)\n return edges_str"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _regular_gate(self, gate, lines, ctrl_lines, used_lines):\n imax = max(lines)\n imin = min(lines)\n\n delta_pos = self._gate_offset(gate)\n gate_width = self._gate_width(gate)\n gate_height = self._gate_height(gate)\n\n name = self._gate_name(gate)\n\n lines = list(range(imin, imax + 1))\n\n tex_str = \"\"\n pos = self.pos[lines[0]]\n\n node_str = \"\\n\\\\node[none] ({}) at ({},-{}) {{}};\"\n for l in lines:\n node1 = node_str.format(self._op(l), pos, l)\n if l in used_lines:\n tex_str += self._phase(l, pos)\n node2 = (\"\\n\\\\node[none,minimum height={}cm,outer sep=0] ({}) at\"\n \" ({},-{}) {{}};\").format(gate_height, self._op(l, offset=1), pos + gate_width / 2., l)\n node3 = node_str.format(self._op(l, offset=2), pos + gate_width, l)\n tex_str += node1 + node2 + node3\n tex_str += (\"\\n\\\\draw[operator,edgestyle,outer sep={width}cm]\"\n \" ([yshift={half_height}cm]{op1})\"\n \" rectangle ([yshift=-{half_height}cm]{op2}) node[pos=.5]{{\\\\verb|{name}|}};\"\n ).format(width=gate_width, op1=self._op(imin), op2=self._op(imax, offset=2),\n half_height=.5 * gate_height, name=name)\n for l in lines:\n self.pos[l] = pos + gate_width / 2.\n self.op_count[l] += 3\n for l in range(min(ctrl_lines + lines), max(ctrl_lines + lines) + 1):\n self.pos[l] = pos + delta_pos + gate_width\n return tex_str", "response": "Draw a regular gate."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nsimulating a Quil program and return the wavefunction.", "response": "def wavefunction(self, quil_program: Program,\n memory_map: Any = None) -> Wavefunction:\n \"\"\"\n Simulate a Quil program and return the wavefunction.\n\n .. note:: If your program contains measurements or noisy gates, this method may not do what\n you want. If the execution of ``quil_program`` is **non-deterministic** then the\n final wavefunction only represents a stochastically generated sample and the\n wavefunctions returned by *different* ``wavefunction`` calls *will generally be\n different*.\n\n :param quil_program: A Quil program.\n :param memory_map: An assignment of classical registers to values, representing an initial\n state for the QAM's classical memory.\n\n This is expected to be of type Dict[str, List[Union[int, float]]],\n where the keys are memory region names and the values are arrays of\n initialization data.\n\n For now, we also support input of type Dict[MemoryReference, Any],\n but this is deprecated and will be removed in a future release.\n :return: A Wavefunction object representing the state of the QVM.\n \"\"\"\n\n if memory_map is not None:\n quil_program = self.augment_program_with_memory_values(quil_program, memory_map)\n\n return self.connection._wavefunction(quil_program=quil_program,\n random_seed=self.random_seed)"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef expectation(self, prep_prog: Program,\n pauli_terms: Union[PauliSum, List[PauliTerm]],\n memory_map: Any = None) -> Union[float, np.ndarray]:\n \"\"\"\n Calculate the expectation value of Pauli operators given a state prepared by prep_program.\n\n If ``pauli_terms`` is a ``PauliSum`` then the returned value is a single ``float``,\n otherwise the returned value is an array of values, one for each ``PauliTerm`` in the\n list.\n\n .. note:: If your program contains measurements or noisy gates, this method may not do what\n you want. If the execution of ``quil_program`` is **non-deterministic** then the\n final wavefunction from which the expectation value is calculated only represents\n a stochastically generated sample and the wavefunctions returned by *different*\n ``wavefunction`` calls *will generally be different*.\n\n :param prep_prog: A program that prepares the state on which we measure the expectation.\n :param pauli_terms: A Pauli representation of a quantum operator.\n :param memory_map: An assignment of classical registers to values, representing an initial\n state for the QAM's classical memory.\n\n This is expected to be of type Dict[str, List[Union[int, float]]],\n where the keys are memory region names and the values are arrays of\n initialization data.\n\n For now, we also support input of type Dict[MemoryReference, Any],\n but this is deprecated and will be removed in a future release.\n :return: Either a float or array floats depending on ``pauli_terms``.\n \"\"\"\n\n is_pauli_sum = False\n if isinstance(pauli_terms, PauliSum):\n progs, coeffs = pauli_terms.get_programs()\n is_pauli_sum = True\n else:\n coeffs = np.array([pt.coefficient for pt in pauli_terms])\n progs = [pt.program for pt in pauli_terms]\n\n if memory_map is not None:\n prep_prog = self.augment_program_with_memory_values(prep_prog, memory_map)\n\n bare_results = self.connection._expectation(prep_prog, progs, random_seed=self.random_seed)\n results = coeffs * bare_results\n if is_pauli_sum:\n return np.sum(results)\n return results", "response": "Calculates the expectation value of a Pauli operator given a state prepared by prep_prog and returns a numpy array of values depending on the state of the Pauli operator."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef run_and_measure(self, quil_program: Program, qubits: List[int] = None, trials: int = 1,\n memory_map: Any = None) -> np.ndarray:\n \"\"\"\n Run a Quil program once to determine the final wavefunction, and measure multiple times.\n\n Alternatively, consider using ``wavefunction`` and calling ``sample_bitstrings`` on the\n resulting object.\n\n For a large wavefunction and a low-medium number of trials, use this function.\n On the other hand, if you're sampling a small system many times you might want to\n use ``Wavefunction.sample_bitstrings``.\n\n .. note:: If your program contains measurements or noisy gates, this method may not do what\n you want. If the execution of ``quil_program`` is **non-deterministic** then the\n final wavefunction from which the returned bitstrings are sampled itself only\n represents a stochastically generated sample and the outcomes sampled from\n *different* ``run_and_measure`` calls *generally sample different bitstring\n distributions*.\n\n :param quil_program: The program to run and measure\n :param qubits: An optional list of qubits to measure. The order of this list is\n respected in the returned bitstrings. If not provided, all qubits used in\n the program will be measured and returned in their sorted order.\n :param int trials: Number of times to sample from the prepared wavefunction.\n :param memory_map: An assignment of classical registers to values, representing an initial\n state for the QAM's classical memory.\n\n This is expected to be of type Dict[str, List[Union[int, float]]],\n where the keys are memory region names and the values are arrays of\n initialization data.\n\n For now, we also support input of type Dict[MemoryReference, Any],\n but this is deprecated and will be removed in a future release.\n :return: An array of measurement results (0 or 1) of shape (trials, len(qubits))\n \"\"\"\n if qubits is None:\n qubits = sorted(quil_program.get_qubits(indices=True))\n\n if memory_map is not None:\n quil_program = self.augment_program_with_memory_values(quil_program, memory_map)\n\n return self.connection._run_and_measure(quil_program=quil_program, qubits=qubits,\n trials=trials,\n random_seed=self.random_seed)", "response": "Run a Quil program and measure the final wavefunction and return the measurement results."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn an object to identify dataclass fields.", "response": "def field(*, default=MISSING, default_factory=MISSING, init=True, repr=True,\n hash=None, compare=True, metadata=None):\n \"\"\"Return an object to identify dataclass fields.\n\n default is the default value of the field. default_factory is a\n 0-argument function called to initialize a field's value. If init\n is True, the field will be a parameter to the class's __init__()\n function. If repr is True, the field will be included in the\n object's repr(). If hash is True, the field will be included in\n the object's hash(). If compare is True, the field will be used\n in comparison functions. metadata, if specified, must be a\n mapping which is stored but not otherwise examined by dataclass.\n\n It is an error to specify both default and default_factory.\n \"\"\"\n\n if default is not MISSING and default_factory is not MISSING:\n raise ValueError('cannot specify both default and default_factory')\n return Field(default, default_factory, init, repr, hash, compare,\n metadata)"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef dataclass(_cls=None, *, init=True, repr=True, eq=True, order=False,\n unsafe_hash=False, frozen=False):\n \"\"\"Returns the same class as was passed in, with dunder methods\n added based on the fields defined in the class.\n\n Examines PEP 526 __annotations__ to determine fields.\n\n If init is true, an __init__() method is added to the class. If\n repr is true, a __repr__() method is added. If order is true, rich\n comparison dunder methods are added. If unsafe_hash is true, a\n __hash__() method function is added. If frozen is true, fields may\n not be assigned to after instance creation.\n \"\"\"\n\n def wrap(cls):\n return _process_class(cls, init, repr, eq, order, unsafe_hash, frozen)\n\n # See if we're being called as @dataclass or @dataclass().\n if _cls is None:\n # We're called with parens.\n return wrap\n\n # We're called as @dataclass without parens.\n return wrap(_cls)", "response": "Returns the same class as _cls but with dunder methods added based on the fields defined in the class."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a tuple describing the fields of this dataclass or an instance of one.", "response": "def fields(class_or_instance):\n \"\"\"Return a tuple describing the fields of this dataclass.\n\n Accepts a dataclass or an instance of one. Tuple elements are of\n type Field.\n \"\"\"\n\n # Might it be worth caching this, per class?\n try:\n fields = getattr(class_or_instance, _FIELDS)\n except AttributeError:\n raise TypeError('must be called with a dataclass type or instance')\n\n # Exclude pseudo-fields. Note that fields is sorted by insertion\n # order, so the order of the tuple is as the fields were defined.\n return tuple(f for f in fields.values() if f._field_type is _FIELD)"} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns the fields of a dataclass instance as a new dictionary mapping field names to field values.", "response": "def asdict(obj, *, dict_factory=dict):\n \"\"\"Return the fields of a dataclass instance as a new dictionary mapping\n field names to field values.\n\n Example usage:\n\n @dataclass\n class C:\n x: int\n y: int\n\n c = C(1, 2)\n assert asdict(c) == {'x': 1, 'y': 2}\n\n If given, 'dict_factory' will be used instead of built-in dict.\n The function applies recursively to field values that are\n dataclass instances. This will also look into built-in containers:\n tuples, lists, and dicts.\n \"\"\"\n if not _is_dataclass_instance(obj):\n raise TypeError(\"asdict() should be called on dataclass instances\")\n return _asdict_inner(obj, dict_factory)"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef astuple(obj, *, tuple_factory=tuple):\n\n if not _is_dataclass_instance(obj):\n raise TypeError(\"astuple() should be called on dataclass instances\")\n return _astuple_inner(obj, tuple_factory)", "response": "Return the fields of a dataclass instance as a new tuple of field values."} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncreate a new dataclass with the given name and fields.", "response": "def make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True,\n repr=True, eq=True, order=False, unsafe_hash=False,\n frozen=False):\n \"\"\"Return a new dynamically created dataclass.\n\n The dataclass name will be 'cls_name'. 'fields' is an iterable\n of either (name), (name, type) or (name, type, Field) objects. If type is\n omitted, use the string 'typing.Any'. Field objects are created by\n the equivalent of calling 'field(name, type [, Field-info])'.\n\n C = make_dataclass('C', ['x', ('y', int), ('z', int, field(init=False))], bases=(Base,))\n\n is equivalent to:\n\n @dataclass\n class C(Base):\n x: 'typing.Any'\n y: int\n z: int = field(init=False)\n\n For the bases and namespace parameters, see the builtin type() function.\n\n The parameters init, repr, eq, order, unsafe_hash, and frozen are passed to\n dataclass().\n \"\"\"\n\n if namespace is None:\n namespace = {}\n else:\n # Copy namespace since we're going to mutate it.\n namespace = namespace.copy()\n\n # While we're looking through the field names, validate that they\n # are identifiers, are not keywords, and not duplicates.\n seen = set()\n anns = {}\n for item in fields:\n if isinstance(item, str):\n name = item\n tp = 'typing.Any'\n elif len(item) == 2:\n name, tp, = item\n elif len(item) == 3:\n name, tp, spec = item\n namespace[name] = spec\n else:\n raise TypeError(f'Invalid field: {item!r}')\n\n if not isinstance(name, str) or not name.isidentifier():\n raise TypeError(f'Field names must be valid identifers: {name!r}')\n if keyword.iskeyword(name):\n raise TypeError(f'Field names must not be keywords: {name!r}')\n if name in seen:\n raise TypeError(f'Field name duplicated: {name!r}')\n\n seen.add(name)\n anns[name] = tp\n\n namespace['__annotations__'] = anns\n # We use `types.new_class()` instead of simply `type()` to allow dynamic creation\n # of generic dataclassses.\n cls = types.new_class(cls_name, bases, {}, lambda ns: ns.update(namespace))\n return dataclass(cls, init=init, repr=repr, eq=eq, order=order,\n unsafe_hash=unsafe_hash, frozen=frozen)"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns a new object with the specified fields replaced with new values.", "response": "def replace(obj, **changes):\n \"\"\"Return a new object replacing specified fields with new values.\n\n This is especially useful for frozen classes. Example usage:\n\n @dataclass(frozen=True)\n class C:\n x: int\n y: int\n\n c = C(1, 2)\n c1 = replace(c, x=3)\n assert c1.x == 3 and c1.y == 2\n \"\"\"\n\n # We're going to mutate 'changes', but that's okay because it's a\n # new dict, even if called with 'replace(obj, **my_changes)'.\n\n if not _is_dataclass_instance(obj):\n raise TypeError(\"replace() should be called on dataclass instances\")\n\n # It's an error to have init=False fields in 'changes'.\n # If a field is not in 'changes', read its value from the provided obj.\n\n for f in getattr(obj, _FIELDS).values():\n if not f.init:\n # Error if this field is specified in changes.\n if f.name in changes:\n raise ValueError(f'field {f.name} is declared with '\n 'init=False, it cannot be specified with '\n 'replace()')\n continue\n\n if f.name not in changes:\n changes[f.name] = getattr(obj, f.name)\n\n # Create the new object, which calls __init__() and\n # __post_init__() (if defined), using all of the init fields we've\n # added and/or left in 'changes'. If there are values supplied in\n # changes that aren't fields, this will correctly raise a\n # TypeError.\n return obj.__class__(**changes)"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ngenerates a randomized benchmarking experiment on the given qubits.", "response": "def generate_rb_sequence(self, depth: int, gateset: List[Gate],\n seed: int = None, interleaver: Program = None) -> List[Program]:\n \"\"\"\n Construct a randomized benchmarking experiment on the given qubits, decomposing into\n gateset. If interleaver is not provided, the returned sequence will have the form\n\n C_1 C_2 ... C_(depth-1) C_inv ,\n\n where each C is a Clifford element drawn from gateset, C_{< depth} are randomly selected,\n and C_inv is selected so that the entire sequence composes to the identity. If an\n interleaver G (which must be a Clifford, and which will be decomposed into the native\n gateset) is provided, then the sequence instead takes the form\n\n C_1 G C_2 G ... C_(depth-1) G C_inv .\n\n The JSON response is a list of lists of indices, or Nones. In the former case, they are the\n index of the gate in the gateset.\n\n :param int depth: The number of Clifford gates to include in the randomized benchmarking\n experiment. This is different than the number of gates in the resulting experiment.\n :param list gateset: A list of pyquil gates to decompose the Clifford elements into. These\n must generate the clifford group on the qubits of interest. e.g. for one qubit\n [RZ(np.pi/2), RX(np.pi/2)].\n :param seed: A positive integer used to seed the PRNG.\n :param interleaver: A Program object that encodes a Clifford element.\n :return: A list of pyquil programs. Each pyquil program is a circuit that represents an\n element of the Clifford group. When these programs are composed, the resulting Program\n will be the randomized benchmarking experiment of the desired depth. e.g. if the return\n programs are called cliffords then `sum(cliffords, Program())` will give the randomized\n benchmarking experiment, which will compose to the identity program.\n \"\"\""} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef qaoa_ansatz(gammas, betas):\n return Program([exponentiate_commuting_pauli_sum(h_cost)(g)\n + exponentiate_commuting_pauli_sum(h_driver)(b)\n for g, b in zip(gammas, betas)])", "response": "Function that returns a QAOA ansatz program for a list of angles betas and gammas."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef list_devices(connection: ForestConnection = None):\n # For the record, the dictionary stored in \"devices\" that we're getting back is keyed on device\n # names and has this structure in its values:\n #\n # {\n # \"is_online\": a boolean indicating the availability of the device,\n # \"is_retuning\": a boolean indicating whether the device is busy retuning,\n # \"specs\": a Specs object describing the entire device, serialized as a dictionary,\n # \"isa\": an ISA object describing the entire device, serialized as a dictionary,\n # \"noise_model\": a NoiseModel object describing the entire device, serialized as a dictionary\n # }\n if connection is None:\n connection = ForestConnection()\n\n session = connection.session\n url = connection.forest_cloud_endpoint + '/devices'\n return sorted(get_json(session, url)[\"devices\"].keys())", "response": "Query the Forest 2. 0 server for a list of underlying QPU devices."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef list_lattices(device_name: str = None, num_qubits: int = None,\n connection: ForestConnection = None):\n \"\"\"\n Query the Forest 2.0 server for its knowledge of lattices. Optionally filters by underlying\n device name and lattice qubit count.\n\n :return: A dictionary keyed on lattice names and valued in dictionaries of the form\n {\n \"device_name\": device_name,\n \"qubits\": num_qubits\n }\n \"\"\"\n if connection is None:\n connection = ForestConnection()\n session = connection.session\n url = connection.forest_cloud_endpoint + \"/lattices\"\n try:\n response = get_json(session, url,\n params={\"device_name\": device_name,\n \"num_qubits\": num_qubits})\n\n return response[\"lattices\"]\n except Exception as e:\n raise ValueError(\"\"\"\n list_lattices encountered an error when querying the Forest 2.0 endpoint.\n\n Some common causes for this error include:\n\n * You don't have valid user authentication information. Very likely this is because you\n haven't yet been invited to try QCS. We plan on making our device information publicly\n accessible soon, but in the meanwhile, you'll have to use default QVM configurations and\n to use `list_quantum_computers` with `qpus = False`.\n\n * You do have user authentication information, but it is missing or modified. You can find\n this either in the environment variables FOREST_API_KEY and FOREST_USER_ID or in the\n config file (stored by default at ~/.qcs_config, but with location settable through the\n environment variable QCS_CONFIG), which contains the subsection\n\n [Rigetti Forest]\n user_id = your_user_id\n key = your_api_key\n\n * You're missing an address for the Forest 2.0 server endpoint, or the address is invalid.\n This too can be set through the environment variable FOREST_URL or by changing the\n following lines in the QCS config file:\n\n [Rigetti Forest]\n url = https://forest-server.qcs.rigetti.com\n\n For the record, here's the original exception: {}\n \"\"\".format(repr(e)))", "response": "Query the Forest 2. 0 server for its knowledge of lattices."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _get_raw_lattice_data(lattice_name: str = None):\n from pyquil.api._base_connection import get_session, get_json\n from requests.exceptions import MissingSchema\n session = get_session()\n config = PyquilConfig()\n\n try:\n res = get_json(session, f\"{config.forest_url}/lattices/{lattice_name}\")\n except MissingSchema:\n raise ValueError(f\"Error finding lattice `{lattice_name}` at Forest 2.0 server \"\n f\"\"\"endpoint `{config.forest_url}`.\n\n Most likely, you're missing an address for the Forest 2.0 server endpoint, or the\n address is invalid. This can be set through the environment variable FOREST_URL or\n by changing the following lines in the QCS config file (by default, at ~/.qcs_config):\n\n [Rigetti Forest]\n url = https://rigetti.com/valid/forest/url\"\"\")\n return res[\"lattice\"]", "response": "Returns a dictionary of raw data for a single lattice."} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns the bitstring corresponding to the given index in lexical order that corresponds to the given qubit_num.", "response": "def get_bitstring_from_index(index, qubit_num):\n \"\"\"\n Returns the bitstring in lexical order that corresponds to the given index in 0 to 2^(qubit_num)\n :param int index:\n :param int qubit_num:\n :return: the bitstring\n :rtype: str\n \"\"\"\n if index > (2**qubit_num - 1):\n raise IndexError(\"Index {} too large for {} qubits.\".format(index, qubit_num))\n return bin(index)[2:].rjust(qubit_num, '0')"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nround up the next multiple.", "response": "def _round_to_next_multiple(n, m):\n \"\"\"\n Round up the the next multiple.\n\n :param n: The number to round up.\n :param m: The multiple.\n :return: The rounded number\n \"\"\"\n return n if n % m == 0 else n + m - n % m"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _octet_bits(o):\n if not isinstance(o, integer_types):\n raise TypeError(\"o should be an int\")\n if not (0 <= o <= 255):\n raise ValueError(\"o should be between 0 and 255 inclusive\")\n bits = [0] * 8\n for i in range(8):\n if 1 == o & 1:\n bits[i] = 1\n o = o >> 1\n return bits", "response": "Get the bits of an octet in LSB - to - MSB order."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nparses a bit packed string and return a wavefunction object.", "response": "def from_bit_packed_string(coef_string):\n \"\"\"\n From a bit packed string, unpacks to get the wavefunction\n :param bytes coef_string:\n :return:\n \"\"\"\n num_octets = len(coef_string)\n\n # Parse the wavefunction\n wf = np.zeros(num_octets // OCTETS_PER_COMPLEX_DOUBLE, dtype=np.cfloat)\n for i, p in enumerate(range(0, num_octets, OCTETS_PER_COMPLEX_DOUBLE)):\n re_be = coef_string[p: p + OCTETS_PER_DOUBLE_FLOAT]\n im_be = coef_string[p + OCTETS_PER_DOUBLE_FLOAT: p + OCTETS_PER_COMPLEX_DOUBLE]\n re = struct.unpack('>d', re_be)[0]\n im = struct.unpack('>d', im_be)[0]\n wf[i] = complex(re, im)\n\n return Wavefunction(wf)"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nparse a wavefunction array of complex amplitudes and returns a dictionary of outcomes and associated probabilities.", "response": "def get_outcome_probs(self):\n \"\"\"\n Parses a wavefunction (array of complex amplitudes) and returns a dictionary of\n outcomes and associated probabilities.\n\n :return: A dict with outcomes as keys and probabilities as values.\n :rtype: dict\n \"\"\"\n outcome_dict = {}\n qubit_num = len(self)\n for index, amplitude in enumerate(self.amplitudes):\n outcome = get_bitstring_from_index(index, qubit_num)\n outcome_dict[outcome] = abs(amplitude) ** 2\n return outcome_dict"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef pretty_print_probabilities(self, decimal_digits=2):\n outcome_dict = {}\n qubit_num = len(self)\n for index, amplitude in enumerate(self.amplitudes):\n outcome = get_bitstring_from_index(index, qubit_num)\n prob = round(abs(amplitude) ** 2, decimal_digits)\n if prob != 0.:\n outcome_dict[outcome] = prob\n return outcome_dict", "response": "Prints outcome probabilities ignoring all outcomes with approximately zero probabilities\n and rounding the probabilities to decimal_digits."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef pretty_print(self, decimal_digits=2):\n outcome_dict = {}\n qubit_num = len(self)\n pp_string = \"\"\n for index, amplitude in enumerate(self.amplitudes):\n outcome = get_bitstring_from_index(index, qubit_num)\n amplitude = round(amplitude.real, decimal_digits) + \\\n round(amplitude.imag, decimal_digits) * 1.j\n if amplitude != 0.:\n outcome_dict[outcome] = amplitude\n pp_string += str(amplitude) + \"|{}> + \".format(outcome)\n if len(pp_string) >= 3:\n pp_string = pp_string[:-3] # remove the dangling + if it is there\n return pp_string", "response": "Returns a string repr of the wavefunction ignoring all outcomes with approximately zero and rounding the amplitudes to a certain number of decimal digits."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef plot(self, qubit_subset=None):\n import matplotlib.pyplot as plt\n prob_dict = self.get_outcome_probs()\n if qubit_subset:\n sub_dict = {}\n qubit_num = len(self)\n for i in qubit_subset:\n if i > (2**qubit_num - 1):\n raise IndexError(\"Index {} too large for {} qubits.\".format(i, qubit_num))\n else:\n sub_dict[get_bitstring_from_index(i, qubit_num)] = prob_dict[get_bitstring_from_index(i, qubit_num)]\n prob_dict = sub_dict\n plt.bar(range(len(prob_dict)), prob_dict.values(), align='center', color='#6CAFB7')\n plt.xticks(range(len(prob_dict)), prob_dict.keys())\n plt.show()", "response": "Plots a bar chart with bitstring on the x axis and probability on the y axis."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nsamples bitstrings from the distribution defined by the wavefunction.", "response": "def sample_bitstrings(self, n_samples):\n \"\"\"\n Sample bitstrings from the distribution defined by the wavefunction.\n\n :param n_samples: The number of bitstrings to sample\n :return: An array of shape (n_samples, n_qubits)\n \"\"\"\n possible_bitstrings = np.array(list(itertools.product((0, 1), repeat=len(self))))\n inds = np.random.choice(2 ** len(self), n_samples, p=self.probabilities())\n bitstrings = possible_bitstrings[inds, :]\n return bitstrings"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_default_settings():\n settings = dict()\n settings['gate_shadow'] = True\n settings['lines'] = ({'style': 'very thin', 'double_classical': True,\n 'init_quantum': True, 'double_lines_sep': .04})\n settings['gates'] = ({'HGate': {'width': .5, 'offset': .3, 'pre_offset': .1},\n 'XGate': {'width': .35, 'height': .35, 'offset': .1},\n 'SwapGate': {'width': .35, 'height': .35, 'offset': .1},\n 'Rx': {'width': 1., 'height': .8, 'pre_offset': .2, 'offset': .3},\n 'Ry': {'width': 1., 'height': .8, 'pre_offset': .2, 'offset': .3},\n 'Rz': {'width': 1., 'height': .8, 'pre_offset': .2, 'offset': .3},\n 'EntangleGate': {'width': 1.8, 'offset': .2, 'pre_offset': .2},\n 'DeallocateQubitGate': {'height': .15, 'offset': .2, 'width': .2, 'pre_offset': .1},\n 'AllocateQubitGate': {'height': .15, 'width': .2, 'offset': .1, 'pre_offset': .1,\n 'draw_id': False},\n 'MeasureGate': {'width': 0.75, 'offset': .2, 'height': .5, 'pre_offset': .2}})\n settings['control'] = {'size': .1, 'shadow': False}\n return settings", "response": "Returns the default settings for generating LaTeX circuits."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef header(settings):\n packages = (r\"\\documentclass[convert={density=300,outext=.png}]{standalone}\",\n r\"\\usepackage[margin=1in]{geometry}\",\n r\"\\usepackage[hang,small,bf]{caption}\",\n r\"\\usepackage{tikz}\",\n r\"\\usepackage{braket}\",\n r\"\\usetikzlibrary{backgrounds,shadows.blur,fit,decorations.pathreplacing,shapes}\")\n\n init = (r\"\\begin{document}\",\n r\"\\begin{tikzpicture}[scale=0.8, transform shape]\")\n\n gate_style = (r\"\\tikzstyle{basicshadow}=\"\n r\"[blur shadow={shadow blur steps=8, shadow xshift=0.7pt, shadow yshift=-0.7pt, shadow scale=1.02}]\")\n\n if not (settings.get('gate_shadow') or settings.get('control', {}).get('shadow')):\n gate_style = \"\"\n\n gate_style += r\"\\tikzstyle{basic}=[draw,fill=white,\"\n if settings.get('gate_shadow'):\n gate_style += \"basicshadow\"\n gate_style += \"]\\n\"\n\n gate_style += (\"\\\\tikzstyle{{operator}}=[basic,minimum size=1.5em]\\n\\\\tikzstyle{{phase}}=[fill=black,shape=circle,\"\n \"minimum size={}\"\n \"cm,inner sep=0pt,outer sep=0pt,draw=black\").format(settings.get('control', {}).get('size', 0))\n if settings.get('control', {}).get('shadow'):\n gate_style += \",basicshadow\"\n gate_style += (\"]\\n\\\\tikzstyle{none}=[inner sep=0pt,outer sep=-.5pt,\"\n \"minimum height=0.5cm+1pt]\\n\"\n \"\\\\tikzstyle{measure}=[operator,inner sep=0pt,minimum \"\n + \"height={}cm, minimum width={}cm]\\n\".format(\n settings.get('gates', {}).get('MeasureGate', {}).get('height', 0),\n settings.get('gates', {}).get('MeasureGate', {}).get('width', 0))\n + \"\\\\tikzstyle{xstyle}=[circle,basic,minimum height=\")\n x_gate_radius = min(settings.get('gates', {}).get('XGate', {}).get('height', 0),\n settings.get('gates', {}).get('XGate', {}).get('width', 0))\n gate_style += (\"{x_rad}cm,minimum width={x_rad}cm,inner sep=0pt,{linestyle}]\\n\").format(\n x_rad=x_gate_radius,\n linestyle=settings.get('lines', {}).get('style', \"\"))\n if settings.get('gate_shadow'):\n gate_style += (\"\\\\tikzset{\\nshadowed/.style={preaction={transform canvas={shift={(0.5pt,-0.5pt)}},\"\n \" draw=gray, opacity=0.4}},\\n}\\n\")\n gate_style += \"\\\\tikzstyle{swapstyle}=[inner sep=-1pt, outer sep=-1pt, minimum width=0pt]\"\n edge_style = (\"\\\\tikzstyle{edgestyle}=[\" + settings.get('lines', {}).get('style', \"\") + \"]\")\n return \"\\n\".join((\"\\n\".join(packages), \"\\n\".join(init), gate_style, edge_style))", "response": "Writes the header of the LaTeX document."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _check_kraus_ops(n, kraus_ops):\n for k in kraus_ops:\n if not np.shape(k) == (2 ** n, 2 ** n):\n raise ValueError(\n \"Kraus operators for {0} qubits must have shape {1}x{1}: {2}\".format(n, 2 ** n, k))\n\n kdk_sum = sum(np.transpose(k).conjugate().dot(k) for k in kraus_ops)\n if not np.allclose(kdk_sum, np.eye(2 ** n), atol=1e-3):\n raise ValueError(\n \"Kraus operator not correctly normalized: sum_j K_j^*K_j == {}\".format(kdk_sum))", "response": "Verify that the Kraus operators are of the correct shape and satisfy the correct normalization."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _create_kraus_pragmas(name, qubit_indices, kraus_ops):\n\n pragmas = [Pragma(\"ADD-KRAUS\",\n [name] + list(qubit_indices),\n \"({})\".format(\" \".join(map(format_parameter, np.ravel(k)))))\n for k in kraus_ops]\n return pragmas", "response": "Generates the pragmas to define a Kraus map for a specific gate on some qubits."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef pauli_kraus_map(probabilities):\n if len(probabilities) not in [4, 16]:\n raise ValueError(\"Currently we only support one or two qubits, \"\n \"so the provided list of probabilities must have length 4 or 16.\")\n if not np.allclose(sum(probabilities), 1.0, atol=1e-3):\n raise ValueError(\"Probabilities must sum to one.\")\n\n paulis = [np.eye(2), np.array([[0, 1], [1, 0]]), np.array([[0, -1j], [1j, 0]]), np.array([[1, 0], [0, -1]])]\n\n if len(probabilities) == 4:\n operators = paulis\n else:\n operators = np.kron(paulis, paulis)\n\n return [coeff * op for coeff, op in zip(np.sqrt(probabilities), operators)]", "response": "r Generate the Kraus operators corresponding to a pauli channel."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef damping_kraus_map(p=0.10):\n damping_op = np.sqrt(p) * np.array([[0, 1],\n [0, 0]])\n\n residual_kraus = np.diag([1, np.sqrt(1 - p)])\n return [residual_kraus, damping_op]", "response": "Generates the Kraus operators corresponding to an amplitude damping\n noise channel."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef dephasing_kraus_map(p=0.10):\n return [np.sqrt(1 - p) * np.eye(2), np.sqrt(p) * np.diag([1, -1])]", "response": "Generates the Kraus operators corresponding to a dephasing channel."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef tensor_kraus_maps(k1, k2):\n return [np.kron(k1j, k2l) for k1j in k1 for k2l in k2]", "response": "Generates the Kraus map corresponding to the composition\n of two maps on different qubits."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef combine_kraus_maps(k1, k2):\n return [np.dot(k1j, k2l) for k1j in k1 for k2l in k2]", "response": "Generates the Kraus map corresponding to the composition\n of two Kraus operators."} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ngenerate the Kraus map corresponding to the composition followed by an amplitude damping channel followed by an amplitude damping channel.", "response": "def damping_after_dephasing(T1, T2, gate_time):\n \"\"\"\n Generate the Kraus map corresponding to the composition\n of a dephasing channel followed by an amplitude damping channel.\n\n :param float T1: The amplitude damping time\n :param float T2: The dephasing time\n :param float gate_time: The gate duration.\n :return: A list of Kraus operators.\n \"\"\"\n assert T1 >= 0\n assert T2 >= 0\n\n if T1 != INFINITY:\n damping = damping_kraus_map(p=1 - np.exp(-float(gate_time) / float(T1)))\n else:\n damping = [np.eye(2)]\n\n if T2 != INFINITY:\n gamma_phi = float(gate_time) / float(T2)\n if T1 != INFINITY:\n if T2 > 2 * T1:\n raise ValueError(\"T2 is upper bounded by 2 * T1\")\n gamma_phi -= float(gate_time) / float(2 * T1)\n\n dephasing = dephasing_kraus_map(p=.5 * (1 - np.exp(-2 * gamma_phi)))\n else:\n dephasing = [np.eye(2)]\n return combine_kraus_maps(damping, dephasing)"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning the matrix and noisy name for the ideal gate.", "response": "def get_noisy_gate(gate_name, params):\n \"\"\"\n Look up the numerical gate representation and a proposed 'noisy' name.\n\n :param str gate_name: The Quil gate name\n :param Tuple[float] params: The gate parameters.\n :return: A tuple (matrix, noisy_name) with the representation of the ideal gate matrix\n and a proposed name for the noisy version.\n :rtype: Tuple[np.array, str]\n \"\"\"\n params = tuple(params)\n if gate_name == \"I\":\n assert params == ()\n return np.eye(2), \"NOISY-I\"\n if gate_name == \"RX\":\n angle, = params\n if np.isclose(angle, np.pi / 2, atol=ANGLE_TOLERANCE):\n return (np.array([[1, -1j],\n [-1j, 1]]) / np.sqrt(2),\n \"NOISY-RX-PLUS-90\")\n elif np.isclose(angle, -np.pi / 2, atol=ANGLE_TOLERANCE):\n return (np.array([[1, 1j],\n [1j, 1]]) / np.sqrt(2),\n \"NOISY-RX-MINUS-90\")\n elif np.isclose(angle, np.pi, atol=ANGLE_TOLERANCE):\n return (np.array([[0, -1j],\n [-1j, 0]]),\n \"NOISY-RX-PLUS-180\")\n elif np.isclose(angle, -np.pi, atol=ANGLE_TOLERANCE):\n return (np.array([[0, 1j],\n [1j, 0]]),\n \"NOISY-RX-MINUS-180\")\n elif gate_name == \"CZ\":\n assert params == ()\n return np.diag([1, 1, 1, -1]), \"NOISY-CZ\"\n raise NoisyGateUndefined(\"Undefined gate and params: {}{}\\n\"\n \"Please restrict yourself to I, RX(+/-pi), RX(+/-pi/2), CZ\"\n .format(gate_name, params))"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ngetting all Gates appearing in a Program.", "response": "def _get_program_gates(prog):\n \"\"\"\n Get all gate applications appearing in prog.\n\n :param Program prog: The program\n :return: A list of all Gates in prog (without duplicates).\n :rtype: List[Gate]\n \"\"\"\n return sorted({i for i in prog if isinstance(i, Gate)}, key=lambda g: g.out())"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _decoherence_noise_model(gates, T1=30e-6, T2=30e-6, gate_time_1q=50e-9,\n gate_time_2q=150e-09, ro_fidelity=0.95):\n \"\"\"\n The default noise parameters\n\n - T1 = 30 us\n - T2 = 30 us\n - 1q gate time = 50 ns\n - 2q gate time = 150 ns\n\n are currently typical for near-term devices.\n\n This function will define new gates and add Kraus noise to these gates. It will translate\n the input program to use the noisy version of the gates.\n\n :param Sequence[Gate] gates: The gates to provide the noise model for.\n :param Union[Dict[int,float],float] T1: The T1 amplitude damping time either globally or in a\n dictionary indexed by qubit id. By default, this is 30 us.\n :param Union[Dict[int,float],float] T2: The T2 dephasing time either globally or in a\n dictionary indexed by qubit id. By default, this is also 30 us.\n :param float gate_time_1q: The duration of the one-qubit gates, namely RX(+pi/2) and RX(-pi/2).\n By default, this is 50 ns.\n :param float gate_time_2q: The duration of the two-qubit gates, namely CZ.\n By default, this is 150 ns.\n :param Union[Dict[int,float],float] ro_fidelity: The readout assignment fidelity\n :math:`F = (p(0|0) + p(1|1))/2` either globally or in a dictionary indexed by qubit id.\n :return: A NoiseModel with the appropriate Kraus operators defined.\n \"\"\"\n all_qubits = set(sum(([t.index for t in g.qubits] for g in gates), []))\n if isinstance(T1, dict):\n all_qubits.update(T1.keys())\n if isinstance(T2, dict):\n all_qubits.update(T2.keys())\n if isinstance(ro_fidelity, dict):\n all_qubits.update(ro_fidelity.keys())\n\n if not isinstance(T1, dict):\n T1 = {q: T1 for q in all_qubits}\n\n if not isinstance(T2, dict):\n T2 = {q: T2 for q in all_qubits}\n\n if not isinstance(ro_fidelity, dict):\n ro_fidelity = {q: ro_fidelity for q in all_qubits}\n\n noisy_identities_1q = {\n q: damping_after_dephasing(T1.get(q, INFINITY), T2.get(q, INFINITY), gate_time_1q)\n for q in all_qubits\n }\n noisy_identities_2q = {\n q: damping_after_dephasing(T1.get(q, INFINITY), T2.get(q, INFINITY), gate_time_2q)\n for q in all_qubits\n }\n kraus_maps = []\n for g in gates:\n targets = tuple(t.index for t in g.qubits)\n key = (g.name, tuple(g.params))\n if g.name in NO_NOISE:\n continue\n matrix, _ = get_noisy_gate(g.name, g.params)\n\n if len(targets) == 1:\n noisy_I = noisy_identities_1q[targets[0]]\n else:\n if len(targets) != 2:\n raise ValueError(\"Noisy gates on more than 2Q not currently supported\")\n\n # note this ordering of the tensor factors is necessary due to how the QVM orders\n # the wavefunction basis\n noisy_I = tensor_kraus_maps(noisy_identities_2q[targets[1]],\n noisy_identities_2q[targets[0]])\n kraus_maps.append(KrausModel(g.name, tuple(g.params), targets,\n combine_kraus_maps(noisy_I, [matrix]),\n # FIXME (Nik): compute actual avg gate fidelity for this simple\n # noise model\n 1.0))\n aprobs = {}\n for q, f_ro in ro_fidelity.items():\n aprobs[q] = np.array([[f_ro, 1. - f_ro],\n [1. - f_ro, f_ro]])\n\n return NoiseModel(kraus_maps, aprobs)", "response": "This function returns a NoiseModel object that can be used to generate the decoherence noise for a set of gates."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _noise_model_program_header(noise_model):\n from pyquil.quil import Program\n p = Program()\n defgates = set()\n for k in noise_model.gates:\n\n # obtain ideal gate matrix and new, noisy name by looking it up in the NOISY_GATES dict\n try:\n ideal_gate, new_name = get_noisy_gate(k.gate, tuple(k.params))\n\n # if ideal version of gate has not yet been DEFGATE'd, do this\n if new_name not in defgates:\n p.defgate(new_name, ideal_gate)\n defgates.add(new_name)\n except NoisyGateUndefined:\n print(\"WARNING: Could not find ideal gate definition for gate {}\".format(k.gate),\n file=sys.stderr)\n new_name = k.gate\n\n # define noisy version of gate on specific targets\n p.define_noisy_gate(new_name, k.targets, k.kraus_ops)\n\n # define noisy readouts\n for q, ap in noise_model.assignment_probs.items():\n p.define_noisy_readout(q, p00=ap[0, 0], p11=ap[1, 1])\n return p", "response": "Generates the header for a noise model program."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef apply_noise_model(prog, noise_model):\n new_prog = _noise_model_program_header(noise_model)\n for i in prog:\n if isinstance(i, Gate):\n try:\n _, new_name = get_noisy_gate(i.name, tuple(i.params))\n new_prog += Gate(new_name, [], i.qubits)\n except NoisyGateUndefined:\n new_prog += i\n else:\n new_prog += i\n return new_prog", "response": "Applies a noise model to a program and returns a new program with the noisy gateset translated to a noisy - fied version of the program."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef add_decoherence_noise(prog, T1=30e-6, T2=30e-6, gate_time_1q=50e-9, gate_time_2q=150e-09,\n ro_fidelity=0.95):\n \"\"\"\n Add generic damping and dephasing noise to a program.\n\n This high-level function is provided as a convenience to investigate the effects of a\n generic noise model on a program. For more fine-grained control, please investigate\n the other methods available in the ``pyquil.noise`` module.\n\n In an attempt to closely model the QPU, noisy versions of RX(+-pi/2) and CZ are provided;\n I and parametric RZ are noiseless, and other gates are not allowed. To use this function,\n you need to compile your program to this native gate set.\n\n The default noise parameters\n\n - T1 = 30 us\n - T2 = 30 us\n - 1q gate time = 50 ns\n - 2q gate time = 150 ns\n\n are currently typical for near-term devices.\n\n This function will define new gates and add Kraus noise to these gates. It will translate\n the input program to use the noisy version of the gates.\n\n :param prog: A pyquil program consisting of I, RZ, CZ, and RX(+-pi/2) instructions\n :param Union[Dict[int,float],float] T1: The T1 amplitude damping time either globally or in a\n dictionary indexed by qubit id. By default, this is 30 us.\n :param Union[Dict[int,float],float] T2: The T2 dephasing time either globally or in a\n dictionary indexed by qubit id. By default, this is also 30 us.\n :param float gate_time_1q: The duration of the one-qubit gates, namely RX(+pi/2) and RX(-pi/2).\n By default, this is 50 ns.\n :param float gate_time_2q: The duration of the two-qubit gates, namely CZ.\n By default, this is 150 ns.\n :param Union[Dict[int,float],float] ro_fidelity: The readout assignment fidelity\n :math:`F = (p(0|0) + p(1|1))/2` either globally or in a dictionary indexed by qubit id.\n :return: A new program with noisy operators.\n \"\"\"\n gates = _get_program_gates(prog)\n noise_model = _decoherence_noise_model(\n gates,\n T1=T1,\n T2=T2,\n gate_time_1q=gate_time_1q,\n gate_time_2q=gate_time_2q,\n ro_fidelity=ro_fidelity\n )\n return apply_noise_model(prog, noise_model)", "response": "This function adds generic damping and dephasing noise to a program."} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nensuring that an array p with bitstring probabilities has a separate axis for each qubit such that p [ i j... k ) gives the estimated probability of bitstring i.", "response": "def _bitstring_probs_by_qubit(p):\n \"\"\"\n Ensure that an array ``p`` with bitstring probabilities has a separate axis for each qubit such\n that ``p[i,j,...,k]`` gives the estimated probability of bitstring ``ij...k``.\n\n This should not allocate much memory if ``p`` is already in ``C``-contiguous order (row-major).\n\n :param np.array p: An array that enumerates bitstring probabilities. When flattened out\n ``p = [p_00...0, p_00...1, ...,p_11...1]``. The total number of elements must therefore be a\n power of 2.\n :return: A reshaped view of ``p`` with a separate length-2 axis for each bit.\n \"\"\"\n p = np.asarray(p, order=\"C\")\n num_qubits = int(round(np.log2(p.size)))\n return p.reshape((2,) * num_qubits)"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nestimates the probability distribution over all bitstrings.", "response": "def estimate_bitstring_probs(results):\n \"\"\"\n Given an array of single shot results estimate the probability distribution over all bitstrings.\n\n :param np.array results: A 2d array where the outer axis iterates over shots\n and the inner axis over bits.\n :return: An array with as many axes as there are qubit and normalized such that it sums to one.\n ``p[i,j,...,k]`` gives the estimated probability of bitstring ``ij...k``.\n :rtype: np.array\n \"\"\"\n nshots, nq = np.shape(results)\n outcomes = np.array([int(\"\".join(map(str, r)), 2) for r in results])\n probs = np.histogram(outcomes, bins=np.arange(-.5, 2 ** nq, 1))[0] / float(nshots)\n return _bitstring_probs_by_qubit(probs)"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ngiving a 2d array of single shot results p and a list of bit - index matrices ts applies local 2x2 transforms to each bit index.", "response": "def _apply_local_transforms(p, ts):\n \"\"\"\n Given a 2d array of single shot results (outer axis iterates over shots, inner axis over bits)\n and a list of assignment probability matrices (one for each bit in the readout, ordered like\n the inner axis of results) apply local 2x2 matrices to each bit index.\n\n :param np.array p: An array that enumerates a function indexed by bitstrings::\n\n f(ijk...) = p[i,j,k,...]\n\n :param Sequence[np.array] ts: A sequence of 2x2 transform-matrices, one for each bit.\n :return: ``p_transformed`` an array with as many dimensions as there are bits with the result of\n contracting p along each axis by the corresponding bit transformation.\n\n p_transformed[ijk...] = f'(ijk...) = sum_lmn... ts[0][il] ts[1][jm] ts[2][kn] f(lmn...)\n\n :rtype: np.array\n \"\"\"\n p_corrected = _bitstring_probs_by_qubit(p)\n nq = p_corrected.ndim\n for idx, trafo_idx in enumerate(ts):\n\n # this contraction pattern looks like\n # 'ij,abcd...jklm...->abcd...iklm...' so it properly applies a \"local\"\n # transformation to a single tensor-index without changing the order of\n # indices\n einsum_pat = ('ij,' + _CHARS[:idx] + 'j' + _CHARS[idx:nq - 1]\n + '->' + _CHARS[:idx] + 'i' + _CHARS[idx:nq - 1])\n p_corrected = np.einsum(einsum_pat, trafo_idx, p_corrected)\n\n return p_corrected"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef correct_bitstring_probs(p, assignment_probabilities):\n return _apply_local_transforms(p, (np.linalg.inv(ap) for ap in assignment_probabilities))", "response": "Given a 2d array of corrupted bitstring probabilities and a list of assignment probability matrices compute the corrected probabilities for each measured bitstring."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef bitstring_probs_to_z_moments(p):\n zmat = np.array([[1, 1],\n [1, -1]])\n return _apply_local_transforms(p, (zmat for _ in range(p.ndim)))", "response": "Convert between bitstring probabilities and joint Z moment expectations."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nestimate the readout assignment probabilities for a given qubit.", "response": "def estimate_assignment_probs(q, trials, cxn, p0=None):\n \"\"\"\n Estimate the readout assignment probabilities for a given qubit ``q``.\n The returned matrix is of the form::\n\n [[p00 p01]\n [p10 p11]]\n\n :param int q: The index of the qubit.\n :param int trials: The number of samples for each state preparation.\n :param Union[QVMConnection,QPUConnection] cxn: The quantum abstract machine to sample from.\n :param Program p0: A header program to prepend to the state preparation programs.\n :return: The assignment probability matrix\n :rtype: np.array\n \"\"\"\n from pyquil.quil import Program\n if p0 is None: # pragma no coverage\n p0 = Program()\n results_i = np.sum(cxn.run(p0 + Program(I(q), MEASURE(q, 0)), [0], trials))\n results_x = np.sum(cxn.run(p0 + Program(X(q), MEASURE(q, 0)), [0], trials))\n\n p00 = 1. - results_i / float(trials)\n p11 = results_x / float(trials)\n return np.array([[p00, 1 - p11],\n [1 - p00, p11]])"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef unpack_kraus_matrix(m):\n m = np.asarray(m, dtype=complex)\n if m.ndim == 3:\n m = m[0] + 1j * m[1]\n if not m.ndim == 2: # pragma no coverage\n raise ValueError(\"Need 2d array.\")\n if not m.shape[0] == m.shape[1]: # pragma no coverage\n raise ValueError(\"Need square matrix.\")\n return m", "response": "Helper to optionally unpack a JSON - compatible representation of a complex Kraus matrix."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn a dictionary representation of a KrausModel.", "response": "def to_dict(self):\n \"\"\"\n Create a dictionary representation of a KrausModel.\n\n For example::\n\n {\n \"gate\": \"RX\",\n \"params\": np.pi,\n \"targets\": [0],\n \"kraus_ops\": [ # In this example single Kraus op = ideal RX(pi) gate\n [[[0, 0], # element-wise real part of matrix\n [0, 0]],\n [[0, -1], # element-wise imaginary part of matrix\n [-1, 0]]]\n ],\n \"fidelity\": 1.0\n }\n\n :return: A JSON compatible dictionary representation.\n :rtype: Dict[str,Any]\n \"\"\"\n res = self._asdict()\n res['kraus_ops'] = [[k.real.tolist(), k.imag.tolist()] for k in self.kraus_ops]\n return res"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef from_dict(d):\n kraus_ops = [KrausModel.unpack_kraus_matrix(k) for k in d['kraus_ops']]\n return KrausModel(d['gate'], d['params'], d['targets'], kraus_ops, d['fidelity'])", "response": "Recreate a KrausModel from the dictionary representation."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ncreating a JSON serializable representation of the noise model.", "response": "def to_dict(self):\n \"\"\"\n Create a JSON serializable representation of the noise model.\n\n For example::\n\n {\n \"gates\": [\n # list of embedded dictionary representations of KrausModels here [...]\n ]\n \"assignment_probs\": {\n \"0\": [[.8, .1],\n [.2, .9]],\n \"1\": [[.9, .4],\n [.1, .6]],\n }\n }\n\n :return: A dictionary representation of self.\n :rtype: Dict[str,Any]\n \"\"\"\n return {\n \"gates\": [km.to_dict() for km in self.gates],\n \"assignment_probs\": {str(qid): a.tolist() for qid, a in self.assignment_probs.items()},\n }"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef from_dict(d):\n return NoiseModel(\n gates=[KrausModel.from_dict(t) for t in d[\"gates\"]],\n assignment_probs={int(qid): np.array(a) for qid, a in d[\"assignment_probs\"].items()},\n )", "response": "Re - create the noise model from a dictionary representation."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn all defined noisy gates of a particular name.", "response": "def gates_by_name(self, name):\n \"\"\"\n Return all defined noisy gates of a particular gate name.\n\n :param str name: The gate name.\n :return: A list of noise models representing that gate.\n :rtype: Sequence[KrausModel]\n \"\"\"\n return [g for g in self.gates if g.gate == name]"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nextract the attributes from PYQUIL_PROGRAM_PROPERTIES on the Program object program into a dictionary.", "response": "def _extract_attribute_dictionary_from_program(program: Program) -> Dict[str, Any]:\n \"\"\"\n Collects the attributes from PYQUIL_PROGRAM_PROPERTIES on the Program object program\n into a dictionary.\n\n :param program: Program to collect attributes from.\n :return: Dictionary of attributes, keyed on the string attribute name.\n \"\"\"\n attrs = {}\n for prop in PYQUIL_PROGRAM_PROPERTIES:\n attrs[prop] = getattr(program, prop)\n return attrs"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nunpacking a rpcq PyQuilExecutableResponse object into a pyQuil Program object.", "response": "def _extract_program_from_pyquil_executable_response(response: PyQuilExecutableResponse) -> Program:\n \"\"\"\n Unpacks a rpcq PyQuilExecutableResponse object into a pyQuil Program object.\n\n :param response: PyQuilExecutableResponse object to be unpacked.\n :return: Resulting pyQuil Program object.\n \"\"\"\n p = Program(response.program)\n for attr, val in response.attributes.items():\n setattr(p, attr, val)\n return p"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _collect_classical_memory_write_locations(program: Program) -> List[Optional[Tuple[int, int]]]:\n ro_size = None\n for instr in program:\n if isinstance(instr, Declare) and instr.name == \"ro\":\n if ro_size is not None:\n raise ValueError(\"I found multiple places where a register named `ro` is declared! \"\n \"Please only declare one register named `ro`.\")\n ro_size = instr.memory_size\n\n measures_by_qubit: Dict[int, int] = Counter()\n ro_sources: Dict[int, Tuple[int, int]] = {}\n\n for instr in program:\n if isinstance(instr, Measurement):\n q = instr.qubit.index\n if instr.classical_reg:\n offset = instr.classical_reg.offset\n assert instr.classical_reg.name == \"ro\", instr.classical_reg.name\n if offset in ro_sources:\n _log.warning(f\"Overwriting the measured result in register \"\n f\"{instr.classical_reg} from qubit {ro_sources[offset]} \"\n f\"to qubit {q}\")\n # we track how often each qubit is measured (per shot) and into which register it is\n # measured in its n-th measurement.\n ro_sources[offset] = (q, measures_by_qubit[q])\n measures_by_qubit[q] += 1\n if ro_size:\n return [ro_sources.get(i) for i in range(ro_size)]\n elif ro_sources:\n raise ValueError(\"Found MEASURE instructions, but no 'ro' or 'ro_table' \"\n \"region was declared.\")\n else:\n return []", "response": "Collect classical memory locations that are the destination of MEASURE instructions that are written into the target register."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _collect_memory_descriptors(program: Program) -> Dict[str, ParameterSpec]:\n return {\n instr.name: ParameterSpec(type=instr.memory_type, length=instr.memory_size)\n for instr in program if isinstance(instr, Declare)\n }", "response": "Collect Declare instructions that are important for building the patch table."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef teleport(start_index, end_index, ancilla_index):\n program = make_bell_pair(end_index, ancilla_index)\n\n ro = program.declare('ro', memory_size=3)\n\n # do the teleportation\n program.inst(CNOT(start_index, ancilla_index))\n program.inst(H(start_index))\n\n # measure the results and store them in classical registers [0] and [1]\n program.measure(start_index, ro[0])\n program.measure(ancilla_index, ro[1])\n\n program.if_then(ro[1], X(2))\n program.if_then(ro[0], Z(2))\n\n program.measure(end_index, ro[2])\n\n print(program)\n return program", "response": "Teleport a qubit from start to end using an ancilla qubit\n "} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef die_program(number_of_sides):\n prog = Program()\n n_qubits = qubits_needed(number_of_sides)\n ro = prog.declare('ro', 'BIT', n_qubits)\n # Hadamard initialize.\n for q in range(n_qubits):\n prog.inst(H(q))\n # Measure everything.\n for q in range(n_qubits):\n prog.measure(q, ro[q])\n return prog", "response": "Generates a quantum program to roll a die of n faces."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef process_results(results):\n raw_results = results[0]\n processing_result = 0\n for each_qubit_measurement in raw_results:\n processing_result = 2*processing_result + each_qubit_measurement\n # Convert from 0 indexed to 1 indexed\n die_value = processing_result + 1\n return die_value", "response": "Convert n digit binary result from the QVM to a value on a die."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nroll an n - sided quantum die.", "response": "def roll_die(qvm, number_of_sides):\n \"\"\"\n Roll an n-sided quantum die.\n \"\"\"\n die_compiled = qvm.compile(die_program(number_of_sides))\n return process_results(qvm.run(die_compiled))"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef controlled(self, control_qubit):\n control_qubit = unpack_qubit(control_qubit)\n\n self.modifiers.insert(0, \"CONTROLLED\")\n self.qubits.insert(0, control_qubit)\n\n return self", "response": "Add the CONTROLLED modifier to the gate with the given control qubit."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef out(self):\n def format_matrix_element(element):\n \"\"\"\n Formats a parameterized matrix element.\n\n :param element: {int, float, complex, str} The parameterized element to format.\n \"\"\"\n if isinstance(element, integer_types) or isinstance(element, (float, complex, np.int_)):\n return format_parameter(element)\n elif isinstance(element, string_types):\n return element\n elif isinstance(element, Expression):\n return str(element)\n else:\n raise TypeError(\"Invalid matrix element: %r\" % element)\n\n if self.parameters:\n result = \"DEFGATE {}({}):\\n\".format(self.name, ', '.join(map(str, self.parameters)))\n else:\n result = \"DEFGATE {}:\\n\".format(self.name)\n\n for row in self.matrix:\n result += \" \"\n fcols = [format_matrix_element(col) for col in row]\n result += \", \".join(fcols)\n result += \"\\n\"\n return result", "response": "Prints a readable Quil string representation of this gate."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_constructor(self):\n if self.parameters:\n return lambda *params: lambda *qubits: \\\n Gate(name=self.name, params=list(params), qubits=list(map(unpack_qubit, qubits)))\n else:\n return lambda *qubits: Gate(name=self.name, params=[], qubits=list(map(unpack_qubit, qubits)))", "response": "Returns a function that creates this gate on variable qubit indices. E. g.\n mygate. get_constructor ) returns the constructor that applies the gate to qubit 1."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef num_args(self):\n rows = len(self.matrix)\n return int(np.log2(rows))", "response": "Returns the number of arguments the gate takes."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef gates_in_isa(isa):\n gates = []\n for q in isa.qubits:\n if q.dead:\n # TODO: dead qubits may in the future lead to some implicit re-indexing\n continue\n if q.type in [\"Xhalves\"]:\n gates.extend([\n Gate(\"I\", [], [unpack_qubit(q.id)]),\n Gate(\"RX\", [np.pi / 2], [unpack_qubit(q.id)]),\n Gate(\"RX\", [-np.pi / 2], [unpack_qubit(q.id)]),\n Gate(\"RX\", [np.pi], [unpack_qubit(q.id)]),\n Gate(\"RX\", [-np.pi], [unpack_qubit(q.id)]),\n Gate(\"RZ\", [THETA], [unpack_qubit(q.id)]),\n ])\n else: # pragma no coverage\n raise ValueError(\"Unknown qubit type: {}\".format(q.type))\n\n for e in isa.edges:\n if e.dead:\n continue\n targets = [unpack_qubit(t) for t in e.targets]\n if e.type in [\"CZ\", \"ISWAP\"]:\n gates.append(Gate(e.type, [], targets))\n gates.append(Gate(e.type, [], targets[::-1]))\n elif e.type in [\"CPHASE\"]:\n gates.append(Gate(e.type, [THETA], targets))\n gates.append(Gate(e.type, [THETA], targets[::-1]))\n else: # pragma no coverage\n raise ValueError(\"Unknown edge type: {}\".format(e.type))\n return gates", "response": "Generates the full gates set associated with an ISA."} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ngenerates an ISA object from a NetworkX graph.", "response": "def isa_from_graph(graph: nx.Graph, oneq_type='Xhalves', twoq_type='CZ') -> ISA:\n \"\"\"\n Generate an ISA object from a NetworkX graph.\n\n :param graph: The graph\n :param oneq_type: The type of 1-qubit gate. Currently 'Xhalves'\n :param twoq_type: The type of 2-qubit gate. One of 'CZ' or 'CPHASE'.\n \"\"\"\n all_qubits = list(range(max(graph.nodes) + 1))\n qubits = [Qubit(i, type=oneq_type, dead=i not in graph.nodes) for i in all_qubits]\n edges = [Edge(sorted((a, b)), type=twoq_type, dead=False) for a, b in graph.edges]\n return ISA(qubits, edges)"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef specs_from_graph(graph: nx.Graph):\n qspecs = [QubitSpecs(id=q, fRO=0.90, f1QRB=0.99, T1=30e-6, T2=30e-6, fActiveReset=0.99)\n for q in graph.nodes]\n especs = [EdgeSpecs(targets=(q1, q2), fBellState=0.90, fCZ=0.90, fCZ_std_err=0.05, fCPHASE=0.80)\n for q1, q2 in graph.edges]\n return Specs(qspecs, especs)", "response": "Generate a Specs object from a NetworkX graph with placeholder values for the actual specs."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef isa_to_graph(isa: ISA) -> nx.Graph:\n return nx.from_edgelist(e.targets for e in isa.edges if not e.dead)", "response": "Construct a NetworkX qubit topology from an ISA object."} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncreating a JSON - serializable representation of the ISA.", "response": "def to_dict(self):\n \"\"\"\n Create a JSON-serializable representation of the ISA.\n\n The dictionary representation is of the form::\n\n {\n \"1Q\": {\n \"0\": {\n \"type\": \"Xhalves\"\n },\n \"1\": {\n \"type\": \"Xhalves\",\n \"dead\": True\n },\n ...\n },\n \"2Q\": {\n \"1-4\": {\n \"type\": \"CZ\"\n },\n \"1-5\": {\n \"type\": \"CZ\"\n },\n ...\n },\n ...\n }\n\n :return: A dictionary representation of self.\n :rtype: Dict[str, Any]\n \"\"\"\n\n def _maybe_configure(o, t):\n # type: (Union[Qubit,Edge], str) -> dict\n \"\"\"\n Exclude default values from generated dictionary.\n\n :param Union[Qubit,Edge] o: The object to serialize\n :param str t: The default value for ``o.type``.\n :return: d\n \"\"\"\n d = {}\n if o.type != t:\n d[\"type\"] = o.type\n if o.dead:\n d[\"dead\"] = o.dead\n return d\n\n return {\n \"1Q\": {\"{}\".format(q.id): _maybe_configure(q, DEFAULT_QUBIT_TYPE) for q in self.qubits},\n \"2Q\": {\"{}-{}\".format(*edge.targets): _maybe_configure(edge, DEFAULT_EDGE_TYPE)\n for edge in self.edges}\n }"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef from_dict(d):\n return ISA(\n qubits=sorted([Qubit(id=int(qid),\n type=q.get(\"type\", DEFAULT_QUBIT_TYPE),\n dead=q.get(\"dead\", False))\n for qid, q in d[\"1Q\"].items()],\n key=lambda qubit: qubit.id),\n edges=sorted([Edge(targets=[int(q) for q in eid.split('-')],\n type=e.get(\"type\", DEFAULT_EDGE_TYPE),\n dead=e.get(\"dead\", False))\n for eid, e in d[\"2Q\"].items()],\n key=lambda edge: edge.targets),\n )", "response": "Re - create the ISA from a dictionary representation."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef fBellStates(self):\n return {tuple(es.targets): es.fBellState for es in self.edges_specs}", "response": "Get a dictionary of two - qubit Bell state fidelities keyed by targets."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nget a dictionary of CZ fidelities normalized to unity.", "response": "def fCZs(self):\n \"\"\"\n Get a dictionary of CZ fidelities (normalized to unity) from the specs,\n keyed by targets (qubit-qubit pairs).\n\n :return: A dictionary of CZ fidelities, normalized to unity.\n :rtype: Dict[tuple(int, int), float]\n \"\"\"\n return {tuple(es.targets): es.fCZ for es in self.edges_specs}"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef fCZ_std_errs(self):\n return {tuple(es.targets): es.fCZ_std_err for es in self.edges_specs}", "response": "Returns a dictionary of the standard errors of the CZ fidelities in the edges specs keyed by targets."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef fCPHASEs(self):\n return {tuple(es.targets): es.fCPHASE for es in self.edges_specs}", "response": "Get a dictionary of CPHASE fidelities keyed by targets."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncreating a JSON - serializable representation of the device Specs.", "response": "def to_dict(self):\n \"\"\"\n Create a JSON-serializable representation of the device Specs.\n\n The dictionary representation is of the form::\n\n {\n '1Q': {\n \"0\": {\n \"f1QRB\": 0.99,\n \"T1\": 20e-6,\n ...\n },\n \"1\": {\n \"f1QRB\": 0.989,\n \"T1\": 19e-6,\n ...\n },\n ...\n },\n '2Q': {\n \"1-4\": {\n \"fBellState\": 0.93,\n \"fCZ\": 0.92,\n \"fCZ_std_err\": 0.03,\n \"fCPHASE\": 0.91\n },\n \"1-5\": {\n \"fBellState\": 0.9,\n \"fCZ\": 0.89,\n \"fCZ_std_err\": 0.05,\n \"fCPHASE\": 0.88\n },\n ...\n },\n ...\n }\n\n :return: A dctionary representation of self.\n :rtype: Dict[str, Any]\n \"\"\"\n return {\n '1Q': {\n \"{}\".format(qs.id): {\n 'f1QRB': qs.f1QRB,\n 'fRO': qs.fRO,\n 'T1': qs.T1,\n 'T2': qs.T2,\n 'fActiveReset': qs.fActiveReset\n } for qs in self.qubits_specs\n },\n '2Q': {\n \"{}-{}\".format(*es.targets): {\n 'fBellState': es.fBellState,\n 'fCZ': es.fCZ,\n 'fCZ_std_err': es.fCZ_std_err,\n 'fCPHASE': es.fCPHASE\n } for es in self.edges_specs\n }\n }"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef from_dict(d):\n return Specs(\n qubits_specs=sorted([QubitSpecs(id=int(q),\n fRO=qspecs.get('fRO'),\n f1QRB=qspecs.get('f1QRB'),\n T1=qspecs.get('T1'),\n T2=qspecs.get('T2'),\n fActiveReset=qspecs.get('fActiveReset'))\n for q, qspecs in d[\"1Q\"].items()],\n key=lambda qubit_specs: qubit_specs.id),\n edges_specs=sorted([EdgeSpecs(targets=[int(q) for q in e.split('-')],\n fBellState=especs.get('fBellState'),\n fCZ=especs.get('fCZ'),\n fCZ_std_err=especs.get('fCZ_std_err'),\n fCPHASE=especs.get('fCPHASE'))\n for e, especs in d[\"2Q\"].items()],\n key=lambda edge_specs: edge_specs.targets)\n )", "response": "Re - create the Specs object from a dictionary representation."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_isa(self, oneq_type='Xhalves', twoq_type='CZ') -> ISA:\n qubits = [Qubit(id=q.id, type=oneq_type, dead=q.dead) for q in self._isa.qubits]\n edges = [Edge(targets=e.targets, type=twoq_type, dead=e.dead) for e in self._isa.edges]\n return ISA(qubits, edges)", "response": "Construct an ISA suitable for targeting by compilation."} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ninitializes a QAM into a fresh state.", "response": "def load(self, executable):\n \"\"\"\n Initialize a QAM into a fresh state.\n\n :param executable: Load a compiled executable onto the QAM.\n \"\"\"\n if self.status == 'loaded':\n warnings.warn(\"Overwriting previously loaded executable.\")\n assert self.status in ['connected', 'done', 'loaded']\n\n self._variables_shim = {}\n self._executable = executable\n self._bitstrings = None\n self.status = 'loaded'\n return self"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nwrite a value into a memory region on the QAM at a specified offset.", "response": "def write_memory(self, *, region_name: str, offset: int = 0, value=None):\n \"\"\"\n Writes a value into a memory region on the QAM at a specified offset.\n\n :param region_name: Name of the declared memory region on the QAM.\n :param offset: Integer offset into the memory region to write to.\n :param value: Value to store at the indicated location.\n \"\"\"\n assert self.status in ['loaded', 'done']\n\n aref = ParameterAref(name=region_name, index=offset)\n self._variables_shim[aref] = value\n\n return self"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef read_memory(self, *, region_name: str):\n assert self.status == 'done'\n if region_name != \"ro\":\n raise QAMError(\"Currently only allowed to read measurement data from ro.\")\n if self._bitstrings is None:\n raise QAMError(\"Bitstrings have not yet been populated. Something has gone wrong.\")\n\n return self._bitstrings", "response": "Reads from a memory region named region_name on the QAM."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef read_from_memory_region(self, *, region_name: str):\n warnings.warn(\"pyquil.api._qam.QAM.read_from_memory_region is deprecated, please use \"\n \"pyquil.api._qam.QAM.read_memory instead.\",\n DeprecationWarning)\n assert self.status == 'done'\n if region_name != \"ro\":\n raise QAMError(\"Currently only allowed to read measurement data from ro.\")\n if self._bitstrings is None:\n raise QAMError(\"Bitstrings have not yet been populated. Something has gone wrong.\")\n\n return self._bitstrings", "response": "Reads from a memory region."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef reset(self):\n self._variables_shim = {}\n self._executable = None\n self._bitstrings = None\n\n self.status = 'connected'", "response": "Reset the Quantum Abstract Machine to its initial state."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef meyer_penny_program():\n prog = pq.Program()\n ro = prog.declare('ro', memory_size=2)\n picard_register = ro[1]\n answer_register = ro[0]\n\n then_branch = pq.Program(X(0))\n else_branch = pq.Program(I(0))\n\n # Prepare Qubits in Heads state or superposition, respectively\n prog.inst(X(0), H(1))\n # Q puts the coin into a superposition\n prog.inst(H(0))\n # Picard makes a decision and acts accordingly\n prog.measure(1, picard_register)\n prog.if_then(picard_register, then_branch, else_branch)\n # Q undoes his superposition operation\n prog.inst(H(0))\n # The outcome is recorded into the answer register\n prog.measure(0, answer_register)\n\n return prog", "response": "Returns the program to simulate the Meyer - Penny Game\n "} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef term_with_coeff(term, coeff):\n if not isinstance(coeff, Number):\n raise ValueError(\"coeff must be a Number\")\n new_pauli = term.copy()\n # We cast to a complex number to ensure that internally the coefficients remain compatible.\n new_pauli.coefficient = complex(coeff)\n return new_pauli", "response": "Returns a PauliTerm that duplicates term but sets coeff\n "} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef simplify_pauli_sum(pauli_sum):\n\n # You might want to use a defaultdict(list) here, but don't because\n # we want to do our best to preserve the order of terms.\n like_terms = OrderedDict()\n for term in pauli_sum.terms:\n key = term.operations_as_set()\n if key in like_terms:\n like_terms[key].append(term)\n else:\n like_terms[key] = [term]\n\n terms = []\n for term_list in like_terms.values():\n first_term = term_list[0]\n if len(term_list) == 1 and not np.isclose(first_term.coefficient, 0.0):\n terms.append(first_term)\n else:\n coeff = sum(t.coefficient for t in term_list)\n for t in term_list:\n if list(t._ops.items()) != list(first_term._ops.items()):\n warnings.warn(\"The term {} will be combined with {}, but they have different \"\n \"orders of operations. This doesn't matter for QVM or \"\n \"wavefunction simulation but may be important when \"\n \"running on an actual device.\"\n .format(t.id(sort_ops=False), first_term.id(sort_ops=False)))\n\n if not np.isclose(coeff, 0.0):\n terms.append(term_with_coeff(term_list[0], coeff))\n return PauliSum(terms)", "response": "Simplify the sum of Pauli operators according to Pauli algebra rules."} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncheck if commuting a PauliTerm commutes with a list of other terms by natural calculation.", "response": "def check_commutation(pauli_list, pauli_two):\n \"\"\"\n Check if commuting a PauliTerm commutes with a list of other terms by natural calculation.\n Uses the result in Section 3 of arXiv:1405.5749v2, modified slightly here to check for the\n number of anti-coincidences (which must always be even for commuting PauliTerms)\n instead of the no. of coincidences, as in the paper.\n\n :param list pauli_list: A list of PauliTerm objects\n :param PauliTerm pauli_two_term: A PauliTerm object\n :returns: True if pauli_two object commutes with pauli_list, False otherwise\n :rtype: bool\n \"\"\"\n\n def coincident_parity(p1, p2):\n non_similar = 0\n p1_indices = set(p1._ops.keys())\n p2_indices = set(p2._ops.keys())\n for idx in p1_indices.intersection(p2_indices):\n if p1[idx] != p2[idx]:\n non_similar += 1\n return non_similar % 2 == 0\n\n for term in pauli_list:\n if not coincident_parity(term, pauli_two):\n return False\n return True"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ngathering the Pauli terms of pauli_terms variable into commuting sets.", "response": "def commuting_sets(pauli_terms):\n \"\"\"Gather the Pauli terms of pauli_terms variable into commuting sets\n\n Uses algorithm defined in (Raeisi, Wiebe, Sanders, arXiv:1108.4318, 2011)\n to find commuting sets. Except uses commutation check from arXiv:1405.5749v2\n\n :param PauliSum pauli_terms: A PauliSum object\n :returns: List of lists where each list contains a commuting set\n :rtype: list\n \"\"\"\n\n m_terms = len(pauli_terms.terms)\n m_s = 1\n groups = []\n groups.append([pauli_terms.terms[0]])\n for j in range(1, m_terms):\n isAssigned_bool = False\n for p in range(m_s): # check if it commutes with each group\n if isAssigned_bool is False:\n\n if check_commutation(groups[p], pauli_terms.terms[j]):\n isAssigned_bool = True\n groups[p].append(pauli_terms.terms[j])\n if isAssigned_bool is False:\n m_s += 1\n groups.append([pauli_terms.terms[j]])\n return groups"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ntest to see if a PauliTerm or PauliSum is a scalar multiple of identity.", "response": "def is_identity(term):\n \"\"\"\n Tests to see if a PauliTerm or PauliSum is a scalar multiple of identity\n\n :param term: Either a PauliTerm or PauliSum\n :returns: True if the PauliTerm or PauliSum is a scalar multiple of identity, False otherwise\n :rtype: bool\n \"\"\"\n if isinstance(term, PauliTerm):\n return (len(term) == 0) and (not np.isclose(term.coefficient, 0))\n elif isinstance(term, PauliSum):\n return (len(term.terms) == 1) and (len(term.terms[0]) == 0) and \\\n (not np.isclose(term.terms[0].coefficient, 0))\n else:\n raise TypeError(\"is_identity only checks PauliTerms and PauliSum objects!\")"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef exponential_map(term):\n if not np.isclose(np.imag(term.coefficient), 0.0):\n raise TypeError(\"PauliTerm coefficient must be real\")\n\n coeff = term.coefficient.real\n term.coefficient = term.coefficient.real\n\n def exp_wrap(param):\n prog = Program()\n if is_identity(term):\n prog.inst(X(0))\n prog.inst(PHASE(-param * coeff, 0))\n prog.inst(X(0))\n prog.inst(PHASE(-param * coeff, 0))\n elif is_zero(term):\n pass\n else:\n prog += _exponentiate_general_case(term, param)\n return prog\n\n return exp_wrap", "response": "Returns a function f that creates a Program corresponding to exp - 1j*alpha * term."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef exponentiate_commuting_pauli_sum(pauli_sum):\n if not isinstance(pauli_sum, PauliSum):\n raise TypeError(\"Argument 'pauli_sum' must be a PauliSum.\")\n\n fns = [exponential_map(term) for term in pauli_sum]\n\n def combined_exp_wrap(param):\n return Program([f(param) for f in fns])\n\n return combined_exp_wrap", "response": "Returns a function that creates a program that takes a PauliSum and returns the combined PauliSum."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _exponentiate_general_case(pauli_term, param):\n\n def reverse_hack(p):\n # A hack to produce a *temporary* program which reverses p.\n revp = Program()\n revp.inst(list(reversed(p.instructions)))\n return revp\n\n quil_prog = Program()\n change_to_z_basis = Program()\n change_to_original_basis = Program()\n cnot_seq = Program()\n prev_index = None\n highest_target_index = None\n\n for index, op in pauli_term:\n if 'X' == op:\n change_to_z_basis.inst(H(index))\n change_to_original_basis.inst(H(index))\n\n elif 'Y' == op:\n change_to_z_basis.inst(RX(np.pi / 2.0, index))\n change_to_original_basis.inst(RX(-np.pi / 2.0, index))\n\n elif 'I' == op:\n continue\n\n if prev_index is not None:\n cnot_seq.inst(CNOT(prev_index, index))\n\n prev_index = index\n highest_target_index = index\n\n # building rotation circuit\n quil_prog += change_to_z_basis\n quil_prog += cnot_seq\n quil_prog.inst(RZ(2.0 * pauli_term.coefficient * param, highest_target_index))\n quil_prog += reverse_hack(cnot_seq)\n quil_prog += change_to_original_basis\n\n return quil_prog", "response": "Returns a Quil program object corresponding to the exponential of\n the pauli_term object."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ngenerates trotterization coefficients for a given number of Trotter steps.", "response": "def suzuki_trotter(trotter_order, trotter_steps):\n \"\"\"\n Generate trotterization coefficients for a given number of Trotter steps.\n\n U = exp(A + B) is approximated as exp(w1*o1)exp(w2*o2)... This method returns\n a list [(w1, o1), (w2, o2), ... , (wm, om)] of tuples where o=0 corresponds\n to the A operator, o=1 corresponds to the B operator, and w is the\n coefficient in the exponential. For example, a second order Suzuki-Trotter\n approximation to exp(A + B) results in the following\n [(0.5/trotter_steps, 0), (1/trotteri_steps, 1),\n (0.5/trotter_steps, 0)] * trotter_steps.\n\n :param int trotter_order: order of Suzuki-Trotter approximation\n :param int trotter_steps: number of steps in the approximation\n :returns: List of tuples corresponding to the coefficient and operator\n type: o=0 is A and o=1 is B.\n :rtype: list\n \"\"\"\n p1 = p2 = p4 = p5 = 1.0 / (4 - (4 ** (1. / 3)))\n p3 = 1 - 4 * p1\n trotter_dict = {1: [(1, 0), (1, 1)],\n 2: [(0.5, 0), (1, 1), (0.5, 0)],\n 3: [(7.0 / 24, 0), (2.0 / 3.0, 1), (3.0 / 4.0, 0), (-2.0 / 3.0, 1),\n (-1.0 / 24, 0), (1.0, 1)],\n 4: [(p5 / 2, 0), (p5, 1), (p5 / 2, 0),\n (p4 / 2, 0), (p4, 1), (p4 / 2, 0),\n (p3 / 2, 0), (p3, 1), (p3 / 2, 0),\n (p2 / 2, 0), (p2, 1), (p2 / 2, 0),\n (p1 / 2, 0), (p1, 1), (p1 / 2, 0)]}\n\n order_slices = [(x0 / trotter_steps, x1) for x0, x1 in trotter_dict[trotter_order]]\n order_slices = order_slices * trotter_steps\n return order_slices"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef is_zero(pauli_object):\n if isinstance(pauli_object, PauliTerm):\n return np.isclose(pauli_object.coefficient, 0)\n elif isinstance(pauli_object, PauliSum):\n return len(pauli_object.terms) == 1 and np.isclose(pauli_object.terms[0].coefficient, 0)\n else:\n raise TypeError(\"is_zero only checks PauliTerms and PauliSum objects!\")", "response": "Tests to see if a PauliTerm or PauliSum is zero."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncreate a Quil program that approximates exp ( t ) where A and B are the same.", "response": "def trotterize(first_pauli_term, second_pauli_term, trotter_order=1,\n trotter_steps=1):\n \"\"\"\n Create a Quil program that approximates exp( (A + B)t) where A and B are\n PauliTerm operators.\n\n :param PauliTerm first_pauli_term: PauliTerm denoted `A`\n :param PauliTerm second_pauli_term: PauliTerm denoted `B`\n :param int trotter_order: Optional argument indicating the Suzuki-Trotter\n approximation order--only accepts orders 1, 2, 3, 4.\n :param int trotter_steps: Optional argument indicating the number of products\n to decompose the exponential into.\n\n :return: Quil program\n :rtype: Program\n \"\"\"\n\n if not (1 <= trotter_order < 5):\n raise ValueError(\"trotterize only accepts trotter_order in {1, 2, 3, 4}.\")\n\n commutator = (first_pauli_term * second_pauli_term) + \\\n (-1 * second_pauli_term * first_pauli_term)\n\n prog = Program()\n if is_zero(commutator):\n param_exp_prog_one = exponential_map(first_pauli_term)\n exp_prog = param_exp_prog_one(1)\n prog += exp_prog\n param_exp_prog_two = exponential_map(second_pauli_term)\n exp_prog = param_exp_prog_two(1)\n prog += exp_prog\n return prog\n\n order_slices = suzuki_trotter(trotter_order, trotter_steps)\n for coeff, operator in order_slices:\n if operator == 0:\n param_prog = exponential_map(coeff * first_pauli_term)\n exp_prog = param_prog(1)\n prog += exp_prog\n else:\n param_prog = exponential_map(coeff * second_pauli_term)\n exp_prog = param_prog(1)\n prog += exp_prog\n return prog"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn an identifier string for the PauliTerm.", "response": "def id(self, sort_ops=True):\n \"\"\"\n Returns an identifier string for the PauliTerm (ignoring the coefficient).\n\n Don't use this to compare terms. This function will not work with qubits that\n aren't sortable.\n\n :param sort_ops: Whether to sort operations by qubit. This is True by default for\n backwards compatibility but will change in a future version. Callers should never rely\n on comparing id's for testing equality. See ``operations_as_set`` instead.\n :return: A string representation of this term's operations.\n :rtype: string\n \"\"\"\n if len(self._ops) == 0 and not sort_ops:\n # This is nefariously backwards-compatibility breaking. There's potentially\n # lots of code floating around that says is_identity = term.id() == ''\n # Please use `is_identity(term)`!\n # Therefore, we only return 'I' when sort_ops is set to False, which is the newer\n # way of calling this function and implies the user knows what they're doing.\n return 'I'\n\n if sort_ops and len(self._ops) > 1:\n warnings.warn(\"`PauliTerm.id()` will not work on PauliTerms where the qubits are not \"\n \"sortable and should be avoided in favor of `operations_as_set`.\",\n FutureWarning)\n return ''.join(\"{}{}\".format(self._ops[q], q) for q in sorted(self._ops.keys()))\n else:\n return ''.join(\"{}{}\".format(p, q) for q, p in self._ops.items())"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef copy(self):\n new_term = PauliTerm(\"I\", 0, 1.0) # create new object\n # manually copy all attributes over\n for key in self.__dict__.keys():\n val = self.__dict__[key]\n if isinstance(val, (dict, list, set)): # mutable types\n new_term.__dict__[key] = copy.copy(val)\n else: # immutable types\n new_term.__dict__[key] = val\n\n return new_term", "response": "Creates a new PauliTerm with a completely new dictionary\n of operators\n "} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nallocating a Pauli Term from a list of operators and indices.", "response": "def from_list(cls, terms_list, coefficient=1.0):\n \"\"\"\n Allocates a Pauli Term from a list of operators and indices. This is more efficient than\n multiplying together individual terms.\n\n :param list terms_list: A list of tuples, e.g. [(\"X\", 0), (\"Y\", 1)]\n :return: PauliTerm\n \"\"\"\n if not all([isinstance(op, tuple) for op in terms_list]):\n raise TypeError(\"The type of terms_list should be a list of (name, index) \"\n \"tuples suitable for PauliTerm().\")\n\n pterm = PauliTerm(\"I\", 0)\n assert all([op[0] in PAULI_OPS for op in terms_list])\n\n indices = [op[1] for op in terms_list]\n assert all(_valid_qubit(index) for index in indices)\n\n # this is because from_list doesn't call simplify in order to be more efficient.\n if len(set(indices)) != len(indices):\n raise ValueError(\"Elements of PauliTerm that are allocated using from_list must \"\n \"be on disjoint qubits. Use PauliTerm multiplication to simplify \"\n \"terms instead.\")\n\n for op, index in terms_list:\n if op != \"I\":\n pterm._ops[index] = op\n if not isinstance(coefficient, Number):\n raise ValueError(\"coefficient of PauliTerm must be a Number.\")\n pterm.coefficient = complex(coefficient)\n return pterm"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nconstructing a PauliTerm from a compact string.", "response": "def from_compact_str(cls, str_pauli_term):\n \"\"\"Construct a PauliTerm from the result of str(pauli_term)\n \"\"\"\n coef_str, str_pauli_term = str_pauli_term.split('*')\n try:\n coef = int(coef_str)\n except ValueError:\n try:\n coef = float(coef_str)\n except ValueError:\n coef = complex(coef_str)\n\n op = sI() * coef\n if str_pauli_term == 'I':\n return op\n ma = re.fullmatch(r'(([XYZ])(\\d+))+', str_pauli_term)\n if ma is None:\n raise ValueError(f\"Could not parse pauli string {str_pauli_term}\")\n for ma in re.finditer(r'([XYZ])(\\d+)', str_pauli_term):\n op *= cls(ma.group(1), int(ma.group(2)))\n\n return op"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning a string representation of this PauliTerm with its coefficient and with implicit qubit indices.", "response": "def pauli_string(self, qubits=None):\n \"\"\"\n Return a string representation of this PauliTerm without its coefficient and with\n implicit qubit indices.\n\n If a list of qubits is provided, each character in the resulting string represents\n a Pauli operator on the corresponding qubit. If qubit indices are not provided as input,\n the returned string will be all non-identity operators in the order. This doesn't make\n much sense, so please provide a list of qubits. Not providing a list of qubits is\n deprecated.\n\n >>> p = PauliTerm(\"X\", 0) * PauliTerm(\"Y\", 1, 1.j)\n >>> p.pauli_string()\n \"XY\"\n >>> p.pauli_string(qubits=[0])\n \"X\"\n >>> p.pauli_string(qubits=[0, 2])\n \"XI\"\n\n :param list qubits: The list of qubits to represent, given as ints. If None, defaults to\n all qubits in this PauliTerm.\n :return: The string representation of this PauliTerm, sans coefficient\n \"\"\"\n if qubits is None:\n warnings.warn(\"Please provide a list of qubits when using PauliTerm.pauli_string\",\n DeprecationWarning)\n qubits = self.get_qubits()\n\n return ''.join(self[q] for q in qubits)"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a list of all the qubits in the sum of terms.", "response": "def get_qubits(self):\n \"\"\"\n The support of all the operators in the PauliSum object.\n\n :returns: A list of all the qubits in the sum of terms.\n :rtype: list\n \"\"\"\n return list(set().union(*[term.get_qubits() for term in self.terms]))"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ngets a list of Pyquil Program corresponding to each term in the PauliSum and a coefficient for each program in the PauliSum.", "response": "def get_programs(self):\n \"\"\"\n Get a Pyquil Program corresponding to each term in the PauliSum and a coefficient\n for each program\n\n :return: (programs, coefficients)\n \"\"\"\n programs = [term.program for term in self.terms]\n coefficients = np.array([term.coefficient for term in self.terms])\n return programs, coefficients"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ngets JSON from a Forest endpoint.", "response": "def get_json(session, url, params: dict = None):\n \"\"\"\n Get JSON from a Forest endpoint.\n \"\"\"\n res = session.get(url, params=params)\n if res.status_code >= 400:\n raise parse_error(res)\n return res.json()"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\npost JSON to the Forest endpoint.", "response": "def post_json(session, url, json):\n \"\"\"\n Post JSON to the Forest endpoint.\n \"\"\"\n res = session.post(url, json=json)\n if res.status_code >= 400:\n raise parse_error(res)\n return res"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nparsing the server error response and returns a new error object.", "response": "def parse_error(res):\n \"\"\"\n Every server error should contain a \"status\" field with a human readable explanation of\n what went wrong as well as a \"error_type\" field indicating the kind of error that can be mapped\n to a Python type.\n\n There's a fallback error UnknownError for other types of exceptions (network issues, api\n gateway problems, etc.)\n \"\"\"\n try:\n body = res.json()\n except JSONDecodeError:\n raise UnknownApiError(res.text)\n\n if 'error_type' not in body:\n raise UnknownApiError(str(body))\n\n error_type = body['error_type']\n status = body['status']\n\n if re.search(r\"[0-9]+ qubits were requested, but the QVM is limited to [0-9]+ qubits.\", status):\n return TooManyQubitsError(status)\n\n error_cls = error_mapping.get(error_type, UnknownApiError)\n return error_cls(status)"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_session():\n config = PyquilConfig()\n session = requests.Session()\n retry_adapter = HTTPAdapter(max_retries=Retry(total=3,\n method_whitelist=['POST'],\n status_forcelist=[502, 503, 504, 521, 523],\n backoff_factor=0.2,\n raise_on_status=False))\n\n session.mount(\"http://\", retry_adapter)\n session.mount(\"https://\", retry_adapter)\n\n # We need this to get binary payload for the wavefunction call.\n session.headers.update({\"Accept\": \"application/octet-stream\",\n \"X-User-Id\": config.user_id,\n \"X-Api-Key\": config.api_key})\n\n session.headers.update({\n 'Content-Type': 'application/json; charset=utf-8'\n })\n\n return session", "response": "Create a requests session to access the REST API"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef validate_noise_probabilities(noise_parameter):\n if not noise_parameter:\n return\n if not isinstance(noise_parameter, list):\n raise TypeError(\"noise_parameter must be a list\")\n if any([not isinstance(value, float) for value in noise_parameter]):\n raise TypeError(\"noise_parameter values should all be floats\")\n if len(noise_parameter) != 3:\n raise ValueError(\"noise_parameter lists must be of length 3\")\n if sum(noise_parameter) > 1 or sum(noise_parameter) < 0:\n raise ValueError(\"sum of entries in noise_parameter must be between 0 and 1 (inclusive)\")\n if any([value < 0 for value in noise_parameter]):\n raise ValueError(\"noise_parameter values should all be non-negative\")", "response": "Validate the noise parameter values for depolarizing noise."} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nchecking the validity of qubits for the payload.", "response": "def validate_qubit_list(qubit_list):\n \"\"\"\n Check the validity of qubits for the payload.\n\n :param list|range qubit_list: List of qubits to be validated.\n \"\"\"\n if not isinstance(qubit_list, (list, range)):\n raise TypeError(\"run_items must be a list\")\n if any(not isinstance(i, integer_types) or i < 0 for i in qubit_list):\n raise TypeError(\"run_items list must contain positive integer values\")\n return qubit_list"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nprepare the classical memory for serialization.", "response": "def prepare_register_list(register_dict: Dict[str, Union[bool, Sequence[int]]]):\n \"\"\"\n Canonicalize classical addresses for the payload and ready MemoryReference instances\n for serialization.\n\n This function will cast keys that are iterables of int-likes to a list of Python\n ints. This is to support specifying the register offsets as ``range()`` or numpy\n arrays. This mutates ``register_dict``.\n\n :param register_dict: The classical memory to retrieve. Specified as a dictionary:\n the keys are the names of memory regions, and the values are either (1) a list of\n integers for reading out specific entries in that memory region, or (2) True, for\n reading out the entire memory region.\n \"\"\"\n if not isinstance(register_dict, dict):\n raise TypeError(\"register_dict must be a dict but got \" + repr(register_dict))\n\n for k, v in register_dict.items():\n if isinstance(v, bool):\n assert v # If boolean v must be True\n continue\n\n indices = [int(x) for x in v] # support ranges, numpy, ...\n\n if not all(x >= 0 for x in indices):\n raise TypeError(\"Negative indices into classical arrays are not allowed.\")\n register_dict[k] = indices\n\n return register_dict"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef run_and_measure_payload(quil_program, qubits, trials, random_seed):\n if not quil_program:\n raise ValueError(\"You have attempted to run an empty program.\"\n \" Please provide gates or measure instructions to your program.\")\n\n if not isinstance(quil_program, Program):\n raise TypeError(\"quil_program must be a Quil program object\")\n qubits = validate_qubit_list(qubits)\n if not isinstance(trials, integer_types):\n raise TypeError(\"trials must be an integer\")\n\n payload = {\"type\": TYPE_MULTISHOT_MEASURE,\n \"qubits\": list(qubits),\n \"trials\": trials,\n \"compiled-quil\": quil_program.out()}\n\n if random_seed is not None:\n payload['rng-seed'] = random_seed\n\n return payload", "response": "Returns a REST payload for a single measure program."} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nrests payload for wavefunction", "response": "def wavefunction_payload(quil_program, random_seed):\n \"\"\"REST payload for :py:func:`ForestConnection._wavefunction`\"\"\"\n if not isinstance(quil_program, Program):\n raise TypeError(\"quil_program must be a Quil program object\")\n\n payload = {'type': TYPE_WAVEFUNCTION,\n 'compiled-quil': quil_program.out()}\n\n if random_seed is not None:\n payload['rng-seed'] = random_seed\n\n return payload"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nresting payload for ForestConnection. _expectation", "response": "def expectation_payload(prep_prog, operator_programs, random_seed):\n \"\"\"REST payload for :py:func:`ForestConnection._expectation`\"\"\"\n if operator_programs is None:\n operator_programs = [Program()]\n\n if not isinstance(prep_prog, Program):\n raise TypeError(\"prep_prog variable must be a Quil program object\")\n\n payload = {'type': TYPE_EXPECTATION,\n 'state-preparation': prep_prog.out(),\n 'operators': [x.out() for x in operator_programs]}\n\n if random_seed is not None:\n payload['rng-seed'] = random_seed\n\n return payload"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef qvm_run_payload(quil_program, classical_addresses, trials,\n measurement_noise, gate_noise, random_seed):\n \"\"\"REST payload for :py:func:`ForestConnection._qvm_run`\"\"\"\n if not quil_program:\n raise ValueError(\"You have attempted to run an empty program.\"\n \" Please provide gates or measure instructions to your program.\")\n if not isinstance(quil_program, Program):\n raise TypeError(\"quil_program must be a Quil program object\")\n classical_addresses = prepare_register_list(classical_addresses)\n if not isinstance(trials, integer_types):\n raise TypeError(\"trials must be an integer\")\n\n payload = {\"type\": TYPE_MULTISHOT,\n \"addresses\": classical_addresses,\n \"trials\": trials,\n \"compiled-quil\": quil_program.out()}\n\n if measurement_noise is not None:\n payload[\"measurement-noise\"] = measurement_noise\n if gate_noise is not None:\n payload[\"gate-noise\"] = gate_noise\n if random_seed is not None:\n payload['rng-seed'] = random_seed\n\n return payload", "response": "Returns the payload for a single QVM run."} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning a REST payload for a Quil program.", "response": "def quilc_compile_payload(quil_program, isa, specs):\n \"\"\"REST payload for :py:func:`ForestConnection._quilc_compile`\"\"\"\n if not quil_program:\n raise ValueError(\"You have attempted to compile an empty program.\"\n \" Please provide an actual program.\")\n if not isinstance(quil_program, Program):\n raise TypeError(\"quil_program must be a Program object.\")\n if not isinstance(isa, ISA):\n raise TypeError(\"isa must be an ISA object.\")\n if not isinstance(specs, Specs):\n raise TypeError(\"specs must be a Specs object.\")\n\n payload = {\"uncompiled-quil\": quil_program.out(),\n \"target-device\": {\n \"isa\": isa.to_dict(),\n \"specs\": specs.to_dict()}}\n\n return payload"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nrun a Forest run_and_measure job.", "response": "def _run_and_measure(self, quil_program, qubits, trials, random_seed) -> np.ndarray:\n \"\"\"\n Run a Forest ``run_and_measure`` job.\n\n Users should use :py:func:`WavefunctionSimulator.run_and_measure` instead of calling\n this directly.\n \"\"\"\n payload = run_and_measure_payload(quil_program, qubits, trials, random_seed)\n response = post_json(self.session, self.sync_endpoint + \"/qvm\", payload)\n return np.asarray(response.json())"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _wavefunction(self, quil_program, random_seed) -> Wavefunction:\n\n payload = wavefunction_payload(quil_program, random_seed)\n response = post_json(self.session, self.sync_endpoint + \"/qvm\", payload)\n return Wavefunction.from_bit_packed_string(response.content)", "response": "Run a Forest wavefunction job."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _expectation(self, prep_prog, operator_programs, random_seed) -> np.ndarray:\n if isinstance(operator_programs, Program):\n warnings.warn(\"You have provided a Program rather than a list of Programs. The results \"\n \"from expectation will be line-wise expectation values of the \"\n \"operator_programs.\", SyntaxWarning)\n\n payload = expectation_payload(prep_prog, operator_programs, random_seed)\n response = post_json(self.session, self.sync_endpoint + \"/qvm\", payload)\n return np.asarray(response.json())", "response": "Run a Forest expectation job."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nrunning a Forest run job on a QVM.", "response": "def _qvm_run(self, quil_program, classical_addresses, trials,\n measurement_noise, gate_noise, random_seed) -> np.ndarray:\n \"\"\"\n Run a Forest ``run`` job on a QVM.\n\n Users should use :py:func:`QVM.run` instead of calling this directly.\n \"\"\"\n payload = qvm_run_payload(quil_program, classical_addresses, trials,\n measurement_noise, gate_noise, random_seed)\n response = post_json(self.session, self.sync_endpoint + \"/qvm\", payload)\n\n ram = response.json()\n\n for k in ram.keys():\n ram[k] = np.array(ram[k])\n\n return ram"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _qvm_get_version_info(self) -> dict:\n response = post_json(self.session, self.sync_endpoint, {'type': 'version'})\n split_version_string = response.text.split()\n try:\n qvm_version = split_version_string[0]\n except ValueError:\n raise TypeError(f'Malformed version string returned by the QVM: {response.text}')\n return qvm_version", "response": "Get version information for the QVM."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _quilc_compile(self, quil_program, isa, specs):\n payload = quilc_compile_payload(quil_program, isa, specs)\n response = post_json(self.session, self.sync_endpoint + \"/\", payload)\n unpacked_response = response.json()\n return unpacked_response", "response": "Send a quilc job to Forest."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _get_flipped_protoquil_program(program: Program) -> Program:\n program = program.copy()\n to_measure = []\n while len(program) > 0:\n inst = program.instructions[-1]\n if isinstance(inst, Measurement):\n program.pop()\n to_measure.append((inst.qubit, inst.classical_reg))\n else:\n break\n\n program += Pragma('PRESERVE_BLOCK')\n for qu, addr in to_measure[::-1]:\n program += RX(pi, qu)\n program += Pragma('END_PRESERVE_BLOCK')\n\n for qu, addr in to_measure[::-1]:\n program += Measurement(qubit=qu, classical_reg=addr)\n\n return program", "response": "For symmetrization generate a program where X gates are added before measurement instructions."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef list_quantum_computers(connection: ForestConnection = None,\n qpus: bool = True,\n qvms: bool = True) -> List[str]:\n \"\"\"\n List the names of available quantum computers\n\n :param connection: An optional :py:class:ForestConnection` object. If not specified,\n the default values for URL endpoints will be used, and your API key\n will be read from ~/.pyquil_config. If you deign to change any\n of these parameters, pass your own :py:class:`ForestConnection` object.\n :param qpus: Whether to include QPU's in the list.\n :param qvms: Whether to include QVM's in the list.\n \"\"\"\n if connection is None:\n connection = ForestConnection()\n\n qc_names: List[str] = []\n if qpus:\n qc_names += list(list_lattices(connection=connection).keys())\n\n if qvms:\n qc_names += ['9q-square-qvm', '9q-square-noisy-qvm']\n\n return qc_names", "response": "Returns a list of available quantum computers."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _parse_name(name: str, as_qvm: bool, noisy: bool) -> Tuple[str, str, bool]:\n parts = name.split('-')\n if len(parts) >= 2 and parts[-2] == 'noisy' and parts[-1] in ['qvm', 'pyqvm']:\n if as_qvm is not None and (not as_qvm):\n raise ValueError(\"The provided qc name indicates you are getting a noisy QVM, \"\n \"but you have specified `as_qvm=False`\")\n\n if noisy is not None and (not noisy):\n raise ValueError(\"The provided qc name indicates you are getting a noisy QVM, \"\n \"but you have specified `noisy=False`\")\n\n qvm_type = parts[-1]\n noisy = True\n prefix = '-'.join(parts[:-2])\n return prefix, qvm_type, noisy\n\n if len(parts) >= 1 and parts[-1] in ['qvm', 'pyqvm']:\n if as_qvm is not None and (not as_qvm):\n raise ValueError(\"The provided qc name indicates you are getting a QVM, \"\n \"but you have specified `as_qvm=False`\")\n qvm_type = parts[-1]\n if noisy is None:\n noisy = False\n prefix = '-'.join(parts[:-1])\n return prefix, qvm_type, noisy\n\n if as_qvm is not None and as_qvm:\n qvm_type = 'qvm'\n else:\n qvm_type = None\n\n if noisy is None:\n noisy = False\n\n return name, qvm_type, noisy", "response": "Parse a name into a tuple of the prefix qvm_type and noisy."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ntake the output of _parse_name to create a canonical name.", "response": "def _canonicalize_name(prefix, qvm_type, noisy):\n \"\"\"Take the output of _parse_name to create a canonical name.\n \"\"\"\n if noisy:\n noise_suffix = '-noisy'\n else:\n noise_suffix = ''\n\n if qvm_type is None:\n qvm_suffix = ''\n elif qvm_type == 'qvm':\n qvm_suffix = '-qvm'\n elif qvm_type == 'pyqvm':\n qvm_suffix = '-pyqvm'\n else:\n raise ValueError(f\"Unknown qvm_type {qvm_type}\")\n\n name = f'{prefix}{noise_suffix}{qvm_suffix}'\n return name"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a QuantumComputer object that can be used to run a QVM.", "response": "def _get_qvm_qc(name: str, qvm_type: str, device: AbstractDevice, noise_model: NoiseModel = None,\n requires_executable: bool = False,\n connection: ForestConnection = None) -> QuantumComputer:\n \"\"\"Construct a QuantumComputer backed by a QVM.\n\n This is a minimal wrapper over the QuantumComputer, QVM, and QVMCompiler constructors.\n\n :param name: A string identifying this particular quantum computer.\n :param qvm_type: The type of QVM. Either qvm or pyqvm.\n :param device: A device following the AbstractDevice interface.\n :param noise_model: An optional noise model\n :param requires_executable: Whether this QVM will refuse to run a :py:class:`Program` and\n only accept the result of :py:func:`compiler.native_quil_to_executable`. Setting this\n to True better emulates the behavior of a QPU.\n :param connection: An optional :py:class:`ForestConnection` object. If not specified,\n the default values for URL endpoints will be used.\n :return: A QuantumComputer backed by a QVM with the above options.\n \"\"\"\n if connection is None:\n connection = ForestConnection()\n\n return QuantumComputer(name=name,\n qam=_get_qvm_or_pyqvm(\n qvm_type=qvm_type,\n connection=connection,\n noise_model=noise_model,\n device=device,\n requires_executable=requires_executable),\n device=device,\n compiler=QVMCompiler(\n device=device,\n endpoint=connection.compiler_endpoint))"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nconstruct a QVM with the provided topology.", "response": "def _get_qvm_with_topology(name: str, topology: nx.Graph,\n noisy: bool = False,\n requires_executable: bool = True,\n connection: ForestConnection = None,\n qvm_type: str = 'qvm') -> QuantumComputer:\n \"\"\"Construct a QVM with the provided topology.\n\n :param name: A name for your quantum computer. This field does not affect behavior of the\n constructed QuantumComputer.\n :param topology: A graph representing the desired qubit connectivity.\n :param noisy: Whether to include a generic noise model. If you want more control over\n the noise model, please construct your own :py:class:`NoiseModel` and use\n :py:func:`_get_qvm_qc` instead of this function.\n :param requires_executable: Whether this QVM will refuse to run a :py:class:`Program` and\n only accept the result of :py:func:`compiler.native_quil_to_executable`. Setting this\n to True better emulates the behavior of a QPU.\n :param connection: An optional :py:class:`ForestConnection` object. If not specified,\n the default values for URL endpoints will be used.\n :param qvm_type: The type of QVM. Either 'qvm' or 'pyqvm'.\n :return: A pre-configured QuantumComputer\n \"\"\"\n # Note to developers: consider making this function public and advertising it.\n device = NxDevice(topology=topology)\n if noisy:\n noise_model = decoherence_noise_with_asymmetric_ro(gates=gates_in_isa(device.get_isa()))\n else:\n noise_model = None\n return _get_qvm_qc(name=name, qvm_type=qvm_type, connection=connection, device=device,\n noise_model=noise_model, requires_executable=requires_executable)"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _get_9q_square_qvm(name: str, noisy: bool,\n connection: ForestConnection = None,\n qvm_type: str = 'qvm') -> QuantumComputer:\n \"\"\"\n A nine-qubit 3x3 square lattice.\n\n This uses a \"generic\" lattice not tied to any specific device. 9 qubits is large enough\n to do vaguely interesting algorithms and small enough to simulate quickly.\n\n :param name: The name of this QVM\n :param connection: The connection to use to talk to external services\n :param noisy: Whether to construct a noisy quantum computer\n :param qvm_type: The type of QVM. Either 'qvm' or 'pyqvm'.\n :return: A pre-configured QuantumComputer\n \"\"\"\n topology = nx.convert_node_labels_to_integers(nx.grid_2d_graph(3, 3))\n return _get_qvm_with_topology(name=name, connection=connection,\n topology=topology,\n noisy=noisy,\n requires_executable=True,\n qvm_type=qvm_type)", "response": "Returns a QuantumComputer for a 9x3 square QVM."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _get_unrestricted_qvm(name: str, noisy: bool,\n n_qubits: int = 34,\n connection: ForestConnection = None,\n qvm_type: str = 'qvm') -> QuantumComputer:\n \"\"\"\n A qvm with a fully-connected topology.\n\n This is obviously the least realistic QVM, but who am I to tell users what they want.\n\n :param name: The name of this QVM\n :param noisy: Whether to construct a noisy quantum computer\n :param n_qubits: 34 qubits ought to be enough for anybody.\n :param connection: The connection to use to talk to external services\n :param qvm_type: The type of QVM. Either 'qvm' or 'pyqvm'.\n :return: A pre-configured QuantumComputer\n \"\"\"\n topology = nx.complete_graph(n_qubits)\n return _get_qvm_with_topology(name=name, connection=connection,\n topology=topology,\n noisy=noisy,\n requires_executable=False,\n qvm_type=qvm_type)", "response": "Returns a QVM with a fully - connected topology."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _get_qvm_based_on_real_device(name: str, device: Device,\n noisy: bool, connection: ForestConnection = None,\n qvm_type: str = 'qvm'):\n \"\"\"\n A qvm with a based on a real device.\n\n This is the most realistic QVM.\n\n :param name: The full name of this QVM\n :param device: The device from :py:func:`get_lattice`.\n :param noisy: Whether to construct a noisy quantum computer by using the device's\n associated noise model.\n :param connection: An optional :py:class:`ForestConnection` object. If not specified,\n the default values for URL endpoints will be used.\n :return: A pre-configured QuantumComputer based on the named device.\n \"\"\"\n if noisy:\n noise_model = device.noise_model\n else:\n noise_model = None\n return _get_qvm_qc(name=name, connection=connection, device=device,\n noise_model=noise_model, requires_executable=True,\n qvm_type=qvm_type)", "response": "Returns a QVM with a based on a real device."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get_qc(name: str, *, as_qvm: bool = None, noisy: bool = None,\n connection: ForestConnection = None) -> QuantumComputer:\n \"\"\"\n Get a quantum computer.\n\n A quantum computer is an object of type :py:class:`QuantumComputer` and can be backed\n either by a QVM simulator (\"Quantum/Quil Virtual Machine\") or a physical Rigetti QPU (\"Quantum\n Processing Unit\") made of superconducting qubits.\n\n You can choose the quantum computer to target through a combination of its name and optional\n flags. There are multiple ways to get the same quantum computer. The following are equivalent::\n\n >>> qc = get_qc(\"Aspen-1-16Q-A-noisy-qvm\")\n >>> qc = get_qc(\"Aspen-1-16Q-A\", as_qvm=True, noisy=True)\n\n and will construct a simulator of an Aspen-1 lattice with a noise model based on device\n characteristics. We also provide a means for constructing generic quantum simulators that\n are not related to a given piece of Rigetti hardware::\n\n >>> qc = get_qc(\"9q-square-qvm\")\n >>> qc = get_qc(\"9q-square\", as_qvm=True)\n\n Finally, you can get request a QVM with \"no\" topology of a given number of qubits\n (technically, it's a fully connected graph among the given number of qubits) with::\n\n >>> qc = get_qc(\"5q-qvm\") # or \"6q-qvm\", or \"34q-qvm\", ...\n\n These less-realistic, fully-connected QVMs will also be more lenient on what types of programs\n they will ``run``. Specifically, you do not need to do any compilation. For the other, realistic\n QVMs you must use :py:func:`qc.compile` or :py:func:`qc.compiler.native_quil_to_executable`\n prior to :py:func:`qc.run`.\n\n The Rigetti QVM must be downloaded from https://www.rigetti.com/forest and run as a server\n alongside your python program. To use pyQuil's built-in QVM, replace all ``\"-qvm\"`` suffixes\n with ``\"-pyqvm\"``::\n\n >>> qc = get_qc(\"5q-pyqvm\")\n\n Redundant flags are acceptable, but conflicting flags will raise an exception::\n\n >>> qc = get_qc(\"9q-square-qvm\") # qc is fully specified by its name\n >>> qc = get_qc(\"9q-square-qvm\", as_qvm=True) # redundant, but ok\n >>> qc = get_qc(\"9q-square-qvm\", as_qvm=False) # Error!\n\n Use :py:func:`list_quantum_computers` to retrieve a list of known qc names.\n\n This method is provided as a convenience to quickly construct and use QVM's and QPU's.\n Power users may wish to have more control over the specification of a quantum computer\n (e.g. custom noise models, bespoke topologies, etc.). This is possible by constructing\n a :py:class:`QuantumComputer` object by hand. Please refer to the documentation on\n :py:class:`QuantumComputer` for more information.\n\n :param name: The name of the desired quantum computer. This should correspond to a name\n returned by :py:func:`list_quantum_computers`. Names ending in \"-qvm\" will return\n a QVM. Names ending in \"-pyqvm\" will return a :py:class:`PyQVM`. Names ending in\n \"-noisy-qvm\" will return a QVM with a noise model. Otherwise, we will return a QPU with\n the given name.\n :param as_qvm: An optional flag to force construction of a QVM (instead of a QPU). If\n specified and set to ``True``, a QVM-backed quantum computer will be returned regardless\n of the name's suffix\n :param noisy: An optional flag to force inclusion of a noise model. If\n specified and set to ``True``, a quantum computer with a noise model will be returned\n regardless of the name's suffix. The noise model for QVMs based on a real QPU\n is an empirically parameterized model based on real device noise characteristics.\n The generic QVM noise model is simple T1 and T2 noise plus readout error. See\n :py:func:`~pyquil.noise.decoherence_noise_with_asymmetric_ro`.\n :param connection: An optional :py:class:`ForestConnection` object. If not specified,\n the default values for URL endpoints will be used. If you deign to change any\n of these parameters, pass your own :py:class:`ForestConnection` object.\n :return: A pre-configured QuantumComputer\n \"\"\"\n # 1. Parse name, check for redundant options, canonicalize names.\n prefix, qvm_type, noisy = _parse_name(name, as_qvm, noisy)\n del as_qvm # do not use after _parse_name\n name = _canonicalize_name(prefix, qvm_type, noisy)\n\n # 2. Check for unrestricted {n}q-qvm\n ma = re.fullmatch(r'(\\d+)q', prefix)\n if ma is not None:\n n_qubits = int(ma.group(1))\n if qvm_type is None:\n raise ValueError(\"Please name a valid device or run as a QVM\")\n return _get_unrestricted_qvm(name=name, connection=connection,\n noisy=noisy, n_qubits=n_qubits, qvm_type=qvm_type)\n\n # 3. Check for \"9q-square\" qvm\n if prefix == '9q-generic' or prefix == '9q-square':\n if prefix == '9q-generic':\n warnings.warn(\"Please prefer '9q-square' instead of '9q-generic'\", DeprecationWarning)\n\n if qvm_type is None:\n raise ValueError(\"The device '9q-square' is only available as a QVM\")\n return _get_9q_square_qvm(name=name, connection=connection, noisy=noisy, qvm_type=qvm_type)\n\n # 4. Not a special case, query the web for information about this device.\n device = get_lattice(prefix)\n if qvm_type is not None:\n # 4.1 QVM based on a real device.\n return _get_qvm_based_on_real_device(name=name, device=device,\n noisy=noisy, connection=connection, qvm_type=qvm_type)\n else:\n # 4.2 A real device\n if noisy is not None and noisy:\n warnings.warn(\"You have specified `noisy=True`, but you're getting a QPU. This flag \"\n \"is meant for controlling noise models on QVMs.\")\n return QuantumComputer(name=name,\n qam=QPU(\n endpoint=pyquil_config.qpu_url,\n user=pyquil_config.user_id),\n device=device,\n compiler=QPUCompiler(\n quilc_endpoint=pyquil_config.quilc_url,\n qpu_compiler_endpoint=pyquil_config.qpu_compiler_url,\n device=device,\n name=prefix))", "response": "Returns a QuantumComputer object that can be used to create a new system of the given name."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a target ISA for this QuantumComputer s device.", "response": "def get_isa(self, oneq_type: str = 'Xhalves',\n twoq_type: str = 'CZ') -> ISA:\n \"\"\"\n Return a target ISA for this QuantumComputer's device.\n\n See :py:func:`AbstractDevice.get_isa` for more.\n\n :param oneq_type: The family of one-qubit gates to target\n :param twoq_type: The family of two-qubit gates to target\n \"\"\"\n return self.device.get_isa(oneq_type=oneq_type, twoq_type=twoq_type)"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef run(self, executable: Executable,\n memory_map: Dict[str, List[Union[int, float]]] = None) -> np.ndarray:\n \"\"\"\n Run a quil executable. If the executable contains declared parameters, then a memory\n map must be provided, which defines the runtime values of these parameters.\n\n :param executable: The program to run. You are responsible for compiling this first.\n :param memory_map: The mapping of declared parameters to their values. The values\n are a list of floats or integers.\n :return: A numpy array of shape (trials, len(ro-register)) that contains 0s and 1s.\n \"\"\"\n self.qam.load(executable)\n if memory_map:\n for region_name, values_list in memory_map.items():\n for offset, value in enumerate(values_list):\n # TODO gh-658: have write_memory take a list rather than value + offset\n self.qam.write_memory(region_name=region_name, offset=offset, value=value)\n return self.qam.run() \\\n .wait() \\\n .read_memory(region_name='ro')", "response": "Runs a quil executable and returns the result."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nrunning a program in such a way that the readout error is made collectively symmetric.", "response": "def run_symmetrized_readout(self, program: Program, trials: int) -> np.ndarray:\n \"\"\"\n Run a quil program in such a way that the readout error is made collectively symmetric\n\n This means the probability of a bitstring ``b`` being mistaken for a bitstring ``c`` is\n the same as the probability of ``not(b)`` being mistaken for ``not(c)``\n\n A more general symmetrization would guarantee that the probability of ``b`` being\n mistaken for ``c`` depends only on which bit of ``c`` are different from ``b``. This\n would require choosing random subsets of bits to flip.\n\n In a noisy device, the probability of accurately reading the 0 state might be higher\n than that of the 1 state. This makes correcting for readout more difficult. This\n function runs the program normally ``(trials//2)`` times. The other half of the time,\n it will insert an ``X`` gate prior to any ``MEASURE`` instruction and then flip the\n measured classical bit back.\n\n See :py:func:`run` for this function's parameter descriptions.\n \"\"\"\n flipped_program = _get_flipped_protoquil_program(program)\n if trials % 2 != 0:\n raise ValueError(\"Using symmetrized measurement functionality requires that you \"\n \"take an even number of trials.\")\n half_trials = trials // 2\n flipped_program = flipped_program.wrap_in_numshots_loop(shots=half_trials)\n flipped_executable = self.compile(flipped_program)\n\n executable = self.compile(program.wrap_in_numshots_loop(half_trials))\n samples = self.run(executable)\n flipped_samples = self.run(flipped_executable)\n double_flipped_samples = np.logical_not(flipped_samples).astype(int)\n results = np.concatenate((samples, double_flipped_samples), axis=0)\n np.random.shuffle(results)\n return results"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nrunning the provided state preparation program and measure all the qubits on this QuantumComputer.", "response": "def run_and_measure(self, program: Program, trials: int) -> Dict[int, np.ndarray]:\n \"\"\"\n Run the provided state preparation program and measure all qubits.\n\n This will measure all the qubits on this QuantumComputer, not just qubits\n that are used in the program.\n\n The returned data is a dictionary keyed by qubit index because qubits for a given\n QuantumComputer may be non-contiguous and non-zero-indexed. To turn this dictionary\n into a 2d numpy array of bitstrings, consider::\n\n bitstrings = qc.run_and_measure(...)\n bitstring_array = np.vstack(bitstrings[q] for q in qc.qubits()).T\n bitstring_array.shape # (trials, len(qc.qubits()))\n\n .. note::\n\n In contrast to :py:class:`QVMConnection.run_and_measure`, this method simulates\n noise correctly for noisy QVMs. However, this method is slower for ``trials > 1``.\n For faster noise-free simulation, consider\n :py:class:`WavefunctionSimulator.run_and_measure`.\n\n :param program: The state preparation program to run and then measure.\n :param trials: The number of times to run the program.\n :return: A dictionary keyed by qubit index where the corresponding value is a 1D array of\n measured bits.\n \"\"\"\n program = program.copy()\n validate_supported_quil(program)\n ro = program.declare('ro', 'BIT', len(self.qubits()))\n for i, q in enumerate(self.qubits()):\n program.inst(MEASURE(q, ro[i]))\n program.wrap_in_numshots_loop(trials)\n executable = self.compile(program)\n bitstring_array = self.run(executable=executable)\n bitstring_dict = {}\n for i, q in enumerate(self.qubits()):\n bitstring_dict[q] = bitstring_array[:, i]\n return bitstring_dict"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef compile(self, program: Program,\n to_native_gates: bool = True,\n optimize: bool = True) -> Union[BinaryExecutableResponse, PyQuilExecutableResponse]:\n \"\"\"\n A high-level interface to program compilation.\n\n Compilation currently consists of two stages. Please see the :py:class:`AbstractCompiler`\n docs for more information. This function does all stages of compilation.\n\n Right now both ``to_native_gates`` and ``optimize`` must be either both set or both\n unset. More modular compilation passes may be available in the future.\n\n :param program: A Program\n :param to_native_gates: Whether to compile non-native gates to native gates.\n :param optimize: Whether to optimize programs to reduce the number of operations.\n :return: An executable binary suitable for passing to :py:func:`QuantumComputer.run`.\n \"\"\"\n flags = [to_native_gates, optimize]\n assert all(flags) or all(not f for f in flags), \"Must turn quilc all on or all off\"\n quilc = all(flags)\n\n if quilc:\n nq_program = self.compiler.quil_to_native_quil(program)\n else:\n nq_program = program\n binary = self.compiler.native_quil_to_executable(nq_program)\n return binary", "response": "Compile a program into a binary suitable for passing to QuumComputer. run."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ntranslate a DataBuffer into a NumPy array.", "response": "def decode_buffer(buffer: dict) -> np.ndarray:\n \"\"\"\n Translate a DataBuffer into a numpy array.\n\n :param buffer: Dictionary with 'data' byte array, 'dtype', and 'shape' fields\n :return: NumPy array of decoded data\n \"\"\"\n buf = np.frombuffer(buffer['data'], dtype=buffer['dtype'])\n return buf.reshape(buffer['shape'])"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _extract_bitstrings(ro_sources: List[Optional[Tuple[int, int]]],\n buffers: Dict[str, np.ndarray]\n ) -> np.ndarray:\n \"\"\"\n De-mux qubit readout results and assemble them into the ro-bitstrings in the correct order.\n\n :param ro_sources: Specification of the ro_sources, cf\n :py:func:`pyquil.api._compiler._collect_classical_memory_write_locations`.\n It is a list whose value ``(q, m)`` at index ``addr`` records that the ``m``-th measurement\n of qubit ``q`` was measured into ``ro`` address ``addr``. A value of `None` means nothing\n was measured into ``ro`` address ``addr``.\n :param buffers: A dictionary of readout results returned from the qpu.\n :return: A numpy array of shape ``(num_shots, len(ro_sources))`` with the readout bits.\n \"\"\"\n # hack to extract num_shots indirectly from the shape of the returned data\n first, *rest = buffers.values()\n num_shots = first.shape[0]\n bitstrings = np.zeros((num_shots, len(ro_sources)), dtype=np.int64)\n for col_idx, src in enumerate(ro_sources):\n if src:\n qubit, meas_idx = src\n buf = buffers[f\"q{qubit}\"]\n if buf.ndim == 1:\n buf = buf.reshape((num_shots, 1))\n bitstrings[:, col_idx] = buf[:, meas_idx]\n return bitstrings", "response": "Extract the bitstrings from the readout results of a classical memory readout."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef load(self, executable):\n super().load(executable)\n if hasattr(self._executable, \"recalculation_table\"):\n recalculation_table = self._executable.recalculation_table\n for memory_reference, recalc_rule in recalculation_table.items():\n # We can only parse complete lines of Quil, so we wrap the arithmetic expression\n # in a valid Quil instruction to parse it.\n # TODO: This hack should be replaced after #687\n expression = parse(f\"RZ({recalc_rule}) 0\")[0].params[0]\n recalculation_table[memory_reference] = expression\n return self", "response": "Load the executable and parse the expressions\n into a pyQuil Expression object."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef run(self, run_priority: Optional[int] = None):\n # This prevents a common error where users expect QVM.run()\n # and QPU.run() to be interchangeable. QPU.run() needs the\n # supplied executable to have been compiled, QVM.run() does not.\n if isinstance(self._executable, Program):\n raise TypeError(\"It looks like you have provided a Program where an Executable\"\n \" is expected. Please use QuantumComputer.compile() to compile\"\n \" your program.\")\n super().run()\n\n request = QPURequest(program=self._executable.program,\n patch_values=self._build_patch_values(),\n id=str(uuid.uuid4()))\n job_priority = run_priority if run_priority is not None else self.priority\n job_id = self.client.call('execute_qpu_request', request=request, user=self.user,\n priority=job_priority)\n results = self._get_buffers(job_id)\n ro_sources = self._executable.ro_sources\n\n if results:\n bitstrings = _extract_bitstrings(ro_sources, results)\n elif not ro_sources:\n warnings.warn(\"You are running a QPU program with no MEASURE instructions. \"\n \"The result of this program will always be an empty array. Are \"\n \"you sure you didn't mean to measure some of your qubits?\")\n bitstrings = np.zeros((0, 0), dtype=np.int64)\n else:\n bitstrings = None\n\n self._bitstrings = bitstrings\n self._last_results = results\n return self", "response": "Runs a pyquil program on the QPU server and returns the result as a numpy array."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nget the decoded result buffers for particular job_id.", "response": "def _get_buffers(self, job_id: str) -> Dict[str, np.ndarray]:\n \"\"\"\n Return the decoded result buffers for particular job_id.\n\n :param job_id: Unique identifier for the job in question\n :return: Decoded buffers or throw an error\n \"\"\"\n buffers = self.client.call('get_buffers', job_id, wait=True)\n return {k: decode_buffer(v) for k, v in buffers.items()}"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _update_variables_shim_with_recalculation_table(self):\n if not hasattr(self._executable, \"recalculation_table\"):\n # No recalculation table, no work to be done here.\n return\n for memory_reference, expression in self._executable.recalculation_table.items():\n # Replace the user-declared memory references with any values the user has written,\n # coerced to a float because that is how we declared it.\n self._variables_shim[memory_reference] = float(self._resolve_memory_references(expression))", "response": "Update self. _variables_shim with the final values that are patched into the gate parameters and the recalculation table."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ntraverses the given Expression and replace any Memory References with whatever values have been provided by the user.", "response": "def _resolve_memory_references(self, expression: Expression) -> Union[float, int]:\n \"\"\"\n Traverse the given Expression, and replace any Memory References with whatever values\n have been so far provided by the user for those memory spaces. Declared memory defaults\n to zero.\n\n :param expression: an Expression\n \"\"\"\n if isinstance(expression, BinaryExp):\n left = self._resolve_memory_references(expression.op1)\n right = self._resolve_memory_references(expression.op2)\n return expression.fn(left, right)\n elif isinstance(expression, Function):\n return expression.fn(self._resolve_memory_references(expression.expression))\n elif isinstance(expression, Parameter):\n raise ValueError(f\"Unexpected Parameter in gate expression: {expression}\")\n elif isinstance(expression, float) or isinstance(expression, int):\n return expression\n elif isinstance(expression, MemoryReference):\n return self._variables_shim.get(ParameterAref(name=expression.name, index=expression.offset), 0)\n else:\n raise ValueError(f\"Unexpected expression in gate parameter: {expression}\")"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn the index of the first bit that changed between a and b.", "response": "def changed_bit_pos(a, b):\n \"\"\"\n Return the index of the first bit that changed between `a` an `b`.\n Return None if there are no changed bits.\n \"\"\"\n c = a ^ b\n n = 0\n while c > 0:\n if c & 1 == 1:\n return n\n c >>= 1\n n += 1\n return None"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ngenerates the Gray code for num_bits bits.", "response": "def gray(num_bits):\n \"\"\"\n Generate the Gray code for `num_bits` bits.\n \"\"\"\n last = 0\n x = 0\n n = 1 << num_bits\n while n > x:\n bit_string = bin(n + x ^ x // 2)[3:]\n value = int(bit_string, 2)\n yield bit_string, value, changed_bit_pos(last, value)\n last = value\n x += 1"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ngives a one - qubit gate matrix U construct a controlled - U on all pointer qubits.", "response": "def controlled(num_ptr_bits, U):\n \"\"\"\n Given a one-qubit gate matrix U, construct a controlled-U on all pointer\n qubits.\n \"\"\"\n d = 2 ** (1 + num_ptr_bits)\n m = np.eye(d)\n m[d - 2:, d - 2:] = U\n return m"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nflip back the pointer qubits that were previously flipped indicated by the flags bits_set.", "response": "def fixup(p, data_bits, ptr_bits, bits_set):\n \"\"\"\n Flip back the pointer qubits that were previously flipped indicated by\n the flags `bits_set`.\n \"\"\"\n for i in range(ptr_bits):\n if 0 != bits_set & (1 << i):\n p.inst(X(data_bits + i))"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nmake a pointer gate on num_qubits.", "response": "def pointer_gate(num_qubits, U):\n \"\"\"\n Make a pointer gate on `num_qubits`. The one-qubit gate U will act on the\n qubit addressed by the pointer qubits interpreted as an unsigned binary\n integer.\n\n There are P = floor(lg(num_qubits)) pointer qubits, and qubits numbered\n\n N - 1\n N - 2\n ...\n N - P\n\n are those reserved to represent the pointer. The first N - P qubits\n are the qubits which the one-qubit gate U can act on.\n \"\"\"\n ptr_bits = int(floor(np.log2(num_qubits)))\n data_bits = num_qubits - ptr_bits\n ptr_state = 0\n assert ptr_bits > 0\n\n program = pq.Program()\n\n program.defgate(\"CU\", controlled(ptr_bits, U))\n\n for _, target_qubit, changed in gray(ptr_bits):\n if changed is None:\n for ptr_qubit in range(num_qubits - ptr_bits, num_qubits):\n program.inst(X(ptr_qubit))\n ptr_state ^= 1 << (ptr_qubit - data_bits)\n else:\n program.inst(X(data_bits + changed))\n ptr_state ^= 1 << changed\n\n if target_qubit < data_bits:\n control_qubits = tuple(data_bits + i for i in range(ptr_bits))\n program.inst((\"CU\",) + control_qubits + (target_qubit,))\n\n fixup(program, data_bits, ptr_bits, ptr_state)\n return program"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn the amplitude damping Kraus operators", "response": "def relaxation_operators(p):\n \"\"\"\n Return the amplitude damping Kraus operators\n \"\"\"\n k0 = np.array([[1.0, 0.0], [0.0, np.sqrt(1 - p)]])\n k1 = np.array([[0.0, np.sqrt(p)], [0.0, 0.0]])\n return k0, k1"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef dephasing_operators(p):\n k0 = np.eye(2) * np.sqrt(1 - p / 2)\n k1 = np.sqrt(p / 2) * Z\n return k0, k1", "response": "Return the phase damping Kraus operators"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef depolarizing_operators(p):\n k0 = np.sqrt(1.0 - p) * I\n k1 = np.sqrt(p / 3.0) * X\n k2 = np.sqrt(p / 3.0) * Y\n k3 = np.sqrt(p / 3.0) * Z\n return k0, k1, k2, k3", "response": "Return the phase damping Kraus operators"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef phase_flip_operators(p):\n k0 = np.sqrt(1 - p) * I\n k1 = np.sqrt(p) * Z\n return k0, k1", "response": "Return the phase flip kraus operators"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef bit_flip_operators(p):\n k0 = np.sqrt(1 - p) * I\n k1 = np.sqrt(p) * X\n return k0, k1", "response": "Return the phase flip kraus operators"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning the bitphase flip kraus operators", "response": "def bitphase_flip_operators(p):\n \"\"\"\n Return the bitphase flip kraus operators\n \"\"\"\n k0 = np.sqrt(1 - p) * I\n k1 = np.sqrt(p) * Y\n return k0, k1"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef run(self, quil_program, classical_addresses: List[int] = None,\n trials=1):\n \"\"\"\n Run a Quil program multiple times, accumulating the values deposited in\n a list of classical addresses.\n\n :param Program quil_program: A Quil program.\n :param classical_addresses: The classical memory to retrieve. Specified as a list of\n integers that index into a readout register named ``ro``. This function--and\n particularly this argument--are included for backwards compatibility and will\n be removed in the future.\n :param int trials: Number of shots to collect.\n :return: A list of dictionaries of bits. Each dictionary corresponds to the values in\n `classical_addresses`.\n :rtype: list\n \"\"\"\n if classical_addresses is None:\n caddresses = get_classical_addresses_from_program(quil_program)\n\n else:\n caddresses = {'ro': classical_addresses}\n\n buffers = self._connection._qvm_run(quil_program, caddresses, trials,\n self.measurement_noise, self.gate_noise,\n self.random_seed)\n\n if len(buffers) == 0:\n return []\n if 'ro' in buffers:\n return buffers['ro'].tolist()\n\n raise ValueError(\"You are using QVMConnection.run with multiple readout registers not \"\n \"named `ro`. Please use the new `QuantumComputer` abstraction.\")", "response": "Runs a Quil program multiple times accumulating the values deposited in\n a list of classical addresses."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nrun a Quil program once to determine the final wavefunction and measure multiple times.", "response": "def run_and_measure(self, quil_program, qubits, trials=1):\n \"\"\"\n Run a Quil program once to determine the final wavefunction, and measure multiple times.\n\n :note: If the execution of ``quil_program`` is **non-deterministic**, i.e., if it includes\n measurements and/or noisy quantum gates, then the final wavefunction from which the\n returned bitstrings are sampled itself only represents a stochastically generated sample\n and the outcomes sampled from *different* ``run_and_measure`` calls *generally sample\n different bitstring distributions*.\n\n :param Program quil_program: A Quil program.\n :param list|range qubits: A list of qubits.\n :param int trials: Number of shots to collect.\n :return: A list of a list of bits.\n :rtype: list\n \"\"\"\n # Developer note: This code is for backwards compatibility. It can't be replaced with\n # ForestConnection._run_and_measure because we've turned off the ability to set\n # `needs_compilation` (that usually indicates the user is doing something iffy like\n # using a noise model with this function)\n\n payload = self._run_and_measure_payload(quil_program, qubits, trials)\n response = post_json(self.session, self.sync_endpoint + \"/qvm\", payload)\n return response.json()"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef wavefunction(self, quil_program):\n # Developer note: This code is for backwards compatibility. It can't be replaced with\n # ForestConnection._wavefunction because we've turned off the ability to set\n # `needs_compilation` (that usually indicates the user is doing something iffy like\n # using a noise model with this function)\n\n payload = self._wavefunction_payload(quil_program)\n response = post_json(self.session, self.sync_endpoint + \"/qvm\", payload)\n return Wavefunction.from_bit_packed_string(response.content)", "response": "Simulate a Quil program and get the wavefunction back."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncalculating the expectation value of operators given a state prepared by prep_prog and a list of operators that are provided by operator_programs.", "response": "def expectation(self, prep_prog, operator_programs=None):\n \"\"\"\n Calculate the expectation value of operators given a state prepared by\n prep_program.\n\n :note: If the execution of ``quil_program`` is **non-deterministic**, i.e., if it includes\n measurements and/or noisy quantum gates, then the final wavefunction from which the\n expectation values are computed itself only represents a stochastically generated\n sample. The expectations returned from *different* ``expectation`` calls *will then\n generally be different*.\n\n To measure the expectation of a PauliSum, you probably want to\n do something like this::\n\n progs, coefs = hamiltonian.get_programs()\n expect_coeffs = np.array(cxn.expectation(prep_program, operator_programs=progs))\n return np.real_if_close(np.dot(coefs, expect_coeffs))\n\n :param Program prep_prog: Quil program for state preparation.\n :param list operator_programs: A list of Programs, each specifying an operator whose expectation to compute.\n Default is a list containing only the empty Program.\n :return: Expectation values of the operators.\n :rtype: List[float]\n \"\"\"\n # Developer note: This code is for backwards compatibility. It can't be replaced with\n # ForestConnection._expectation because we've turned off the ability to set\n # `needs_compilation` (that usually indicates the user is doing something iffy like\n # using a noise model with this function)\n\n if isinstance(operator_programs, Program):\n warnings.warn(\n \"You have provided a Program rather than a list of Programs. The results from expectation \"\n \"will be line-wise expectation values of the operator_programs.\", SyntaxWarning)\n\n payload = self._expectation_payload(prep_prog, operator_programs)\n response = post_json(self.session, self.sync_endpoint + \"/qvm\", payload)\n return response.json()"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef pauli_expectation(self, prep_prog, pauli_terms):\n\n is_pauli_sum = False\n if isinstance(pauli_terms, PauliSum):\n progs, coeffs = pauli_terms.get_programs()\n is_pauli_sum = True\n else:\n coeffs = [pt.coefficient for pt in pauli_terms]\n progs = [pt.program for pt in pauli_terms]\n\n bare_results = self.expectation(prep_prog, progs)\n results = [c * r for c, r in zip(coeffs, bare_results)]\n if is_pauli_sum:\n return sum(results)\n return results", "response": "Calculates the expectation value of Pauli operators given a program and a list of PauliTerms."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _maybe_add_noise_to_payload(self, payload):\n if self.measurement_noise is not None:\n payload[\"measurement-noise\"] = self.measurement_noise\n if self.gate_noise is not None:\n payload[\"gate-noise\"] = self.gate_noise", "response": "Add the noise and measurement noise to the payload."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef load(self, executable):\n if self.requires_executable:\n if isinstance(executable, PyQuilExecutableResponse):\n executable = _extract_program_from_pyquil_executable_response(executable)\n else:\n raise TypeError(\"`executable` argument must be a `PyQuilExecutableResponse`. Make \"\n \"sure you have explicitly compiled your program via `qc.compile` \"\n \"or `qc.compiler.native_quil_to_executable(...)` for more \"\n \"fine-grained control. This explicit step is required for running \"\n \"on a QPU.\")\n else:\n if isinstance(executable, PyQuilExecutableResponse):\n executable = _extract_program_from_pyquil_executable_response(executable)\n elif isinstance(executable, Program):\n pass\n else:\n raise TypeError(\"`executable` argument must be a `PyQuilExecutableResponse` or a \"\n \"`Program`. You provided {}\".format(type(executable)))\n\n return super().load(executable)", "response": "Initialize a QAM and load a program."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nruns a Quil program on the QVM multiple times and return the values stored in the classical registers designated by the classical_addresses parameter.", "response": "def run(self):\n \"\"\"\n Run a Quil program on the QVM multiple times and return the values stored in the\n classical registers designated by the classical_addresses parameter.\n\n :return: An array of bitstrings of shape ``(trials, len(classical_addresses))``\n \"\"\"\n\n super().run()\n\n if not isinstance(self._executable, Program):\n # This should really never happen\n # unless a user monkeys with `self.status` and `self._executable`.\n raise ValueError(\"Please `load` an appropriate executable.\")\n\n quil_program = self._executable\n trials = quil_program.num_shots\n classical_addresses = get_classical_addresses_from_program(quil_program)\n\n if self.noise_model is not None:\n quil_program = apply_noise_model(quil_program, self.noise_model)\n\n quil_program = self.augment_program_with_memory_values(quil_program)\n try:\n self._bitstrings = self.connection._qvm_run(quil_program=quil_program,\n classical_addresses=classical_addresses,\n trials=trials,\n measurement_noise=self.measurement_noise,\n gate_noise=self.gate_noise,\n random_seed=self.random_seed)['ro']\n except KeyError:\n warnings.warn(\"You are running a QVM program with no MEASURE instructions. \"\n \"The result of this program will always be an empty array. Are \"\n \"you sure you didn't mean to measure some of your qubits?\")\n self._bitstrings = np.zeros((trials, 0), dtype=np.int64)\n\n return self"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef result(self):\n if not self.is_done():\n raise ValueError(\"Cannot get a result for a program that isn't completed.\")\n\n if self._raw['status'] == 'CANCELLED':\n raise CancellationError(self._raw['result'])\n elif self._raw['status'] == 'ERROR':\n if self._machine == 'QVM':\n raise QVMError(self._raw['result'])\n elif self._machine == 'QPU':\n raise QPUError(self._raw['result'])\n elif self._machine == 'QUILC':\n raise QUILCError(self._raw['result'])\n else:\n raise UnknownApiError(self._raw['result'])\n\n if self._raw['program']['type'] == 'wavefunction':\n return Wavefunction.from_bit_packed_string(\n base64.b64decode(self._raw['result']), self._raw['program']['addresses'])\n elif self._raw['program']['type'] in ['multishot', 'multishot-measure', 'expectation']:\n return np.asarray(self._raw['result'])\n else:\n return self._raw['result']", "response": "Returns the result of the job."} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn the running time for this job.", "response": "def running_time(self):\n \"\"\"\n For how long was the job running?\n :return: Running time, seconds\n :rtype: Optional[float]\n \"\"\"\n if not self.is_done():\n raise ValueError(\"Cannot get running time for a program that isn't completed.\")\n try:\n running_time = float(self._raw['running_time'].split()[0])\n except (ValueError, KeyError, IndexError):\n raise UnknownApiError(str(self._raw))\n return running_time"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _get_metadata(self, key):\n if not self.is_done():\n raise ValueError(\"Cannot get metadata for a program that isn't completed.\")\n\n return self._raw.get(\"metadata\", {}).get(key, None)", "response": "Retrieves a particular key from the metadata dictionary."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning the compiled Quil program or None if the Job has no compiled Quil program.", "response": "def compiled_quil(self):\n \"\"\"\n If the Quil program associated with the Job was compiled (e.g., to translate it to the\n QPU's natural gateset) return this compiled program.\n\n :rtype: Optional[Program]\n \"\"\"\n prog = self._raw.get(\"program\", {}).get(\"compiled-quil\", None)\n if prog is not None:\n return parse_program(prog)\n else:\n # if we failed too early to even get a \"compiled-quil\" field,\n # then alert the user to that problem instead\n if self._raw['status'] == 'ERROR':\n return self.result()"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nleaves - multiplies the given axes of the wf tensor by the given gate matrix.", "response": "def targeted_einsum(gate: np.ndarray,\n wf: np.ndarray,\n wf_target_inds: List[int]\n ) -> np.ndarray:\n \"\"\"Left-multiplies the given axes of the wf tensor by the given gate matrix.\n\n Note that the matrix must have a compatible tensor structure.\n For example, if you have an 6-qubit state vector ``wf`` with shape\n (2, 2, 2, 2, 2, 2), and a 2-qubit unitary operation ``op`` with shape\n (2, 2, 2, 2), and you want to apply ``op`` to the 5th and 3rd qubits\n within ``input_state``, then the output state vector is computed as follows::\n\n output_state = targeted_einsum(op, input_state, [5, 3])\n\n This method also works when the right hand side is a matrix instead of a\n vector. If a unitary circuit's matrix is ``old_effect``, and you append\n a CNOT(q1, q4) operation onto the circuit, where the control q1 is the qubit\n at offset 1 and the target q4 is the qubit at offset 4, then the appended\n circuit's unitary matrix is computed as follows::\n\n new_effect = targeted_left_multiply(CNOT.reshape((2, 2, 2, 2)), old_effect, [1, 4])\n\n :param gate: What to left-multiply the target tensor by.\n :param wf: A tensor to carefully broadcast a left-multiply over.\n :param wf_target_inds: Which axes of the target are being operated on.\n :returns: The output tensor.\n \"\"\"\n k = len(wf_target_inds)\n d = len(wf.shape)\n work_indices = tuple(range(k))\n data_indices = tuple(range(k, k + d))\n used_data_indices = tuple(data_indices[q] for q in wf_target_inds)\n input_indices = work_indices + used_data_indices\n output_indices = list(data_indices)\n for w, t in zip(work_indices, wf_target_inds):\n output_indices[t] = w\n\n # TODO: `out` does not work if input matrices share memory with outputs, as is usually\n # TODO: the case when propogating a wavefunction. This might be fixed in numpy 1.15\n # https://github.com/numpy/numpy/pull/11286\n # It might be worth re-investigating memory savings with `out` when numpy 1.15 becomes\n # commonplace.\n\n return np.einsum(gate, input_indices,\n wf, data_indices,\n output_indices)"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nleaves - multiplies the given axes of the wf tensor by the given gate matrix.", "response": "def targeted_tensordot(gate: np.ndarray,\n wf: np.ndarray,\n wf_target_inds: Sequence[int]\n ) -> np.ndarray:\n \"\"\"Left-multiplies the given axes of the wf tensor by the given gate matrix.\n\n Compare with :py:func:`targeted_einsum`. The semantics of these two functions should be\n identical, except this uses ``np.tensordot`` instead of ``np.einsum``.\n\n :param gate: What to left-multiply the target tensor by.\n :param wf: A tensor to carefully broadcast a left-multiply over.\n :param wf_target_inds: Which axes of the target are being operated on.\n :returns: The output tensor.\n \"\"\"\n gate_n_qubits = gate.ndim // 2\n n_qubits = wf.ndim\n\n # the indices we want to sum over are the final half\n gate_inds = np.arange(gate_n_qubits, 2 * gate_n_qubits)\n assert len(wf_target_inds) == len(gate_inds), (wf_target_inds, gate_inds)\n wf = np.tensordot(gate, wf, (gate_inds, wf_target_inds))\n\n # tensordot dumps \"output\" indices into 0, 1, .. gate_n_qubits\n # we need to move them to the right place.\n\n # First create a list of all the \"unaffected\" indices which is everything but the\n # first `gate_n_qubits`\n axes_ordering = list(range(gate_n_qubits, n_qubits))\n\n # We want to \"insert\" the affected indices into the right place. This means\n # we have to be extra careful about calling list.insert in the correct order.\n # Namely, we have to insert low target indices first.\n where_td_put_them = np.arange(gate_n_qubits)\n sorty = np.argsort(wf_target_inds)\n where_td_put_them = where_td_put_them[sorty]\n sorted_targets = np.asarray(wf_target_inds)[sorty]\n # now that everything is sorted, we can do the insertion.\n for target_ind, from_ind in zip(sorted_targets, where_td_put_them):\n axes_ordering.insert(target_ind, from_ind)\n\n # A quick call to transpose gives us the right thing.\n return wf.transpose(axes_ordering)"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nget the probabilities of measuring a qubit.", "response": "def get_measure_probabilities(wf, qubit):\n \"\"\"\n Get the probabilities of measuring a qubit.\n\n :param wf: The statevector with a dimension for each qubit\n :param qubit: The qubit to measure. We will sum over every axis except this one.\n :return: A vector of classical probabilities.\n \"\"\"\n n_qubits = len(wf.shape)\n all_inds = list(range(n_qubits))\n\n return np.einsum(np.conj(wf), all_inds,\n wf, all_inds,\n [int(qubit)])"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _get_gate_tensor_and_qubits(gate: Gate):\n if len(gate.params) > 0:\n matrix = QUANTUM_GATES[gate.name](*gate.params)\n else:\n matrix = QUANTUM_GATES[gate.name]\n\n qubit_inds = [q.index for q in gate.qubits]\n\n # e.g. 2-qubit matrix is 4x4; turns into (2,2,2,2) tensor.\n tensor = np.reshape(matrix, (2,) * len(qubit_inds) * 2)\n\n return tensor, qubit_inds", "response": "Given a gate instruction turn it into a matrix and extract qubit indices."} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nsamples bitstrings from the wavefunction.", "response": "def sample_bitstrings(self, n_samples):\n \"\"\"\n Sample bitstrings from the distribution defined by the wavefunction.\n\n Qubit 0 is at ``out[:, 0]``.\n\n :param n_samples: The number of bitstrings to sample\n :return: An array of shape (n_samples, n_qubits)\n \"\"\"\n if self.rs is None:\n raise ValueError(\"You have tried to perform a stochastic operation without setting the \"\n \"random state of the simulator. Might I suggest using a PyQVM object?\")\n\n # note on reshape: it puts bitstrings in lexicographical order.\n # would you look at that .. _all_bitstrings returns things in lexicographical order!\n # reminder: qubit 0 is on the left in einsum simulator.\n probabilities = np.abs(self.wf.reshape(-1)) ** 2\n possible_bitstrings = all_bitstrings(self.n_qubits)\n inds = self.rs.choice(2 ** self.n_qubits, n_samples, p=probabilities)\n return possible_bitstrings[inds, :]"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nmeasuring a qubit collapse the wavefunction and return the measurement result.", "response": "def do_measurement(self, qubit: int) -> int:\n \"\"\"\n Measure a qubit, collapse the wavefunction, and return the measurement result.\n\n :param qubit: Index of the qubit to measure.\n :return: measured bit\n \"\"\"\n if self.rs is None:\n raise ValueError(\"You have tried to perform a stochastic operation without setting the \"\n \"random state of the simulator. Might I suggest using a PyQVM object?\")\n\n # Get probabilities\n measurement_probs = get_measure_probabilities(self.wf, qubit)\n\n # Flip a coin and record the result\n measured_bit = int(np.argmax(self.rs.uniform() < np.cumsum(measurement_probs)))\n\n # Zero out amplitudes corresponding to non-measured bistrings\n other_bit = (measured_bit + 1) % 2\n other_bit_indices = (slice(None),) * qubit + \\\n (other_bit,) + \\\n (slice(None),) * (self.n_qubits - qubit - 1)\n self.wf[other_bit_indices] = 0\n\n # Re-normalize amplitudes corresponding to measured bistrings\n meas_bit_indices = (slice(None),) * qubit + \\\n (measured_bit,) + \\\n (slice(None),) * (self.n_qubits - qubit - 1)\n self.wf[meas_bit_indices] /= np.sqrt(measurement_probs[measured_bit])\n return measured_bit"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nperforms a gate. :return: ``self`` to support method chaining.", "response": "def do_gate(self, gate: Gate):\n \"\"\"\n Perform a gate.\n\n :return: ``self`` to support method chaining.\n \"\"\"\n gate_matrix, qubit_inds = _get_gate_tensor_and_qubits(gate=gate)\n # Note to developers: you can use either einsum- or tensordot- based functions.\n # tensordot seems a little faster, but feel free to experiment.\n # self.wf = targeted_einsum(gate=gate_matrix, wf=self.wf, wf_target_inds=qubit_inds)\n self.wf = targeted_tensordot(gate=gate_matrix, wf=self.wf, wf_target_inds=qubit_inds)\n return self"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef do_gate_matrix(self, matrix: np.ndarray,\n qubits: Sequence[int]) -> 'AbstractQuantumSimulator':\n \"\"\"\n Apply an arbitrary unitary; not necessarily a named gate.\n\n :param matrix: The unitary matrix to apply. No checks are done\n :param qubits: A list of qubits to apply the unitary to.\n :return: ``self`` to support method chaining.\n \"\"\"\n # e.g. 2-qubit matrix is 4x4; turns into (2,2,2,2) tensor.\n tensor = np.reshape(matrix, (2,) * len(qubits) * 2)\n\n # Note to developers: you can use either einsum- or tensordot- based functions.\n # tensordot seems a little faster, but feel free to experiment.\n # self.wf = targeted_einsum(gate=gate_matrix, wf=self.wf, wf_target_inds=qubits)\n self.wf = targeted_tensordot(gate=tensor, wf=self.wf, wf_target_inds=qubits)\n return self", "response": "Apply an arbitrary unitary matrix to the current instance of AbstractQuantumSimulator."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef reset(self):\n self.wf.fill(0)\n self.wf[(0,) * self.n_qubits] = complex(1.0, 0)\n return self", "response": "Reset the wavefunction to the |000... 00 > state."} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nsamples bitstrings from the wavefunction.", "response": "def sample_bitstrings(self, n_samples):\n \"\"\"\n Sample bitstrings from the distribution defined by the wavefunction.\n\n Qubit 0 is at ``out[:, 0]``.\n\n :param n_samples: The number of bitstrings to sample\n :return: An array of shape (n_samples, n_qubits)\n \"\"\"\n if self.rs is None:\n raise ValueError(\"You have tried to perform a stochastic operation without setting the \"\n \"random state of the simulator. Might I suggest using a PyQVM object?\")\n probabilities = np.abs(self.wf) ** 2\n possible_bitstrings = all_bitstrings(self.n_qubits)\n inds = self.rs.choice(2 ** self.n_qubits, n_samples, p=probabilities)\n bitstrings = possible_bitstrings[inds, :]\n bitstrings = np.flip(bitstrings, axis=1) # qubit ordering: 0 on the left.\n return bitstrings"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef do_gate(self, gate: Gate):\n unitary = lifted_gate(gate=gate, n_qubits=self.n_qubits)\n self.wf = unitary.dot(self.wf)\n return self", "response": "Perform a gate.\n\n :return: ``self`` to support method chaining."} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\napplying an arbitrary unitary matrix to the current set of named qubits.", "response": "def do_gate_matrix(self, matrix: np.ndarray, qubits: Sequence[int]):\n \"\"\"\n Apply an arbitrary unitary; not necessarily a named gate.\n\n :param matrix: The unitary matrix to apply. No checks are done.\n :param qubits: The qubits to apply the unitary to.\n :return: ``self`` to support method chaining.\n \"\"\"\n unitary = lifted_gate_matrix(matrix, list(qubits), n_qubits=self.n_qubits)\n self.wf = unitary.dot(self.wf)\n return self"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef expectation(self, operator: Union[PauliTerm, PauliSum]):\n if not isinstance(operator, PauliSum):\n operator = PauliSum([operator])\n\n return sum(_term_expectation(self.wf, term, n_qubits=self.n_qubits) for term in operator)", "response": "Compute the expectation of an operator."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef sample_bitstrings(self, n_samples):\n if self.rs is None:\n raise ValueError(\"You have tried to perform a stochastic operation without setting the \"\n \"random state of the simulator. Might I suggest using a PyQVM object?\")\n probabilities = np.real_if_close(np.diagonal(self.density))\n possible_bitstrings = all_bitstrings(self.n_qubits)\n inds = self.rs.choice(2 ** self.n_qubits, n_samples, p=probabilities)\n bitstrings = possible_bitstrings[inds, :]\n bitstrings = np.flip(bitstrings, axis=1) # qubit ordering: 0 on the left.\n return bitstrings", "response": "Sample bitstrings from the distribution defined by the wavefunction."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef do_gate(self, gate: Gate) -> 'AbstractQuantumSimulator':\n unitary = lifted_gate(gate=gate, n_qubits=self.n_qubits)\n self.density = unitary.dot(self.density).dot(np.conj(unitary).T)\n return self", "response": "Perform a gate.\n\n :return: ``self`` to support method chaining."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef do_gate_matrix(self, matrix: np.ndarray,\n qubits: Sequence[int]) -> 'AbstractQuantumSimulator':\n \"\"\"\n Apply an arbitrary unitary; not necessarily a named gate.\n\n :param matrix: The unitary matrix to apply. No checks are done\n :param qubits: A list of qubits to apply the unitary to.\n :return: ``self`` to support method chaining.\n \"\"\"\n unitary = lifted_gate_matrix(matrix=matrix, qubit_inds=qubits, n_qubits=self.n_qubits)\n self.density = unitary.dot(self.density).dot(np.conj(unitary).T)\n return self", "response": "Apply an arbitrary unitary matrix to the current state."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef do_measurement(self, qubit: int) -> int:\n if self.rs is None:\n raise ValueError(\"You have tried to perform a stochastic operation without setting the \"\n \"random state of the simulator. Might I suggest using a PyQVM object?\")\n measure_0 = lifted_gate_matrix(matrix=P0, qubit_inds=[qubit], n_qubits=self.n_qubits)\n prob_zero = np.trace(measure_0 @ self.density)\n\n # generate random number to 'roll' for measurement\n if self.rs.uniform() < prob_zero:\n # decohere state using the measure_0 operator\n unitary = measure_0 @ (np.eye(2 ** self.n_qubits) / np.sqrt(prob_zero))\n self.density = unitary.dot(self.density).dot(np.conj(unitary.T))\n return 0\n else: # measure one\n measure_1 = lifted_gate_matrix(matrix=P1, qubit_inds=[qubit], n_qubits=self.n_qubits)\n unitary = measure_1 @ (np.eye(2 ** self.n_qubits) / np.sqrt(1 - prob_zero))\n self.density = unitary.dot(self.density).dot(np.conj(unitary.T))\n return 1", "response": "Measure a qubit and collapse the wavefunction\n "} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nruns the ANTLR parser.", "response": "def run_parser(quil):\n # type: (str) -> List[AbstractInstruction]\n \"\"\"\n Run the ANTLR parser.\n\n :param str quil: a single or multiline Quil program\n :return: list of instructions that were parsed\n \"\"\"\n # Step 1: Run the Lexer\n input_stream = InputStream(quil)\n lexer = QuilLexer(input_stream)\n stream = CommonTokenStream(lexer)\n\n # Step 2: Run the Parser\n parser = QuilParser(stream)\n parser.removeErrorListeners()\n parser.addErrorListener(CustomErrorListener())\n tree = parser.quil()\n\n # Step 3: Run the Listener\n pyquil_listener = PyQuilListener()\n walker = ParseTreeWalker()\n walker.walk(pyquil_listener, tree)\n\n return pyquil_listener.result"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns the sequence of items in the sequence that match the given expression.", "response": "def _expression(expression):\n # type: (QuilParser.ExpressionContext) -> Any\n \"\"\"\n NB: Order of operations is already dealt with by the grammar. Here we can simply match on the type.\n \"\"\"\n if isinstance(expression, QuilParser.ParenthesisExpContext):\n return _expression(expression.expression())\n elif isinstance(expression, QuilParser.PowerExpContext):\n if expression.POWER():\n return _binary_exp(expression, operator.pow)\n elif isinstance(expression, QuilParser.MulDivExpContext):\n if expression.TIMES():\n return _binary_exp(expression, operator.mul)\n elif expression.DIVIDE():\n return _binary_exp(expression, operator.truediv)\n elif isinstance(expression, QuilParser.AddSubExpContext):\n if expression.PLUS():\n return _binary_exp(expression, operator.add)\n elif expression.MINUS():\n return _binary_exp(expression, operator.sub)\n elif isinstance(expression, QuilParser.SignedExpContext):\n if expression.sign().PLUS():\n return _expression(expression.expression())\n elif expression.sign().MINUS():\n return -1 * _expression(expression.expression())\n elif isinstance(expression, QuilParser.FunctionExpContext):\n return _apply_function(expression.function(), _expression(expression.expression()))\n elif isinstance(expression, QuilParser.AddrExpContext):\n return _addr(expression.addr())\n elif isinstance(expression, QuilParser.NumberExpContext):\n return _number(expression.number())\n elif isinstance(expression, QuilParser.VariableExpContext):\n return _variable(expression.variable())\n\n raise RuntimeError(\"Unexpected expression type:\" + expression.getText())"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _binary_exp(expression, op):\n # type: (QuilParser.ExpressionContext, Callable) -> Number\n \"\"\"\n Apply an operator to two expressions. Start by evaluating both sides of the operator.\n \"\"\"\n [arg1, arg2] = expression.expression()\n return op(_expression(arg1), _expression(arg2))", "response": "Apply an operator to two expressions."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_expected_tokens(self, parser, interval_set):\n # type: (QuilParser, IntervalSet) -> Iterator\n \"\"\"\n Like the default getExpectedTokens method except that it will fallback to the rule name if the token isn't a\n literal. For instance, instead of for integer it will return the rule name: INT\n \"\"\"\n for tok in interval_set:\n literal_name = parser.literalNames[tok]\n symbolic_name = parser.symbolicNames[tok]\n\n if literal_name != '':\n yield literal_name\n else:\n yield symbolic_name", "response": "Returns an iterator over the expected set of tokens."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef exitCircuitGate(self, ctx: QuilParser.CircuitGateContext):\n gate_name = ctx.name().getText()\n params = [param.getText() for param in ctx.param()]\n qubits = [qubit.getText() for qubit in ctx.circuitQubit()]\n if params:\n self.result.append(RawInstr('{}({}) {}'.format(gate_name, ', '.join(params), ' '.join(qubits))))\n else:\n self.result.append(RawInstr('{} {}'.format(gate_name, ' '.join(qubits))))", "response": "Exit a circuit gate with the given name parameters and qubits."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef unpack_reg_val_pair(classical_reg1, classical_reg2):\n left = unpack_classical_reg(classical_reg1)\n if isinstance(classical_reg2, int) or isinstance(classical_reg2, float):\n right = classical_reg2\n else:\n right = unpack_classical_reg(classical_reg2)\n\n return left, right", "response": "Helper function for typechecking and coercing arguments to constructor for binary classical operators."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef prepare_ternary_operands(classical_reg1, classical_reg2, classical_reg3):\n if isinstance(classical_reg1, int):\n raise TypeError(\"Target operand of comparison must be a memory address\")\n classical_reg1 = unpack_classical_reg(classical_reg1)\n if isinstance(classical_reg2, int):\n raise TypeError(\"Left operand of comparison must be a memory address\")\n classical_reg2 = unpack_classical_reg(classical_reg2)\n if not isinstance(classical_reg3, int):\n classical_reg3 = unpack_classical_reg(classical_reg3)\n\n return classical_reg1, classical_reg2, classical_reg3", "response": "Prepare the operands for ternary classical operators."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nproduce the RX gate.", "response": "def RX(angle, qubit):\n \"\"\"Produces the RX gate::\n\n RX(phi) = [[cos(phi / 2), -1j * sin(phi / 2)],\n [-1j * sin(phi / 2), cos(phi / 2)]]\n\n This gate is a single qubit X-rotation.\n\n :param angle: The angle to rotate around the x-axis on the bloch sphere.\n :param qubit: The qubit apply the gate to.\n :returns: A Gate object.\n \"\"\"\n return Gate(name=\"RX\", params=[angle], qubits=[unpack_qubit(qubit)])"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nproducing the RY gate.", "response": "def RY(angle, qubit):\n \"\"\"Produces the RY gate::\n\n RY(phi) = [[cos(phi / 2), -sin(phi / 2)],\n [sin(phi / 2), cos(phi / 2)]]\n\n This gate is a single qubit Y-rotation.\n\n :param angle: The angle to rotate around the y-axis on the bloch sphere.\n :param qubit: The qubit apply the gate to.\n :returns: A Gate object.\n \"\"\"\n return Gate(name=\"RY\", params=[angle], qubits=[unpack_qubit(qubit)])"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef RZ(angle, qubit):\n return Gate(name=\"RZ\", params=[angle], qubits=[unpack_qubit(qubit)])", "response": "Produces the RZ gate."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef PHASE(angle, qubit):\n return Gate(name=\"PHASE\", params=[angle], qubits=[unpack_qubit(qubit)])", "response": "Produces the PHASE gate."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ngenerates a controlled - X gate instruction.", "response": "def CNOT(control, target):\n \"\"\"Produces a controlled-NOT (controlled-X) gate::\n\n CNOT = [[1, 0, 0, 0],\n [0, 1, 0, 0],\n [0, 0, 0, 1],\n [0, 0, 1, 0]]\n\n This gate applies to two qubit arguments to produce the controlled-not gate instruction.\n\n :param control: The control qubit.\n :param target: The target qubit. The target qubit has an X-gate applied to it if the control\n qubit is in the ``|1>`` state.\n :returns: A Gate object.\n \"\"\"\n return Gate(name=\"CNOT\", params=[], qubits=[unpack_qubit(q) for q in (control, target)])"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ngenerating a controlled - controlled - not gate.", "response": "def CCNOT(control1, control2, target):\n \"\"\"Produces a doubly-controlled NOT gate::\n\n CCNOT = [[1, 0, 0, 0, 0, 0, 0, 0],\n [0, 1, 0, 0, 0, 0, 0, 0],\n [0, 0, 1, 0, 0, 0, 0, 0],\n [0, 0, 0, 1, 0, 0, 0, 0],\n [0, 0, 0, 0, 1, 0, 0, 0],\n [0, 0, 0, 0, 0, 1, 0, 0],\n [0, 0, 0, 0, 0, 0, 0, 1],\n [0, 0, 0, 0, 0, 0, 1, 0]]\n\n This gate applies to three qubit arguments to produce the controlled-controlled-not gate\n instruction.\n\n :param control1: The first control qubit.\n :param control2: The second control qubit.\n :param target: The target qubit. The target qubit has an X-gate applied to it if both control\n qubits are in the excited state.\n :returns: A Gate object.\n \"\"\"\n qubits = [unpack_qubit(q) for q in (control1, control2, target)]\n return Gate(name=\"CCNOT\", params=[], qubits=qubits)"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef CPHASE00(angle, control, target):\n qubits = [unpack_qubit(q) for q in (control, target)]\n return Gate(name=\"CPHASE00\", params=[angle], qubits=qubits)", "response": "Returns a controlled - phase gate that phases the |00 > state."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef CSWAP(control, target_1, target_2):\n qubits = [unpack_qubit(q) for q in (control, target_1, target_2)]\n return Gate(name=\"CSWAP\", params=[], qubits=qubits)", "response": "Generates a controlled - SWAP gate."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ngenerating an ISWAP gate that swaps the state of two qubits.", "response": "def ISWAP(q1, q2):\n \"\"\"Produces an ISWAP gate::\n\n ISWAP = [[1, 0, 0, 0],\n [0, 0, 1j, 0],\n [0, 1j, 0, 0],\n [0, 0, 0, 1]]\n\n This gate swaps the state of two qubits, applying a -i phase to q1 when it\n is in the 1 state and a -i phase to q2 when it is in the 0 state.\n\n :param q1: Qubit 1.\n :param q2: Qubit 2.\n :returns: A Gate object.\n \"\"\"\n return Gate(name=\"ISWAP\", params=[], qubits=[unpack_qubit(q) for q in (q1, q2)])"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef PSWAP(angle, q1, q2):\n return Gate(name=\"PSWAP\", params=[angle], qubits=[unpack_qubit(q) for q in (q1, q2)])", "response": "Returns a parameterized SWAP gate."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef MEASURE(qubit, classical_reg):\n qubit = unpack_qubit(qubit)\n if classical_reg is None:\n address = None\n elif isinstance(classical_reg, int):\n warn(\"Indexing measurement addresses by integers is deprecated. \"\n + \"Replacing this with the MemoryReference ro[i] instead.\")\n address = MemoryReference(\"ro\", classical_reg)\n else:\n address = unpack_classical_reg(classical_reg)\n return Measurement(qubit, address)", "response": "Produce a MEASURE instruction."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef TRUE(classical_reg):\n warn(\"`TRUE a` has been deprecated. Use `MOVE a 1` instead.\")\n if isinstance(classical_reg, int):\n classical_reg = Addr(classical_reg)\n return MOVE(classical_reg, 1)", "response": "Produce a TRUE instruction."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef FALSE(classical_reg):\n warn(\"`FALSE a` has been deprecated. Use `MOVE a 0` instead.\")\n if isinstance(classical_reg, int):\n classical_reg = Addr(classical_reg)\n return MOVE(classical_reg, 0)", "response": "Produce a FALSE instruction."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nproduces an AND instruction.", "response": "def AND(classical_reg1, classical_reg2):\n \"\"\"\n Produce an AND instruction.\n\n NOTE: The order of operands was reversed in pyQuil <=1.9 .\n\n :param classical_reg1: The first classical register, which gets modified.\n :param classical_reg2: The second classical register or immediate value.\n :return: A ClassicalAnd instance.\n \"\"\"\n left, right = unpack_reg_val_pair(classical_reg1, classical_reg2)\n return ClassicalAnd(left, right)"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef IOR(classical_reg1, classical_reg2):\n left, right = unpack_reg_val_pair(classical_reg1, classical_reg2)\n return ClassicalInclusiveOr(left, right)", "response": "Produce an inclusive OR instruction."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nproduce an exclusive OR instruction.", "response": "def XOR(classical_reg1, classical_reg2):\n \"\"\"\n Produce an exclusive OR instruction.\n\n :param classical_reg1: The first classical register, which gets modified.\n :param classical_reg2: The second classical register or immediate value.\n :return: A ClassicalOr instance.\n \"\"\"\n left, right = unpack_reg_val_pair(classical_reg1, classical_reg2)\n return ClassicalExclusiveOr(left, right)"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef MOVE(classical_reg1, classical_reg2):\n left, right = unpack_reg_val_pair(classical_reg1, classical_reg2)\n return ClassicalMove(left, right)", "response": "Produce a MOVE instruction."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef EXCHANGE(classical_reg1, classical_reg2):\n left = unpack_classical_reg(classical_reg1)\n right = unpack_classical_reg(classical_reg2)\n return ClassicalExchange(left, right)", "response": "Produce an EXCHANGE instruction."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef LOAD(target_reg, region_name, offset_reg):\n return ClassicalLoad(unpack_classical_reg(target_reg), region_name, unpack_classical_reg(offset_reg))", "response": "Produce a LOAD instruction."} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nproduces an ADD instruction.", "response": "def ADD(classical_reg, right):\n \"\"\"\n Produce an ADD instruction.\n\n :param classical_reg: Left operand for the arithmetic operation. Also serves as the store target.\n :param right: Right operand for the arithmetic operation.\n :return: A ClassicalAdd instance.\n \"\"\"\n left, right = unpack_reg_val_pair(classical_reg, right)\n return ClassicalAdd(left, right)"} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nproduces a SUB instruction.", "response": "def SUB(classical_reg, right):\n \"\"\"\n Produce a SUB instruction.\n\n :param classical_reg: Left operand for the arithmetic operation. Also serves as the store target.\n :param right: Right operand for the arithmetic operation.\n :return: A ClassicalSub instance.\n \"\"\"\n left, right = unpack_reg_val_pair(classical_reg, right)\n return ClassicalSub(left, right)"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef MUL(classical_reg, right):\n left, right = unpack_reg_val_pair(classical_reg, right)\n return ClassicalMul(left, right)", "response": "Produce a MUL instruction."} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nproducing an DIV instruction.", "response": "def DIV(classical_reg, right):\n \"\"\"\n Produce an DIV instruction.\n\n :param classical_reg: Left operand for the arithmetic operation. Also serves as the store target.\n :param right: Right operand for the arithmetic operation.\n :return: A ClassicalDiv instance.\n \"\"\"\n left, right = unpack_reg_val_pair(classical_reg, right)\n return ClassicalDiv(left, right)"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nproducing an EQ instruction.", "response": "def EQ(classical_reg1, classical_reg2, classical_reg3):\n \"\"\"\n Produce an EQ instruction.\n\n :param classical_reg1: Memory address to which to store the comparison result.\n :param classical_reg2: Left comparison operand.\n :param classical_reg3: Right comparison operand.\n :return: A ClassicalEqual instance.\n \"\"\"\n classical_reg1, classical_reg2, classical_reg3 = prepare_ternary_operands(classical_reg1,\n classical_reg2,\n classical_reg3)\n\n return ClassicalEqual(classical_reg1, classical_reg2, classical_reg3)"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef LT(classical_reg1, classical_reg2, classical_reg3):\n classical_reg1, classical_reg2, classical_reg3 = prepare_ternary_operands(classical_reg1,\n classical_reg2,\n classical_reg3)\n return ClassicalLessThan(classical_reg1, classical_reg2, classical_reg3)", "response": "Produce an LT instruction."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef LE(classical_reg1, classical_reg2, classical_reg3):\n classical_reg1, classical_reg2, classical_reg3 = prepare_ternary_operands(classical_reg1,\n classical_reg2,\n classical_reg3)\n return ClassicalLessEqual(classical_reg1, classical_reg2, classical_reg3)", "response": "Produce an LE instruction."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nproduce an GT instruction.", "response": "def GT(classical_reg1, classical_reg2, classical_reg3):\n \"\"\"\n Produce an GT instruction.\n\n :param classical_reg1: Memory address to which to store the comparison result.\n :param classical_reg2: Left comparison operand.\n :param classical_reg3: Right comparison operand.\n :return: A ClassicalGreaterThan instance.\n \"\"\"\n classical_reg1, classical_reg2, classical_reg3 = prepare_ternary_operands(classical_reg1,\n classical_reg2,\n classical_reg3)\n return ClassicalGreaterThan(classical_reg1, classical_reg2, classical_reg3)"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef GE(classical_reg1, classical_reg2, classical_reg3):\n classical_reg1, classical_reg2, classical_reg3 = prepare_ternary_operands(classical_reg1,\n classical_reg2,\n classical_reg3)\n return ClassicalGreaterEqual(classical_reg1, classical_reg2, classical_reg3)", "response": "Produce an GE instruction."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _if_statement(test, if_function, else_function) -> None:\n if isinstance(test, Addr):\n token = _program_context.set(Program())\n if_function()\n if_program = _program_context.get()\n _program_context.reset(token)\n\n if else_function:\n token = _program_context.set(Program())\n else_function()\n else_program = _program_context.get()\n _program_context.reset(token)\n else:\n else_program = None\n\n program = _program_context.get()\n program.if_then(test, if_program, else_program)\n else:\n if test:\n if_function()\n elif else_function:\n else_function()", "response": "Evaluate an if statement within a MagicQuil block."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _rewrite_function(f):\n source = inspect.getsource(f)\n tree = ast.parse(source)\n _IfTransformer().visit(tree)\n\n ast.fix_missing_locations(tree)\n tree.body[0].name = f.__name__ + '_patched'\n tree.body[0].decorator_list = []\n\n compiled = compile(tree, filename='', mode='exec')\n # The first f_back here gets to the body of magicquil() and the second f_back gets to the user's call site which\n # is what we want. If we didn't add these manually to the globals it wouldn't be possible to call other @magicquil\n # functions from within a @magicquil function.\n prev_globals = inspect.currentframe().f_back.f_back.f_globals\n # For reasons I don't quite understand it's critical to add locals() here otherwise the function will disappear and\n # we won't be able to return it below\n exec(compiled, {**prev_globals, **globals()}, locals())\n return locals()[f.__name__ + '_patched']", "response": "Rewrite a function f to be overridden by any if branches."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a dictionary mapping each qubit to an integer from 0 through N - 1.", "response": "def get_default_qubit_mapping(program):\n \"\"\"\n Takes a program which contains qubit placeholders and provides a mapping to the integers\n 0 through N-1.\n\n The output of this function is suitable for input to :py:func:`address_qubits`.\n\n :param program: A program containing qubit placeholders\n :return: A dictionary mapping qubit placeholder to an addressed qubit from 0 through N-1.\n \"\"\"\n fake_qubits, real_qubits, qubits = _what_type_of_qubit_does_it_use(program)\n if real_qubits:\n warnings.warn(\"This program contains integer qubits, \"\n \"so getting a mapping doesn't make sense.\")\n return {q: q for q in qubits}\n return {qp: Qubit(i) for i, qp in enumerate(qubits)}"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef address_qubits(program, qubit_mapping=None):\n fake_qubits, real_qubits, qubits = _what_type_of_qubit_does_it_use(program)\n if real_qubits:\n if qubit_mapping is not None:\n warnings.warn(\"A qubit mapping was provided but the program does not \"\n \"contain any placeholders to map!\")\n return program\n\n if qubit_mapping is None:\n qubit_mapping = {qp: Qubit(i) for i, qp in enumerate(qubits)}\n else:\n if all(isinstance(v, Qubit) for v in qubit_mapping.values()):\n pass # we good\n elif all(isinstance(v, int) for v in qubit_mapping.values()):\n qubit_mapping = {k: Qubit(v) for k, v in qubit_mapping.items()}\n else:\n raise ValueError(\"Qubit mapping must map to type Qubit or int (but not both)\")\n\n result = []\n for instr in program:\n # Remap qubits on Gate and Measurement instructions\n if isinstance(instr, Gate):\n remapped_qubits = [qubit_mapping[q] for q in instr.qubits]\n result.append(Gate(instr.name, instr.params, remapped_qubits))\n elif isinstance(instr, Measurement):\n result.append(Measurement(qubit_mapping[instr.qubit], instr.classical_reg))\n elif isinstance(instr, Pragma):\n new_args = []\n for arg in instr.args:\n # Pragmas can have arguments that represent things besides qubits, so here we\n # make sure to just look up the QubitPlaceholders.\n if isinstance(arg, QubitPlaceholder):\n new_args.append(qubit_mapping[arg])\n else:\n new_args.append(arg)\n result.append(Pragma(instr.command, new_args, instr.freeform_string))\n # Otherwise simply add it to the result\n else:\n result.append(instr)\n\n new_program = program.copy()\n new_program._instructions = result\n\n return new_program", "response": "Given a program and a dictionary of qubits and labels return a new program which contains all of the qubits and labels assigned to those qubits."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _get_label(placeholder, label_mapping, label_i):\n if placeholder in label_mapping:\n return label_mapping[placeholder], label_mapping, label_i\n\n new_target = Label(\"{}{}\".format(placeholder.prefix, label_i))\n label_i += 1\n label_mapping[placeholder] = new_target\n return new_target, label_mapping, label_i", "response": "Helper function to get the appropriate label for a given placeholder."} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ntaking an iterable of instructions which may contain label placeholders and assigns them all defined values.", "response": "def instantiate_labels(instructions):\n \"\"\"\n Takes an iterable of instructions which may contain label placeholders and assigns\n them all defined values.\n\n :return: list of instructions with all label placeholders assigned to real labels.\n \"\"\"\n label_i = 1\n result = []\n label_mapping = dict()\n for instr in instructions:\n if isinstance(instr, Jump) and isinstance(instr.target, LabelPlaceholder):\n new_target, label_mapping, label_i = _get_label(instr.target, label_mapping, label_i)\n result.append(Jump(new_target))\n elif isinstance(instr, JumpConditional) and isinstance(instr.target, LabelPlaceholder):\n new_target, label_mapping, label_i = _get_label(instr.target, label_mapping, label_i)\n cls = instr.__class__ # Make the correct subclass\n result.append(cls(new_target, instr.condition))\n elif isinstance(instr, JumpTarget) and isinstance(instr.label, LabelPlaceholder):\n new_label, label_mapping, label_i = _get_label(instr.label, label_mapping, label_i)\n result.append(JumpTarget(new_label))\n else:\n result.append(instr)\n\n return result"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a new list of programs with noisy gates inserted between each element of the list.", "response": "def merge_with_pauli_noise(prog_list: Iterable, probabilities: List, qubits: List):\n \"\"\"\n Insert pauli noise channels between each item in the list of programs.\n This noise channel is implemented as a single noisy identity gate acting on the provided qubits.\n This method does not rely on merge_programs and so avoids the inclusion of redundant Kraus Pragmas\n that would occur if merge_programs was called directly on programs with distinct noisy gate definitions.\n\n :param prog_list: an iterable such as a program or a list of programs.\n If a program is provided, a single noise gate will be applied after each gate in the program.\n If a list of programs is provided, the noise gate will be applied after each program.\n :param probabilities: The 4^num_qubits list of probabilities specifying the desired pauli channel.\n There should be either 4 or 16 probabilities specified in the order\n I, X, Y, Z or II, IX, IY, IZ, XI, XX, XY, etc respectively.\n :param qubits: a list of the qubits that the noisy gate should act on.\n :return: A single program with noisy gates inserted between each element of the program list.\n :rtype: Program\n \"\"\"\n p = Program()\n p.defgate(\"pauli_noise\", np.eye(2 ** len(qubits)))\n p.define_noisy_gate(\"pauli_noise\", qubits, pauli_kraus_map(probabilities))\n for elem in prog_list:\n p.inst(Program(elem))\n if isinstance(elem, Measurement):\n continue # do not apply noise after measurement\n p.inst((\"pauli_noise\", *qubits))\n return p"} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nmerges a list of pyQuil programs into a single pyQuil program.", "response": "def merge_programs(prog_list):\n \"\"\"\n Merges a list of pyQuil programs into a single one by appending them in sequence.\n If multiple programs in the list contain the same gate and/or noisy gate definition\n with identical name, this definition will only be applied once. If different definitions\n with the same name appear multiple times in the program list, each will be applied once\n in the order of last occurrence.\n\n :param list prog_list: A list of pyquil programs\n :return: a single pyQuil program\n :rtype: Program\n \"\"\"\n definitions = [gate for prog in prog_list for gate in Program(prog).defined_gates]\n seen = {}\n # Collect definitions in reverse order and reapply definitions in reverse\n # collected order to ensure that the last occurrence of a definition is applied last.\n for definition in reversed(definitions):\n name = definition.name\n if name in seen.keys():\n # Do not add truly identical definitions with the same name\n # If two different definitions share a name, we include each definition so as to provide\n # a waring to the user when the contradictory defgate is called.\n if definition not in seen[name]:\n seen[name].append(definition)\n else:\n seen[name] = [definition]\n new_definitions = [gate for key in seen.keys() for gate in reversed(seen[key])]\n\n p = sum([Program(prog).instructions for prog in prog_list], Program()) # Combine programs without gate definitions\n\n for definition in new_definitions:\n p.defgate(definition.name, definition.matrix, definition.parameters)\n\n return p"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_classical_addresses_from_program(program) -> Dict[str, List[int]]:\n addresses: Dict[str, List[int]] = defaultdict(list)\n flattened_addresses = {}\n\n # Required to use the `classical_reg.address` int attribute.\n # See https://github.com/rigetti/pyquil/issues/388.\n for instr in program:\n if isinstance(instr, Measurement) and instr.classical_reg:\n addresses[instr.classical_reg.name].append(instr.classical_reg.offset)\n\n # flatten duplicates\n for k, v in addresses.items():\n reduced_list = list(set(v))\n reduced_list.sort()\n flattened_addresses[k] = reduced_list\n\n return flattened_addresses", "response": "Returns a sorted list of classical addresses found in the MEASURE instructions in the program."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns a new program with all the DECLARE statements moved to the top of the program.", "response": "def percolate_declares(program: Program) -> Program:\n \"\"\"\n Move all the DECLARE statements to the top of the program. Return a fresh obejct.\n\n :param program: Perhaps jumbled program.\n :return: Program with DECLAREs all at the top and otherwise the same sorted contents.\n \"\"\"\n declare_program = Program()\n instrs_program = Program()\n\n for instr in program:\n if isinstance(instr, Declare):\n declare_program += instr\n else:\n instrs_program += instr\n\n p = declare_program + instrs_program\n p._defined_gates = program._defined_gates\n\n return p"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef validate_protoquil(program: Program) -> None:\n valid_instruction_types = tuple([Pragma, Declare, Halt, Gate, Reset, ResetQubit, Measurement])\n for instr in program.instructions:\n if not isinstance(instr, valid_instruction_types):\n # Instructions like MOVE, NOT, JUMP, JUMP-UNLESS will fail here\n raise ValueError(f\"ProtoQuil validation failed: {instr} is not allowed.\")", "response": "Validates that a program is a subset of ProtoQuil."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nvalidates that a program is supported Quil.", "response": "def validate_supported_quil(program: Program) -> None:\n \"\"\"\n Ensure that a program is supported Quil which can run on any QPU, otherwise raise a ValueError.\n We support a global RESET before any gates, and MEASUREs on each qubit after any gates\n on that qubit. PRAGMAs and DECLAREs are always allowed, and a final HALT instruction is allowed.\n\n :param program: The Quil program to validate.\n \"\"\"\n gates_seen = False\n measured_qubits: Set[int] = set()\n for i, instr in enumerate(program.instructions):\n if isinstance(instr, Pragma) or isinstance(instr, Declare):\n continue\n elif isinstance(instr, Halt):\n if i != len(program.instructions) - 1:\n raise ValueError(f\"Cannot have instructions after HALT\")\n elif isinstance(instr, Gate):\n gates_seen = True\n if any(q.index in measured_qubits for q in instr.qubits):\n raise ValueError(\"Cannot apply gates to qubits that were already measured.\")\n elif isinstance(instr, Reset):\n if gates_seen:\n raise ValueError(\"RESET can only be applied before any gate applications.\")\n elif isinstance(instr, ResetQubit):\n raise ValueError(\"Only global RESETs are currently supported.\")\n elif isinstance(instr, Measurement):\n if instr.qubit.index in measured_qubits:\n raise ValueError(\"Multiple measurements per qubit is not supported.\")\n measured_qubits.add(instr.qubit.index)\n else:\n raise ValueError(f\"Unhandled instruction type in supported Quil validation: {instr}\")"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef copy_everything_except_instructions(self):\n new_prog = Program()\n new_prog._defined_gates = self._defined_gates.copy()\n if self.native_quil_metadata is not None:\n new_prog.native_quil_metadata = self.native_quil_metadata.copy()\n new_prog.num_shots = self.num_shots\n return new_prog", "response": "Returns a new Program object with all the members that live on a Program object."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nperforming a shallow copy of this program and its instructions.", "response": "def copy(self):\n \"\"\"\n Perform a shallow copy of this program.\n\n QuilAtom and AbstractInstruction objects should be treated as immutable to avoid\n strange behavior when performing a copy.\n\n :return: a new Program\n \"\"\"\n new_prog = self.copy_everything_except_instructions()\n new_prog._instructions = self._instructions.copy()\n return new_prog"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef inst(self, *instructions):\n for instruction in instructions:\n if isinstance(instruction, list):\n self.inst(*instruction)\n elif isinstance(instruction, types.GeneratorType):\n self.inst(*instruction)\n elif isinstance(instruction, tuple):\n if len(instruction) == 0:\n raise ValueError(\"tuple should have at least one element\")\n elif len(instruction) == 1:\n self.inst(instruction[0])\n else:\n op = instruction[0]\n if op == \"MEASURE\":\n if len(instruction) == 2:\n self.measure(instruction[1], None)\n else:\n self.measure(instruction[1], instruction[2])\n else:\n params = []\n possible_params = instruction[1]\n rest = instruction[2:]\n if isinstance(possible_params, list):\n params = possible_params\n else:\n rest = [possible_params] + list(rest)\n self.gate(op, params, rest)\n elif isinstance(instruction, string_types):\n self.inst(run_parser(instruction.strip()))\n elif isinstance(instruction, Program):\n if id(self) == id(instruction):\n raise ValueError(\"Nesting a program inside itself is not supported\")\n\n for defgate in instruction._defined_gates:\n self.inst(defgate)\n for instr in instruction._instructions:\n self.inst(instr)\n\n # Implementation note: these two base cases are the only ones which modify the program\n elif isinstance(instruction, DefGate):\n defined_gate_names = [gate.name for gate in self._defined_gates]\n if instruction.name in defined_gate_names:\n warnings.warn(\"Gate {} has already been defined in this program\"\n .format(instruction.name))\n\n self._defined_gates.append(instruction)\n elif isinstance(instruction, AbstractInstruction):\n self._instructions.append(instruction)\n self._synthesized_instructions = None\n else:\n raise TypeError(\"Invalid instruction: {}\".format(instruction))\n\n return self", "response": "Mutates the program object by appending new instructions."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef gate(self, name, params, qubits):\n return self.inst(Gate(name, params, [unpack_qubit(q) for q in qubits]))", "response": "Adds a gate to the program."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef defgate(self, name, matrix, parameters=None):\n return self.inst(DefGate(name, matrix, parameters))", "response": "Define a new static gate."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef define_noisy_gate(self, name, qubit_indices, kraus_ops):\n kraus_ops = [np.asarray(k, dtype=np.complex128) for k in kraus_ops]\n _check_kraus_ops(len(qubit_indices), kraus_ops)\n return self.inst(_create_kraus_pragmas(name, tuple(qubit_indices), kraus_ops))", "response": "Define a noisy static ideal gate with a noisy one defined in terms of a Kraus map."} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ndefines a classical bit flip readout error channel parametrized by p00 and p11.", "response": "def define_noisy_readout(self, qubit, p00, p11):\n \"\"\"\n For this program define a classical bit flip readout error channel parametrized by\n ``p00`` and ``p11``. This models the effect of thermal noise that corrupts the readout\n signal **after** it has interrogated the qubit.\n\n :param int|QubitPlaceholder qubit: The qubit with noisy readout.\n :param float p00: The probability of obtaining the measurement result 0 given that the qubit\n is in state 0.\n :param float p11: The probability of obtaining the measurement result 1 given that the qubit\n is in state 1.\n :return: The Program with an appended READOUT-POVM Pragma.\n :rtype: Program\n \"\"\"\n if not 0. <= p00 <= 1.:\n raise ValueError(\"p00 must be in the interval [0,1].\")\n if not 0. <= p11 <= 1.:\n raise ValueError(\"p11 must be in the interval [0,1].\")\n if not (isinstance(qubit, int) or isinstance(qubit, QubitPlaceholder)):\n raise TypeError(\"qubit must be a non-negative integer, or QubitPlaceholder.\")\n if isinstance(qubit, int) and qubit < 0:\n raise ValueError(\"qubit cannot be negative.\")\n p00 = float(p00)\n p11 = float(p11)\n aprobs = [p00, 1. - p11, 1. - p00, p11]\n aprobs_str = \"({})\".format(\" \".join(format_parameter(p) for p in aprobs))\n pragma = Pragma(\"READOUT-POVM\", [qubit], aprobs_str)\n return self.inst(pragma)"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef measure_all(self, *qubit_reg_pairs):\n if qubit_reg_pairs == ():\n qubit_inds = self.get_qubits(indices=True)\n if len(qubit_inds) == 0:\n return self\n ro = self.declare('ro', 'BIT', max(qubit_inds) + 1)\n for qi in qubit_inds:\n self.inst(MEASURE(qi, ro[qi]))\n else:\n for qubit_index, classical_reg in qubit_reg_pairs:\n self.inst(MEASURE(qubit_index, classical_reg))\n return self", "response": "Measure all classical bits in the program into their specified classical bits in the order\n they were entered."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef while_do(self, classical_reg, q_program):\n label_start = LabelPlaceholder(\"START\")\n label_end = LabelPlaceholder(\"END\")\n self.inst(JumpTarget(label_start))\n self.inst(JumpUnless(target=label_end, condition=classical_reg))\n self.inst(q_program)\n self.inst(Jump(label_start))\n self.inst(JumpTarget(label_end))\n return self", "response": "The while loop instruction."} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nruns if_program if the classical register at index classical reg is 1 run else_program if the classical register at index classical reg is 0 run else_program.", "response": "def if_then(self, classical_reg, if_program, else_program=None):\n \"\"\"\n If the classical register at index classical reg is 1, run if_program, else run\n else_program.\n\n Equivalent to the following construction:\n\n .. code::\n\n IF [c]:\n instrA...\n ELSE:\n instrB...\n =>\n JUMP-WHEN @THEN [c]\n instrB...\n JUMP @END\n LABEL @THEN\n instrA...\n LABEL @END\n\n :param int classical_reg: The classical register to check as the condition\n :param Program if_program: A Quil program to execute if classical_reg is 1\n :param Program else_program: A Quil program to execute if classical_reg is 0. This\n argument is optional and defaults to an empty Program.\n :returns: The Quil Program with the branching instructions added.\n :rtype: Program\n \"\"\"\n else_program = else_program if else_program is not None else Program()\n\n label_then = LabelPlaceholder(\"THEN\")\n label_end = LabelPlaceholder(\"END\")\n self.inst(JumpWhen(target=label_then, condition=unpack_classical_reg(classical_reg)))\n self.inst(else_program)\n self.inst(Jump(label_end))\n self.inst(JumpTarget(label_then))\n self.inst(if_program)\n self.inst(JumpTarget(label_end))\n return self"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ndeclare a quil variable This adds the declaration to the current program and returns a MemoryReference to the base (offset = 0) of the declared memory. .. note:: This function returns a MemoryReference and cannot be chained like some of the other Program methods. Consider using ``inst(DECLARE(...))`` if you would like to chain methods, but please be aware that you must create your own MemoryReferences later on. :param name: Name of the declared variable :param memory_type: Type of the declared memory: 'BIT', 'REAL', 'OCTET' or 'INTEGER' :param memory_size: Number of array elements in the declared memory. :param shared_region: You can declare a variable that shares its underlying memory with another region. This allows aliasing. For example, you can interpret an array of measured bits as an integer. :param offsets: If you are using ``shared_region``, this allows you to share only a part of the parent region. The offset is given by an array type and the number of elements of that type. For example, ``DECLARE target-bit BIT SHARING real-region OFFSET 1 REAL 4 BIT`` will let you use target-bit to poke into the fourth bit of the second real from the leading edge of real-region. :return: a MemoryReference to the start of the declared memory region, ie a memory reference to ``name[0]``.", "response": "def declare(self, name, memory_type='BIT', memory_size=1, shared_region=None, offsets=None):\n \"\"\"DECLARE a quil variable\n\n This adds the declaration to the current program and returns a MemoryReference to the\n base (offset = 0) of the declared memory.\n\n .. note::\n This function returns a MemoryReference and cannot be chained like some\n of the other Program methods. Consider using ``inst(DECLARE(...))`` if you\n would like to chain methods, but please be aware that you must create your\n own MemoryReferences later on.\n\n :param name: Name of the declared variable\n :param memory_type: Type of the declared memory: 'BIT', 'REAL', 'OCTET' or 'INTEGER'\n :param memory_size: Number of array elements in the declared memory.\n :param shared_region: You can declare a variable that shares its underlying memory\n with another region. This allows aliasing. For example, you can interpret an array\n of measured bits as an integer.\n :param offsets: If you are using ``shared_region``, this allows you to share only\n a part of the parent region. The offset is given by an array type and the number\n of elements of that type. For example,\n ``DECLARE target-bit BIT SHARING real-region OFFSET 1 REAL 4 BIT`` will let you use\n target-bit to poke into the fourth bit of the second real from the leading edge of\n real-region.\n :return: a MemoryReference to the start of the declared memory region, ie a memory\n reference to ``name[0]``.\n \"\"\"\n self.inst(Declare(name=name, memory_type=memory_type, memory_size=memory_size,\n shared_region=shared_region, offsets=offsets))\n return MemoryReference(name=name, declared_size=memory_size)"} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nconverts the Quil program to a readable string.", "response": "def _out(self, allow_placeholders):\n \"\"\"\n Converts the Quil program to a readable string.\n\n :param allow_placeholders: Whether to complain if the program contains placeholders.\n \"\"\"\n return '\\n'.join(itertools.chain(\n (dg.out() for dg in self._defined_gates),\n (instr.out(allow_placeholders=allow_placeholders) for instr in self.instructions),\n [''],\n ))"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef get_qubits(self, indices=True):\n qubits = set()\n for instr in self.instructions:\n if isinstance(instr, (Gate, Measurement)):\n qubits |= instr.get_qubits(indices=indices)\n return qubits", "response": "Returns all of the qubit indices used in this program including gate applications and the allocated qubits."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef dagger(self, inv_dict=None, suffix=\"-INV\"):\n if not self.is_protoquil():\n raise ValueError(\"Program must be valid Protoquil\")\n\n daggered = Program()\n\n for gate in self._defined_gates:\n if inv_dict is None or gate.name not in inv_dict:\n if gate.parameters:\n raise TypeError(\"Cannot auto define daggered version of parameterized gates\")\n daggered.defgate(gate.name + suffix, gate.matrix.T.conj())\n\n for gate in reversed(self._instructions):\n if gate.name in QUANTUM_GATES:\n if gate.name == \"S\":\n daggered.inst(QUANTUM_GATES[\"PHASE\"](-pi / 2, *gate.qubits))\n elif gate.name == \"T\":\n daggered.inst(QUANTUM_GATES[\"RZ\"](pi / 4, *gate.qubits))\n elif gate.name == \"ISWAP\":\n daggered.inst(QUANTUM_GATES[\"PSWAP\"](pi / 2, *gate.qubits))\n else:\n negated_params = list(map(lambda x: -1 * x, gate.params))\n daggered.inst(QUANTUM_GATES[gate.name](*(negated_params + gate.qubits)))\n else:\n if inv_dict is None or gate.name not in inv_dict:\n gate_inv_name = gate.name + suffix\n else:\n gate_inv_name = inv_dict[gate.name]\n\n daggered.inst(Gate(gate_inv_name, gate.params, gate.qubits))\n\n return daggered", "response": "Creates a daggered version of the program."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nassigning all placeholder labels to actual values and implicitly declares the ``ro`` register for backwards compatibility. Changed in 1.9: Either all qubits must be defined or all undefined. If qubits are undefined, this method will not help you. You must explicitly call `address_qubits` which will return a new Program. Changed in 1.9: This function now returns ``self`` and updates ``self._synthesized_instructions``. Changed in 2.0: This function will add an instruction to the top of the program to declare a register of bits called ``ro`` if and only if there are no other declarations in the program. :return: This object with the ``_synthesized_instructions`` member set.", "response": "def _synthesize(self):\n \"\"\"\n Assigns all placeholder labels to actual values and implicitly declares the ``ro``\n register for backwards compatibility.\n\n Changed in 1.9: Either all qubits must be defined or all undefined. If qubits are\n undefined, this method will not help you. You must explicitly call `address_qubits`\n which will return a new Program.\n\n Changed in 1.9: This function now returns ``self`` and updates\n ``self._synthesized_instructions``.\n\n Changed in 2.0: This function will add an instruction to the top of the program\n to declare a register of bits called ``ro`` if and only if there are no other\n declarations in the program.\n\n :return: This object with the ``_synthesized_instructions`` member set.\n \"\"\"\n self._synthesized_instructions = instantiate_labels(self._instructions)\n self._synthesized_instructions = implicitly_declare_ro(self._synthesized_instructions)\n return self"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _record_call(func):\n\n @wraps(func)\n def wrapper(*args, **kwargs):\n global global_error_context\n\n # log a call as about to take place\n if global_error_context is not None:\n key = CallLogKey(name=func.__name__,\n args=[serialize_object_for_logging(arg) for arg in args],\n kwargs={k: serialize_object_for_logging(v) for k, v in kwargs.items()})\n\n pre_entry = CallLogValue(timestamp_in=datetime.utcnow(),\n timestamp_out=None,\n return_value=None)\n global_error_context.log[key] = pre_entry\n\n val = func(*args, **kwargs)\n\n # poke the return value of that call in\n if global_error_context is not None:\n post_entry = CallLogValue(timestamp_in=pre_entry.timestamp_in,\n timestamp_out=datetime.utcnow(),\n return_value=serialize_object_for_logging(val))\n global_error_context.log[key] = post_entry\n\n return val\n\n return wrapper", "response": "A decorator that logs a call into the global error context."} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ngenerates a report object from an error generated in a routine decorated with the pyQuil error handler.", "response": "def generate_report(self, exception, trace):\n \"\"\"\n Handle an error generated in a routine decorated with the pyQuil error handler.\n\n :param exception: Exception object that generated this error.\n :param trace: inspect.trace object from the frame that caught the error.\n :return: ErrorReport object\n \"\"\"\n stack_trace = [StacktraceFrame(name=item.function,\n filename=item.filename,\n line_number=item.lineno,\n locals={k: serialize_object_for_logging(v)\n for (k, v) in item.frame.f_locals.items()})\n for item in trace]\n\n system_info = generate_system_info()\n\n report = ErrorReport(stack_trace=stack_trace,\n timestamp=datetime.utcnow(),\n exception=exception,\n system_info=system_info,\n call_log=flatten_log(self.log))\n\n return report"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning a pyQuil. Program that is a ucc circuit with the given rotation parameter.", "response": "def ucc_circuit(theta):\n \"\"\"\n Implements\n\n exp(-i theta X_{0}Y_{1})\n\n :param theta: rotation parameter\n :return: pyquil.Program\n \"\"\"\n generator = sX(0) * sY(1)\n initial_prog = Program().inst(X(1), X(0))\n\n # compiled program\n program = initial_prog + exponentiate(float(theta) * generator) # float is required because pyquil has weird casting behavior\n return program"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef objective_fun(theta, hamiltonian=None,\n quantum_resource=QVMConnection(sync_endpoint='http://localhost:5000')):\n \"\"\"\n Evaluate the Hamiltonian bny operator averaging\n\n :param theta:\n :param hamiltonian:\n :return:\n \"\"\"\n if hamiltonian is None:\n # Hamiltonian is Identity\n return 1.0\n\n if isinstance(hamiltonian, PauliSum):\n result = estimate_locally_commuting_operator(ucc_circuit(theta), hamiltonian,\n 1.0E-6, quantum_resource=quantum_resource)\n result = result[0][0].real # first output is expected value, second is variance, third is shots\n elif isinstance(hamiltonian, np.ndarray) and isinstance(quantum_resource, QVMConnection):\n wf = quantum_resource.wavefunction(ucc_circuit(theta))\n wf = wf.amplitudes.reshape((-1, 1))\n result = np.conj(wf).T.dot(hamiltonian).dot(wf)[0, 0].real\n print(result)\n else:\n raise TypeError(\"type of hamiltonian or qvm is unrecognized\")\n\n return result", "response": "Evaluate the Hamiltonian bny operator averaging\n "} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncheck that this program is a series of quantum gates with terminal MEASURE instructions; pop MEASURE instructions. :param program: The program :return: A new program with MEASURE instructions removed.", "response": "def _make_ram_program(program):\n \"\"\"\n Check that this program is a series of quantum gates with terminal MEASURE instructions; pop\n MEASURE instructions.\n\n :param program: The program\n :return: A new program with MEASURE instructions removed.\n \"\"\"\n new_prog = program.copy_everything_except_instructions()\n last_qubit_operation = {}\n times_qubit_measured = defaultdict(lambda: 0)\n ro_size = None\n qubit_to_ram = {}\n\n for instr in program:\n if isinstance(instr, Pragma):\n new_prog += instr\n elif isinstance(instr, Declare):\n if instr.name == 'ro':\n if instr.memory_type != 'BIT':\n raise NotRunAndMeasureProgramError(\"The readout register `ro` \"\n \"must be of type BIT\")\n ro_size = instr.memory_size\n new_prog += instr\n elif isinstance(instr, Gate):\n for qubit in instr.qubits:\n last_qubit_operation[qubit.index] = 'gate'\n new_prog += instr\n elif isinstance(instr, Measurement):\n if instr.classical_reg is None:\n raise NotRunAndMeasureProgramError(\"No measure-for-effect allowed\")\n if instr.classical_reg.name != 'ro':\n raise NotRunAndMeasureProgramError(\"The readout register must be named `ro`, \"\n \"not {}\".format(instr.classical_reg.name))\n last_qubit_operation[instr.qubit.index] = 'measure'\n times_qubit_measured[instr.qubit.index] += 1\n qubit_to_ram[instr.qubit.index] = instr.classical_reg.offset\n else:\n raise NotRunAndMeasureProgramError(f\"Unsupported r_a_m instruction {instr}\")\n\n for q, lqo in last_qubit_operation.items():\n if lqo != 'measure':\n raise NotRunAndMeasureProgramError(f\"Qubit {q}'s last operation is a gate\")\n\n for q, tqm in times_qubit_measured.items():\n if tqm > 1:\n raise NotRunAndMeasureProgramError(f\"Qubit {q} is measured {tqm} times\")\n\n if ro_size is None:\n raise NotRunAndMeasureProgramError(\"Please declare a readout register\")\n\n return new_prog, qubit_to_ram, ro_size"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef do_program(self, program: Program) -> 'AbstractQuantumSimulator':\n for gate in program:\n if not isinstance(gate, Gate):\n raise ValueError(\"Can only compute the simulate a program composed of `Gate`s\")\n self.do_gate(gate)\n return self", "response": "Perform a sequence of gates contained within a program."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef transition(self):\n instruction = self.program[self.program_counter]\n\n if isinstance(instruction, Gate):\n if instruction.name in self.defined_gates:\n self.wf_simulator.do_gate_matrix(matrix=self.defined_gates[instruction.name],\n qubits=[q.index for q in instruction.qubits])\n else:\n self.wf_simulator.do_gate(gate=instruction)\n\n for noise_type, noise_prob in self.post_gate_noise_probabilities.items():\n self.wf_simulator.do_post_gate_noise(noise_type, noise_prob,\n qubits=[q.index for q in instruction.qubits])\n\n self.program_counter += 1\n\n elif isinstance(instruction, Measurement):\n measured_val = self.wf_simulator.do_measurement(qubit=instruction.qubit.index)\n x = instruction.classical_reg # type: MemoryReference\n self.ram[x.name][x.offset] = measured_val\n self.program_counter += 1\n\n elif isinstance(instruction, Declare):\n if instruction.shared_region is not None:\n raise NotImplementedError(\"SHARING is not (yet) implemented.\")\n\n self.ram[instruction.name] = np.zeros(instruction.memory_size,\n dtype=QUIL_TO_NUMPY_DTYPE[\n instruction.memory_type])\n self.program_counter += 1\n\n elif isinstance(instruction, Pragma):\n # TODO: more stringent checks for what's being pragma'd and warnings\n self.program_counter += 1\n\n elif isinstance(instruction, Jump):\n # unconditional Jump; go directly to Label\n self.program_counter = self.find_label(instruction.target)\n\n elif isinstance(instruction, JumpTarget):\n # Label; pass straight over\n self.program_counter += 1\n\n elif isinstance(instruction, JumpConditional):\n # JumpConditional; check classical reg\n x = instruction.condition # type: MemoryReference\n cond = self.ram[x.name][x.offset]\n if not isinstance(cond, (bool, np.bool, np.int8)):\n raise ValueError(\"{} requires a data type of BIT; not {}\"\n .format(instruction.op, type(cond)))\n dest_index = self.find_label(instruction.target)\n if isinstance(instruction, JumpWhen):\n jump_if_cond = True\n elif isinstance(instruction, JumpUnless):\n jump_if_cond = False\n else:\n raise TypeError(\"Invalid JumpConditional\")\n\n if not (cond ^ jump_if_cond):\n # jumping: set prog counter to JumpTarget\n self.program_counter = dest_index\n else:\n # not jumping: hop over this JumpConditional\n self.program_counter += 1\n\n elif isinstance(instruction, UnaryClassicalInstruction):\n # UnaryClassicalInstruction; set classical reg\n target = instruction.target # type:MemoryReference\n old = self.ram[target.name][target.offset]\n if isinstance(instruction, ClassicalNeg):\n if not isinstance(old, (int, float, np.int, np.float)):\n raise ValueError(\"NEG requires a data type of REAL or INTEGER; not {}\"\n .format(type(old)))\n self.ram[target.name][target.offset] *= -1\n elif isinstance(instruction, ClassicalNot):\n if not isinstance(old, (bool, np.bool)):\n raise ValueError(\"NOT requires a data type of BIT; not {}\"\n .format(type(old)))\n self.ram[target.name][target.offset] = not old\n else:\n raise TypeError(\"Invalid UnaryClassicalInstruction\")\n\n self.program_counter += 1\n\n elif isinstance(instruction, (LogicalBinaryOp, ArithmeticBinaryOp, ClassicalMove)):\n left_ind = instruction.left # type: MemoryReference\n left_val = self.ram[left_ind.name][left_ind.offset]\n if isinstance(instruction.right, MemoryReference):\n right_ind = instruction.right # type: MemoryReference\n right_val = self.ram[right_ind.name][right_ind.offset]\n else:\n right_val = instruction.right\n\n if isinstance(instruction, ClassicalAnd):\n new_val = left_val & right_val\n elif isinstance(instruction, ClassicalInclusiveOr):\n new_val = left_val | right_val\n elif isinstance(instruction, ClassicalExclusiveOr):\n new_val = left_val ^ right_val\n elif isinstance(instruction, ClassicalAdd):\n new_val = left_val + right_val\n elif isinstance(instruction, ClassicalSub):\n new_val = left_val - right_val\n elif isinstance(instruction, ClassicalMul):\n new_val = left_val * right_val\n elif isinstance(instruction, ClassicalDiv):\n new_val = left_val / right_val\n elif isinstance(instruction, ClassicalMove):\n new_val = right_val\n else:\n raise ValueError(\"Unknown BinaryOp {}\".format(type(instruction)))\n self.ram[left_ind.name][left_ind.offset] = new_val\n self.program_counter += 1\n\n elif isinstance(instruction, ClassicalExchange):\n left_ind = instruction.left # type: MemoryReference\n right_ind = instruction.right # type: MemoryReference\n\n tmp = self.ram[left_ind.name][left_ind.offset]\n self.ram[left_ind.name][left_ind.offset] = self.ram[right_ind.name][right_ind.offset]\n self.ram[right_ind.name][right_ind.offset] = tmp\n self.program_counter += 1\n\n elif isinstance(instruction, Reset):\n self.wf_simulator.reset()\n self.program_counter += 1\n\n elif isinstance(instruction, ResetQubit):\n # TODO\n raise NotImplementedError(\"Need to implement in wf simulator\")\n self.program_counter += 1\n\n elif isinstance(instruction, Wait):\n warnings.warn(\"WAIT does nothing for a noiseless simulator\")\n self.program_counter += 1\n\n elif isinstance(instruction, Nop):\n # well that was easy\n self.program_counter += 1\n\n elif isinstance(instruction, DefGate):\n if instruction.parameters is not None and len(instruction.parameters) > 0:\n raise NotImplementedError(\"PyQVM does not support parameterized DEFGATEs\")\n self.defined_gates[instruction.name] = instruction.name\n self.program_counter += 1\n\n elif isinstance(instruction, RawInstr):\n raise NotImplementedError(\"PyQVM does not support raw instructions. \"\n \"Parse your program\")\n\n elif isinstance(instruction, Halt):\n return True\n else:\n raise ValueError(\"Unsupported instruction type: {}\".format(instruction))\n\n # return HALTED (i.e. program_counter is end of program)\n return self.program_counter == len(self.program)", "response": "Implements a QAM - like transition."} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nexecutes a program on the QVM.", "response": "def execute(self, program: Program):\n \"\"\"\n Execute a program on the QVM.\n\n Note that the QAM is stateful. Subsequent calls to :py:func:`execute` will not\n automatically reset the wavefunction or the classical RAM. If this is desired,\n consider starting your program with ``RESET``.\n\n :return: ``self`` to support method chaining.\n \"\"\"\n # TODO: why are DEFGATEs not just included in the list of instructions?\n for dg in program.defined_gates:\n if dg.parameters is not None:\n raise NotImplementedError(\"PyQVM does not support parameterized DEFGATEs\")\n self.defined_gates[dg.name] = dg.matrix\n\n # initialize program counter\n self.program = program\n self.program_counter = 0\n\n halted = len(program) == 0\n while not halted:\n halted = self.transition()\n\n return self"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_benchmarker(endpoint: str = None, timeout: float = 10):\n if endpoint is None:\n config = PyquilConfig()\n endpoint = config.quilc_url\n\n return BenchmarkConnection(endpoint=endpoint, timeout=timeout)", "response": "Retrieve an instance of the appropriate AbstractBenchmarker subclass for a given endpoint."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef generate_rb_sequence(self, depth, gateset, seed=None, interleaver=None):\n\n # Support QubitPlaceholders: we temporarily index to arbitrary integers.\n # `generate_rb_sequence` handles mapping back to the original gateset gates.\n gateset_as_program = address_qubits(sum(gateset, Program()))\n qubits = len(gateset_as_program.get_qubits())\n gateset_for_api = gateset_as_program.out().splitlines()\n if interleaver:\n assert(isinstance(interleaver, Program))\n interleaver = interleaver.out()\n\n depth = int(depth) # needs to be jsonable, no np.int64 please!\n\n payload = RandomizedBenchmarkingRequest(depth=depth,\n qubits=qubits,\n gateset=gateset_for_api,\n seed=seed,\n interleaver=interleaver)\n response = self.client.call('generate_rb_sequence', payload) # type: RandomizedBenchmarkingResponse\n\n programs = []\n for clifford in response.sequence:\n clifford_program = Program()\n # Like below, we reversed the order because the API currently hands back the Clifford\n # decomposition right-to-left.\n for index in reversed(clifford):\n clifford_program.inst(gateset[index])\n programs.append(clifford_program)\n # The programs are returned in \"textbook style\" right-to-left order. To compose them into\n # the correct pyquil program, we reverse the order.\n return list(reversed(programs))", "response": "Generates a randomized benchmarking experiment on the given qubits and gateset."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef unpack_qubit(qubit):\n if isinstance(qubit, integer_types):\n return Qubit(qubit)\n elif isinstance(qubit, Qubit):\n return qubit\n elif isinstance(qubit, QubitPlaceholder):\n return qubit\n else:\n raise TypeError(\"qubit should be an int or Qubit instance\")", "response": "Unpack a qubit from an object."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef unpack_classical_reg(c):\n if isinstance(c, list) or isinstance(c, tuple):\n if len(c) > 2 or len(c) == 0:\n raise ValueError(\"if c is a list/tuple, it should be of length <= 2\")\n if len(c) == 1:\n c = (c[0], 0)\n if not isinstance(c[0], str):\n raise ValueError(\"if c is a list/tuple, its first member should be a string\")\n if not isinstance(c[1], int):\n raise ValueError(\"if c is a list/tuple, its second member should be an int\")\n return MemoryReference(c[0], c[1])\n if isinstance(c, MemoryReference):\n return c\n elif isinstance(c, str):\n return MemoryReference(c, 0)\n else:\n raise TypeError(\"c should be a list of length 2, a pair, a string, or a MemoryReference\")", "response": "Unpacks a classical register into a MemoryReference."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef format_parameter(element):\n if isinstance(element, integer_types) or isinstance(element, np.int_):\n return repr(element)\n elif isinstance(element, float):\n return _check_for_pi(element)\n elif isinstance(element, complex):\n out = ''\n r = element.real\n i = element.imag\n if i == 0:\n return repr(r)\n\n if r != 0:\n out += repr(r)\n\n if i == 1:\n assert np.isclose(r, 0, atol=1e-14)\n out = 'i'\n elif i == -1:\n assert np.isclose(r, 0, atol=1e-14)\n out = '-i'\n elif i < 0:\n out += repr(i) + 'i'\n elif r != 0:\n out += '+' + repr(i) + 'i'\n else:\n out += repr(i) + 'i'\n\n return out\n elif isinstance(element, MemoryReference):\n return str(element)\n elif isinstance(element, Expression):\n return _expression_to_string(element)\n elif isinstance(element, MemoryReference):\n return element.out()\n assert False, \"Invalid parameter: %r\" % element", "response": "Formats a particular parameter for Quil output."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef substitute_array(a, d):\n a = np.asarray(a, order=\"C\")\n return np.array([substitute(v, d) for v in a.flat]).reshape(a.shape)", "response": "Apply substitute to all elements of an array and return the resulting array."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _expression_to_string(expression):\n if isinstance(expression, BinaryExp):\n left = _expression_to_string(expression.op1)\n if isinstance(expression.op1, BinaryExp) and not (\n expression.op1.precedence > expression.precedence\n or expression.op1.precedence == expression.precedence\n and expression.associates in ('left', 'both')):\n left = '(' + left + ')'\n\n right = _expression_to_string(expression.op2)\n if isinstance(expression.op2, BinaryExp) and not (\n expression.precedence < expression.op2.precedence\n or expression.precedence == expression.op2.precedence\n and expression.associates in ('right', 'both')):\n right = '(' + right + ')'\n\n return left + expression.operator + right\n elif isinstance(expression, Function):\n return expression.name + '(' + _expression_to_string(expression.expression) + ')'\n elif isinstance(expression, Parameter):\n return str(expression)\n else:\n return format_parameter(expression)", "response": "Recursively converts an expression to a string taking into account precedence and associativity for placing\n parenthesis"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _contained_parameters(expression):\n if isinstance(expression, BinaryExp):\n return _contained_parameters(expression.op1) | _contained_parameters(expression.op2)\n elif isinstance(expression, Function):\n return _contained_parameters(expression.expression)\n elif isinstance(expression, Parameter):\n return {expression}\n else:\n return set()", "response": "Determines which parameters are contained in this expression."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _check_for_pi(element):\n frac = Fraction(element / np.pi).limit_denominator(8)\n num, den = frac.numerator, frac.denominator\n sign = \"-\" if num < 0 else \"\"\n if num / float(den) == element / np.pi:\n if num == 0:\n return \"0\"\n elif abs(num) == 1 and den == 1:\n return sign + \"pi\"\n elif abs(num) == 1:\n return sign + \"pi/\" + repr(den)\n elif den == 1:\n return repr(num) + \"*pi\"\n else:\n return repr(num) + \"*pi/\" + repr(den)\n else:\n return repr(element)", "response": "Checks to see if there exists a rational number in reduced form for which the difference between element and np. pi is small and 8 is not large."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef add_noise_to_program(prog, T1=30e-6, T2=30e-6, gate_time_1q=50e-9, gate_time_2q=150e-09,\n ro_fidelity=0.95):\n \"\"\"\n Add generic damping and dephasing noise to a program.\n\n .. warning::\n\n This function is deprecated. Please use :py:func:`add_decoherence_noise` instead.\n\n :param prog: A pyquil program consisting of I, RZ, CZ, and RX(+-pi/2) instructions\n :param Union[Dict[int,float],float] T1: The T1 amplitude damping time either globally or in a\n dictionary indexed by qubit id. By default, this is 30 us.\n :param Union[Dict[int,float],float] T2: The T2 dephasing time either globally or in a\n dictionary indexed by qubit id. By default, this is also 30 us.\n :param float gate_time_1q: The duration of the one-qubit gates, namely RX(+pi/2) and RX(-pi/2).\n By default, this is 50 ns.\n :param float gate_time_2q: The duration of the two-qubit gates, namely CZ.\n By default, this is 150 ns.\n :param Union[Dict[int,float],float] ro_fidelity: The readout assignment fidelity\n :math:`F = (p(0|0) + p(1|1))/2` either globally or in a dictionary indexed by qubit id.\n :return: A new program with noisy operators.\n \"\"\"\n warnings.warn(\"pyquil.kraus.add_noise_to_program is deprecated, please use \"\n \"pyquil.noise.add_decoherence_noise instead.\",\n DeprecationWarning)\n return add_decoherence_noise(prog, T1=T1, T2=T2, gate_time_1q=gate_time_1q,\n gate_time_2q=gate_time_2q, ro_fidelity=ro_fidelity)", "response": "Add generic damping and dephasing noise to a program."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ntake a matrix that represents the k - qubit lifts from the right to left of the Hilbert space and lifts the k - qubit gate on adjacent qubits.", "response": "def qubit_adjacent_lifted_gate(i, matrix, n_qubits):\n \"\"\"\n Lifts input k-qubit gate on adjacent qubits starting from qubit i\n to complete Hilbert space of dimension 2 ** num_qubits.\n\n Ex: 1-qubit gate, lifts from qubit i\n Ex: 2-qubit gate, lifts from qubits (i+1, i)\n Ex: 3-qubit gate, lifts from qubits (i+2, i+1, i), operating in that order\n\n In general, this takes a k-qubit gate (2D matrix 2^k x 2^k) and lifts\n it to the complete Hilbert space of dim 2^num_qubits, as defined by\n the right-to-left tensor product (1) in arXiv:1608.03355.\n\n Developer note: Quil and the QVM like qubits to be ordered such that qubit 0 is on the right.\n Therefore, in ``qubit_adjacent_lifted_gate``, ``lifted_pauli``, and ``lifted_state_operator``,\n we build up the lifted matrix by performing the kronecker product from right to left.\n\n Note that while the qubits are addressed in decreasing order,\n starting with num_qubit - 1 on the left and ending with qubit 0 on the\n right (in a little-endian fashion), gates are still lifted to apply\n on qubits in increasing index (right-to-left) order.\n\n :param int i: starting qubit to lift matrix from (incr. index order)\n :param np.array matrix: the matrix to be lifted\n :param int n_qubits: number of overall qubits present in space\n\n :return: matrix representation of operator acting on the\n complete Hilbert space of all num_qubits.\n :rtype: sparse_array\n \"\"\"\n n_rows, n_cols = matrix.shape\n assert n_rows == n_cols, 'Matrix must be square'\n gate_size = np.log2(n_rows)\n assert gate_size == int(gate_size), 'Matrix must be 2^n by 2^n'\n gate_size = int(gate_size)\n\n # Outer-product to lift gate to complete Hilbert space\n\n # bottom: i qubits below target\n bottom_matrix = np.eye(2 ** i, dtype=np.complex128)\n # top: Nq - i (bottom) - gate_size (gate) qubits above target\n top_qubits = n_qubits - i - gate_size\n top_matrix = np.eye(2 ** top_qubits, dtype=np.complex128)\n\n return np.kron(top_matrix, np.kron(matrix, bottom_matrix))"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ngenerates the permutation matrix that permutes two single-particle Hilbert spaces into adjacent positions. ALWAYS swaps j TO k. Recall that Hilbert spaces are ordered in decreasing qubit index order. Hence, j > k implies that j is to the left of k. End results: j == k: nothing happens j > k: Swap j right to k, until j at ind (k) and k at ind (k+1). j < k: Swap j left to k, until j at ind (k) and k at ind (k-1). Done in preparation for arbitrary 2-qubit gate application on ADJACENT qubits. :param int j: starting qubit index :param int k: ending qubit index :param int num_qubits: number of qubits in Hilbert space :param np.array qubit_map: current index mapping of qubits :return: tuple of swap matrix for the specified permutation, and the new qubit_map, after permutation is made :rtype: tuple (np.array, np.array)", "response": "def two_swap_helper(j, k, num_qubits, qubit_map):\n \"\"\"\n Generate the permutation matrix that permutes two single-particle Hilbert\n spaces into adjacent positions.\n\n ALWAYS swaps j TO k. Recall that Hilbert spaces are ordered in decreasing\n qubit index order. Hence, j > k implies that j is to the left of k.\n\n End results:\n j == k: nothing happens\n j > k: Swap j right to k, until j at ind (k) and k at ind (k+1).\n j < k: Swap j left to k, until j at ind (k) and k at ind (k-1).\n\n Done in preparation for arbitrary 2-qubit gate application on ADJACENT\n qubits.\n\n :param int j: starting qubit index\n :param int k: ending qubit index\n :param int num_qubits: number of qubits in Hilbert space\n :param np.array qubit_map: current index mapping of qubits\n\n :return: tuple of swap matrix for the specified permutation,\n and the new qubit_map, after permutation is made\n :rtype: tuple (np.array, np.array)\n \"\"\"\n if not (0 <= j < num_qubits and 0 <= k < num_qubits):\n raise ValueError(\"Permutation SWAP index not valid\")\n\n perm = np.eye(2 ** num_qubits, dtype=np.complex128)\n new_qubit_map = np.copy(qubit_map)\n\n if j == k:\n # nothing happens\n return perm, new_qubit_map\n elif j > k:\n # swap j right to k, until j at ind (k) and k at ind (k+1)\n for i in range(j, k, -1):\n perm = qubit_adjacent_lifted_gate(i - 1, SWAP, num_qubits).dot(perm)\n new_qubit_map[i - 1], new_qubit_map[i] = new_qubit_map[i], new_qubit_map[i - 1]\n elif j < k:\n # swap j left to k, until j at ind (k) and k at ind (k-1)\n for i in range(j, k, 1):\n perm = qubit_adjacent_lifted_gate(i, SWAP, num_qubits).dot(perm)\n new_qubit_map[i], new_qubit_map[i + 1] = new_qubit_map[i + 1], new_qubit_map[i]\n\n return perm, new_qubit_map"} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ngenerate an arbitrary number of Hilbert spaces with a permutation matrix that permutes an arbitrary number of qubits into adjacent positions.", "response": "def permutation_arbitrary(qubit_inds, n_qubits):\n \"\"\"\n Generate the permutation matrix that permutes an arbitrary number of\n single-particle Hilbert spaces into adjacent positions.\n\n Transposes the qubit indices in the order they are passed to a\n contiguous region in the complete Hilbert space, in increasing\n qubit index order (preserving the order they are passed in).\n\n Gates are usually defined as `GATE 0 1 2`, with such an argument ordering\n dictating the layout of the matrix corresponding to GATE. If such an\n instruction is given, actual qubits (0, 1, 2) need to be swapped into the\n positions (2, 1, 0), because the lifting operation taking the 8 x 8 matrix\n of GATE is done in the little-endian (reverse) addressed qubit space.\n\n For example, suppose I have a Quil command CCNOT 20 15 10.\n The median of the qubit indices is 15 - hence, we permute qubits\n [20, 15, 10] into the final map [16, 15, 14] to minimize the number of\n swaps needed, and so we can directly operate with the final CCNOT, when\n lifted from indices [16, 15, 14] to the complete Hilbert space.\n\n Notes: assumes qubit indices are unique (assured in parent call).\n\n See documentation for further details and explanation.\n\n Done in preparation for arbitrary gate application on\n adjacent qubits.\n\n :param qubit_inds: (int) Qubit indices in the order the gate is\n applied to.\n :param int n_qubits: Number of qubits in system\n\n :return:\n perm - permutation matrix providing the desired qubit reordering\n qubit_arr - new indexing of qubits presented in left to right\n decreasing index order. Should be identical to passed 'args'.\n start_i - starting index to lift gate from\n :rtype: tuple (sparse_array, np.array, int)\n \"\"\"\n # Begin construction of permutation\n perm = np.eye(2 ** n_qubits, dtype=np.complex128)\n\n # First, sort the list and find the median.\n sorted_inds = np.sort(qubit_inds)\n med_i = len(qubit_inds) // 2\n med = sorted_inds[med_i]\n\n # The starting position of all specified Hilbert spaces begins at\n # the qubit at (median - med_i)\n start = med - med_i\n # Array of final indices the arguments are mapped to, from\n # high index to low index, left to right ordering\n final_map = np.arange(start, start + len(qubit_inds))[::-1]\n start_i = final_map[-1]\n\n # Note that the lifting operation takes a k-qubit gate operating\n # on the qubits i+k-1, i+k-2, ... i (left to right).\n # two_swap_helper can be used to build the\n # permutation matrix by filling out the final map by sweeping over\n # the qubit_inds from left to right and back again, swapping qubits into\n # position. we loop over the qubit_inds until the final mapping matches\n # the argument.\n qubit_arr = np.arange(n_qubits) # current qubit indexing\n\n made_it = False\n right = True\n while not made_it:\n array = range(len(qubit_inds)) if right else range(len(qubit_inds))[::-1]\n for i in array:\n pmod, qubit_arr = two_swap_helper(np.where(qubit_arr == qubit_inds[i])[0][0],\n final_map[i], n_qubits,\n qubit_arr)\n\n # update permutation matrix\n perm = pmod.dot(perm)\n if np.allclose(qubit_arr[final_map[-1]:final_map[0] + 1][::-1], qubit_inds):\n made_it = True\n break\n\n # for next iteration, go in opposite direction\n right = not right\n\n assert np.allclose(qubit_arr[final_map[-1]:final_map[0] + 1][::-1], qubit_inds)\n return perm, qubit_arr[::-1], start_i"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef lifted_gate_matrix(matrix: np.ndarray, qubit_inds: List[int], n_qubits: int):\n n_rows, n_cols = matrix.shape\n assert n_rows == n_cols, 'Matrix must be square'\n gate_size = np.log2(n_rows)\n assert gate_size == int(gate_size), 'Matrix must be 2^n by 2^n'\n gate_size = int(gate_size)\n\n pi_permutation_matrix, final_map, start_i = permutation_arbitrary(qubit_inds, n_qubits)\n if start_i > 0:\n check = final_map[-gate_size - start_i:-start_i]\n else:\n # Python can't deal with `arr[:-0]`\n check = final_map[-gate_size - start_i:]\n np.testing.assert_allclose(check, qubit_inds)\n\n v_matrix = qubit_adjacent_lifted_gate(start_i, matrix, n_qubits)\n return np.dot(np.conj(pi_permutation_matrix.T),\n np.dot(v_matrix, pi_permutation_matrix))", "response": "Lift a unitary matrix to act on the specified qubits in a full n_qubits - qubit Hilbert space."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef lifted_gate(gate: Gate, n_qubits: int):\n if len(gate.params) > 0:\n matrix = QUANTUM_GATES[gate.name](*gate.params)\n else:\n matrix = QUANTUM_GATES[gate.name]\n\n return lifted_gate_matrix(matrix=matrix,\n qubit_inds=[q.index for q in gate.qubits],\n n_qubits=n_qubits)", "response": "Takes a pyquil gate and returns a lifted version of it."} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns the unitary of a pyQuil program.", "response": "def program_unitary(program, n_qubits):\n \"\"\"\n Return the unitary of a pyQuil program.\n\n :param program: A program consisting only of :py:class:`Gate`.:\n :return: a unitary corresponding to the composition of the program's gates.\n \"\"\"\n umat = np.eye(2 ** n_qubits)\n for instruction in program:\n if isinstance(instruction, Gate):\n unitary = lifted_gate(gate=instruction, n_qubits=n_qubits)\n umat = unitary.dot(umat)\n else:\n raise ValueError(\"Can only compute program unitary for programs composed of `Gate`s\")\n return umat"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef lifted_pauli(pauli_sum: Union[PauliSum, PauliTerm], qubits: List[int]):\n if isinstance(pauli_sum, PauliTerm):\n pauli_sum = PauliSum([pauli_sum])\n\n n_qubits = len(qubits)\n result_hilbert = np.zeros((2 ** n_qubits, 2 ** n_qubits), dtype=np.complex128)\n # left kronecker product corresponds to the correct basis ordering\n for term in pauli_sum.terms:\n term_hilbert = np.array([1])\n for qubit in qubits:\n term_hilbert = np.kron(QUANTUM_GATES[term[qubit]], term_hilbert)\n\n result_hilbert += term_hilbert * term.coefficient\n\n return result_hilbert", "response": "Takes a PauliSum object along with a list of qubits and returns a matrix representation of the pauli sum operator that can be used to lift the pauli sum into a single element of the base class."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef tensor_up(pauli_sum: Union[PauliSum, PauliTerm], qubits: List[int]):\n return lifted_pauli(pauli_sum=pauli_sum, qubits=qubits)", "response": "Returns a matrix representation of a PauliSum object along with a list of qubits."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef lifted_state_operator(state: TensorProductState, qubits: List[int]):\n mat = 1.0\n for qubit in qubits:\n oneq_state = state[qubit]\n assert oneq_state.qubit == qubit\n state_vector = STATES[oneq_state.label][oneq_state.index][:, np.newaxis]\n state_matrix = state_vector @ state_vector.conj().T\n mat = np.kron(state_matrix, mat)\n return mat", "response": "Take a TensorProductState along with a list of qubits and return a matrix\n corresponding to the tensored - up representation of the states density operator form."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _abbrev_program(program: Program, max_len=10):\n program_lines = program.out().splitlines()\n if max_len is not None and len(program_lines) > max_len:\n first_n = max_len // 2\n last_n = max_len - first_n\n excluded = len(program_lines) - max_len\n program_lines = (program_lines[:first_n] + [f'... {excluded} instrs not shown ...']\n + program_lines[-last_n:])\n\n return '; '.join(program_lines)", "response": "Create an abbreviated string representation of a Program."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nprepares the index - th SIC basis state.", "response": "def _one_q_sic_prep(index, qubit):\n \"\"\"Prepare the index-th SIC basis state.\"\"\"\n if index == 0:\n return Program()\n\n theta = 2 * np.arccos(1 / np.sqrt(3))\n zx_plane_rotation = Program([\n RX(-pi / 2, qubit),\n RZ(theta - pi, qubit),\n RX(-pi / 2, qubit),\n ])\n\n if index == 1:\n return zx_plane_rotation\n\n elif index == 2:\n return zx_plane_rotation + RZ(-2 * pi / 3, qubit)\n\n elif index == 3:\n return zx_plane_rotation + RZ(2 * pi / 3, qubit)\n\n raise ValueError(f'Bad SIC index: {index}')"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nprepare the index - th eigenstate of the pauli operator given by label.", "response": "def _one_q_pauli_prep(label, index, qubit):\n \"\"\"Prepare the index-th eigenstate of the pauli operator given by label.\"\"\"\n if index not in [0, 1]:\n raise ValueError(f'Bad Pauli index: {index}')\n\n if label == 'X':\n if index == 0:\n return Program(RY(pi / 2, qubit))\n else:\n return Program(RY(-pi / 2, qubit))\n\n elif label == 'Y':\n if index == 0:\n return Program(RX(-pi / 2, qubit))\n else:\n return Program(RX(pi / 2, qubit))\n\n elif label == 'Z':\n if index == 0:\n return Program()\n else:\n return Program(RX(pi, qubit))\n\n raise ValueError(f'Bad Pauli label: {label}')"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _one_q_state_prep(oneq_state: _OneQState):\n label = oneq_state.label\n if label == 'SIC':\n return _one_q_sic_prep(oneq_state.index, oneq_state.qubit)\n elif label in ['X', 'Y', 'Z']:\n return _one_q_pauli_prep(label, oneq_state.index, oneq_state.qubit)\n else:\n raise ValueError(f\"Bad state label: {label}\")", "response": "Prepare a one qubit state."} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ngenerate a sequence of gate sequences to measure in the local eigenbasis of a Pauli operator.", "response": "def _local_pauli_eig_meas(op, idx):\n \"\"\"\n Generate gate sequence to measure in the eigenbasis of a Pauli operator, assuming\n we are only able to measure in the Z eigenbasis. (Note: The unitary operations of this\n Program are essentially the Hermitian conjugates of those in :py:func:`_one_q_pauli_prep`)\n\n \"\"\"\n if op == 'X':\n return Program(RY(-pi / 2, idx))\n elif op == 'Y':\n return Program(RX(pi / 2, idx))\n elif op == 'Z':\n return Program()\n raise ValueError(f'Unknown operation {op}')"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nconstructing a graph where an edge signifies two experiments are diagonal in a TPB.", "response": "def construct_tpb_graph(experiments: TomographyExperiment):\n \"\"\"\n Construct a graph where an edge signifies two experiments are diagonal in a TPB.\n \"\"\"\n g = nx.Graph()\n for expt in experiments:\n assert len(expt) == 1, 'already grouped?'\n expt = expt[0]\n\n if expt not in g:\n g.add_node(expt, count=1)\n else:\n g.nodes[expt]['count'] += 1\n\n for expt1, expt2 in itertools.combinations(experiments, r=2):\n expt1 = expt1[0]\n expt2 = expt2[0]\n\n if expt1 == expt2:\n continue\n\n max_weight_in = _max_weight_state([expt1.in_state, expt2.in_state])\n max_weight_out = _max_weight_operator([expt1.out_operator, expt2.out_operator])\n if max_weight_in is not None and max_weight_out is not None:\n g.add_edge(expt1, expt2)\n\n return g"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ngroup experiments that are diagonal in a shared tensor product basis with a graph clique removal algorithm.", "response": "def group_experiments_clique_removal(experiments: TomographyExperiment) -> TomographyExperiment:\n \"\"\"\n Group experiments that are diagonal in a shared tensor product basis (TPB) to minimize number\n of QPU runs, using a graph clique removal algorithm.\n\n :param experiments: a tomography experiment\n :return: a tomography experiment with all the same settings, just grouped according to shared\n TPBs.\n \"\"\"\n g = construct_tpb_graph(experiments)\n _, cliqs = clique_removal(g)\n new_cliqs = []\n for cliq in cliqs:\n new_cliq = []\n for expt in cliq:\n # duplicate `count` times\n new_cliq += [expt] * g.nodes[expt]['count']\n\n new_cliqs += [new_cliq]\n\n return TomographyExperiment(new_cliqs, program=experiments.program)"} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nconstructs a PauliTerm operator by taking the non - identity single - qubit operator at each qubit position.", "response": "def _max_weight_operator(ops: Iterable[PauliTerm]) -> Union[None, PauliTerm]:\n \"\"\"Construct a PauliTerm operator by taking the non-identity single-qubit operator at each\n qubit position.\n\n This function will return ``None`` if the input operators do not share a natural tensor\n product basis.\n\n For example, the max_weight_operator of [\"XI\", \"IZ\"] is \"XZ\". Asking for the max weight\n operator of something like [\"XI\", \"ZI\"] will return None.\n \"\"\"\n mapping = dict() # type: Dict[int, str]\n for op in ops:\n for idx, op_str in op:\n if idx in mapping:\n if mapping[idx] != op_str:\n return None\n else:\n mapping[idx] = op_str\n op = functools.reduce(mul, (PauliTerm(op, q) for q, op in mapping.items()), sI())\n return op"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nconstruct a TensorProductState by taking the max - weight state at each qubit position.", "response": "def _max_weight_state(states: Iterable[TensorProductState]) -> Union[None, TensorProductState]:\n \"\"\"Construct a TensorProductState by taking the single-qubit state at each\n qubit position.\n\n This function will return ``None`` if the input states are not compatible\n\n For example, the max_weight_state of [\"(+X, q0)\", \"(-Z, q1)\"] is \"(+X, q0; -Z q1)\". Asking for\n the max weight state of something like [\"(+X, q0)\", \"(+Z, q0)\"] will return None.\n \"\"\"\n mapping = dict() # type: Dict[int, _OneQState]\n for state in states:\n for oneq_state in state.states:\n if oneq_state.qubit in mapping:\n if mapping[oneq_state.qubit] != oneq_state:\n return None\n else:\n mapping[oneq_state.qubit] = oneq_state\n return TensorProductState(list(mapping.values()))"} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ngives an input TomographyExperiment provide a dictionary indicating which ExperimentSettings share a tensor product basis and that ExperimentSettings share a tensor product basis.", "response": "def _max_tpb_overlap(tomo_expt: TomographyExperiment):\n \"\"\"\n Given an input TomographyExperiment, provide a dictionary indicating which ExperimentSettings\n share a tensor product basis\n\n :param tomo_expt: TomographyExperiment, from which to group ExperimentSettings that share a tpb\n and can be run together\n :return: dictionary keyed with ExperimentSetting (specifying a tpb), and with each value being a\n list of ExperimentSettings (diagonal in that tpb)\n \"\"\"\n # initialize empty dictionary\n diagonal_sets = {}\n # loop through ExperimentSettings of the TomographyExperiment\n for expt_setting in tomo_expt:\n # no need to group already grouped TomographyExperiment\n assert len(expt_setting) == 1, 'already grouped?'\n expt_setting = expt_setting[0]\n # calculate max overlap of expt_setting with keys of diagonal_sets\n # keep track of whether a shared tpb was found\n found_tpb = False\n # loop through dict items\n for es, es_list in diagonal_sets.items():\n trial_es_list = es_list + [expt_setting]\n diag_in_term = _max_weight_state(expst.in_state for expst in trial_es_list)\n diag_out_term = _max_weight_operator(expst.out_operator for expst in trial_es_list)\n # max_weight_xxx returns None if the set of xxx's don't share a TPB, so the following\n # conditional is True if expt_setting can be inserted into the current es_list.\n if diag_in_term is not None and diag_out_term is not None:\n found_tpb = True\n assert len(diag_in_term) >= len(es.in_state), \\\n \"Highest weight in-state can't be smaller than the given in-state\"\n assert len(diag_out_term) >= len(es.out_operator), \\\n \"Highest weight out-PauliTerm can't be smaller than the given out-PauliTerm\"\n\n # update the diagonalizing basis (key of dict) if necessary\n if len(diag_in_term) > len(es.in_state) or len(diag_out_term) > len(es.out_operator):\n del diagonal_sets[es]\n new_es = ExperimentSetting(diag_in_term, diag_out_term)\n diagonal_sets[new_es] = trial_es_list\n else:\n diagonal_sets[es] = trial_es_list\n break\n\n if not found_tpb:\n # made it through entire dict without finding any ExperimentSetting with shared tpb,\n # so need to make a new item\n diagonal_sets[expt_setting] = [expt_setting]\n\n return diagonal_sets"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ngrouping experiments that are diagonal in a shared tensor product basis.", "response": "def group_experiments(experiments: TomographyExperiment,\n method: str = 'greedy') -> TomographyExperiment:\n \"\"\"\n Group experiments that are diagonal in a shared tensor product basis (TPB) to minimize number\n of QPU runs.\n\n Background\n ----------\n\n Given some PauliTerm operator, the 'natural' tensor product basis to\n diagonalize this term is the one which diagonalizes each Pauli operator in the\n product term-by-term.\n\n For example, X(1) * Z(0) would be diagonal in the 'natural' tensor product basis\n {(|0> +/- |1>)/Sqrt[2]} * {|0>, |1>}, whereas Z(1) * X(0) would be diagonal\n in the 'natural' tpb {|0>, |1>} * {(|0> +/- |1>)/Sqrt[2]}. The two operators\n commute but are not diagonal in each others 'natural' tpb (in fact, they are\n anti-diagonal in each others 'natural' tpb). This function tests whether two\n operators given as PauliTerms are both diagonal in each others 'natural' tpb.\n\n Note that for the given example of X(1) * Z(0) and Z(1) * X(0), we can construct\n the following basis which simultaneously diagonalizes both operators:\n\n -- |0>' = |0> (|+>) + |1> (|->)\n -- |1>' = |0> (|+>) - |1> (|->)\n -- |2>' = |0> (|->) + |1> (|+>)\n -- |3>' = |0> (-|->) + |1> (|+>)\n\n In this basis, X Z looks like diag(1, -1, 1, -1), and Z X looks like diag(1, 1, -1, -1).\n Notice however that this basis cannot be constructed with single-qubit operations, as each\n of the basis vectors are entangled states.\n\n\n Methods\n -------\n\n The \"greedy\" method will keep a running set of 'buckets' into which grouped ExperimentSettings\n will be placed. Each new ExperimentSetting considered is assigned to the first applicable\n bucket and a new bucket is created if there are no applicable buckets.\n\n The \"clique-removal\" method maps the term grouping problem onto Max Clique graph problem.\n This method constructs a NetworkX graph where an edge exists between two settings that\n share an nTPB and then uses networkx's algorithm for clique removal. This method can give\n you marginally better groupings in certain circumstances, but constructing the\n graph is pretty slow so \"greedy\" is the default.\n\n :param experiments: a tomography experiment\n :param method: method used for grouping; the allowed methods are one of\n ['greedy', 'clique-removal']\n :return: a tomography experiment with all the same settings, just grouped according to shared\n TPBs.\n \"\"\"\n allowed_methods = ['greedy', 'clique-removal']\n assert method in allowed_methods, f\"'method' should be one of {allowed_methods}.\"\n if method == 'greedy':\n return group_experiments_greedy(experiments)\n elif method == 'clique-removal':\n return group_experiments_clique_removal(experiments)"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nmeasure all the observables in a TomographicExperiment.", "response": "def measure_observables(qc: QuantumComputer, tomo_experiment: TomographyExperiment,\n n_shots: int = 10000, progress_callback=None, active_reset=False,\n symmetrize_readout: str = 'exhaustive',\n calibrate_readout: str = 'plus-eig'):\n \"\"\"\n Measure all the observables in a TomographyExperiment.\n\n :param qc: A QuantumComputer which can run quantum programs\n :param tomo_experiment: A suite of tomographic observables to measure\n :param n_shots: The number of shots to take per ExperimentSetting\n :param progress_callback: If not None, this function is called each time a group of\n settings is run with arguments ``f(i, len(tomo_experiment)`` such that the progress\n is ``i / len(tomo_experiment)``.\n :param active_reset: Whether to actively reset qubits instead of waiting several\n times the coherence length for qubits to decay to |0> naturally. Setting this\n to True is much faster but there is a ~1% error per qubit in the reset operation.\n Thermal noise from \"traditional\" reset is not routinely characterized but is of the same\n order.\n :param symmetrize_readout: Method used to symmetrize the readout errors, i.e. set\n p(0|1) = p(1|0). For uncorrelated readout errors, this can be achieved by randomly\n selecting between the POVMs {X.D1.X, X.D0.X} and {D0, D1} (where both D0 and D1 are\n diagonal). However, here we currently support exhaustive symmetrization and loop through\n all possible 2^n POVMs {X/I . POVM . X/I}^n, and obtain symmetrization more generally,\n i.e. set p(00|00) = p(01|01) = .. = p(11|11), as well as p(00|01) = p(01|00) etc. If this\n is None, no symmetrization is performed. The exhaustive method can be specified by setting\n this variable to 'exhaustive' (default value). Set to `None` if no symmetrization is\n desired.\n :param calibrate_readout: Method used to calibrate the readout results. Currently, the only\n method supported is normalizing against the operator's expectation value in its +1\n eigenstate, which can be specified by setting this variable to 'plus-eig' (default value).\n The preceding symmetrization and this step together yield a more accurate estimation of the observable. Set to `None` if no calibration is desired.\n \"\"\"\n # calibration readout only works with symmetrization turned on\n if calibrate_readout is not None and symmetrize_readout is None:\n raise ValueError(\"Readout calibration only works with readout symmetrization turned on\")\n\n # Outer loop over a collection of grouped settings for which we can simultaneously\n # estimate.\n for i, settings in enumerate(tomo_experiment):\n\n log.info(f\"Collecting bitstrings for the {len(settings)} settings: {settings}\")\n\n # 1.1 Prepare a state according to the amalgam of all setting.in_state\n total_prog = Program()\n if active_reset:\n total_prog += RESET()\n max_weight_in_state = _max_weight_state(setting.in_state for setting in settings)\n for oneq_state in max_weight_in_state.states:\n total_prog += _one_q_state_prep(oneq_state)\n\n # 1.2 Add in the program\n total_prog += tomo_experiment.program\n\n # 1.3 Measure the state according to setting.out_operator\n max_weight_out_op = _max_weight_operator(setting.out_operator for setting in settings)\n for qubit, op_str in max_weight_out_op:\n total_prog += _local_pauli_eig_meas(op_str, qubit)\n\n # 2. Symmetrization\n qubits = max_weight_out_op.get_qubits()\n\n if symmetrize_readout == 'exhaustive' and len(qubits) > 0:\n bitstrings, d_qub_idx = _exhaustive_symmetrization(qc, qubits, n_shots, total_prog)\n\n elif symmetrize_readout is None and len(qubits) > 0:\n total_prog_no_symm = total_prog.copy()\n ro = total_prog_no_symm.declare('ro', 'BIT', len(qubits))\n d_qub_idx = {}\n for i, q in enumerate(qubits):\n total_prog_no_symm += MEASURE(q, ro[i])\n # Keep track of qubit-classical register mapping via dict\n d_qub_idx[q] = i\n total_prog_no_symm.wrap_in_numshots_loop(n_shots)\n total_prog_no_symm_native = qc.compiler.quil_to_native_quil(total_prog_no_symm)\n total_prog_no_symm_bin = qc.compiler.native_quil_to_executable(total_prog_no_symm_native)\n bitstrings = qc.run(total_prog_no_symm_bin)\n\n elif len(qubits) == 0:\n # looks like an identity operation\n pass\n\n else:\n raise ValueError(\"Readout symmetrization method must be either 'exhaustive' or None\")\n\n if progress_callback is not None:\n progress_callback(i, len(tomo_experiment))\n\n # 3. Post-process\n # Inner loop over the grouped settings. They only differ in which qubits' measurements\n # we include in the post-processing. For example, if `settings` is Z1, Z2, Z1Z2 and we\n # measure (n_shots, n_qubits=2) obs_strings then the full operator value involves selecting\n # either the first column, second column, or both and multiplying along the row.\n for setting in settings:\n # 3.1 Get the term's coefficient so we can multiply it in later.\n coeff = complex(setting.out_operator.coefficient)\n if not np.isclose(coeff.imag, 0):\n raise ValueError(f\"{setting}'s out_operator has a complex coefficient.\")\n coeff = coeff.real\n\n # 3.2 Special case for measuring the \"identity\" operator, which doesn't make much\n # sense but should happen perfectly.\n if is_identity(setting.out_operator):\n yield ExperimentResult(\n setting=setting,\n expectation=coeff,\n std_err=0.0,\n total_counts=n_shots,\n )\n continue\n\n # 3.3 Obtain statistics from result of experiment\n obs_mean, obs_var = _stats_from_measurements(bitstrings, d_qub_idx, setting, n_shots, coeff)\n\n if calibrate_readout == 'plus-eig':\n # 4 Readout calibration\n # 4.1 Obtain calibration program\n calibr_prog = _calibration_program(qc, tomo_experiment, setting)\n # 4.2 Perform symmetrization on the calibration program\n if symmetrize_readout == 'exhaustive':\n qubs_calibr = setting.out_operator.get_qubits()\n calibr_shots = n_shots\n calibr_results, d_calibr_qub_idx = _exhaustive_symmetrization(qc, qubs_calibr, calibr_shots, calibr_prog)\n\n else:\n raise ValueError(\"Readout symmetrization method must be either 'exhaustive' or None\")\n\n # 4.3 Obtain statistics from the measurement process\n obs_calibr_mean, obs_calibr_var = _stats_from_measurements(calibr_results, d_calibr_qub_idx, setting, calibr_shots)\n # 4.3 Calibrate the readout results\n corrected_mean = obs_mean / obs_calibr_mean\n corrected_var = ratio_variance(obs_mean, obs_var, obs_calibr_mean, obs_calibr_var)\n\n yield ExperimentResult(\n setting=setting,\n expectation=corrected_mean.item(),\n std_err=np.sqrt(corrected_var).item(),\n total_counts=n_shots,\n raw_expectation=obs_mean.item(),\n raw_std_err=np.sqrt(obs_var).item(),\n calibration_expectation=obs_calibr_mean.item(),\n calibration_std_err=np.sqrt(obs_calibr_var).item(),\n calibration_counts=calibr_shots,\n )\n\n elif calibrate_readout is None:\n # No calibration\n yield ExperimentResult(\n setting=setting,\n expectation=obs_mean.item(),\n std_err=np.sqrt(obs_var).item(),\n total_counts=n_shots,\n )\n\n else:\n raise ValueError(\"Calibration readout method must be either 'plus-eig' or None\")"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nconvert a tuple of booleans representing the operation to a Program.", "response": "def _ops_bool_to_prog(ops_bool: Tuple[bool], qubits: List[int]) -> Program:\n \"\"\"\n :param ops_bool: tuple of booleans specifying the operation to be carried out on `qubits`\n :param qubits: list specifying the qubits to be carried operations on\n :return: Program with the operations specified in `ops_bool` on the qubits specified in\n `qubits`\n \"\"\"\n assert len(ops_bool) == len(qubits), \"Mismatch of qubits and operations\"\n prog = Program()\n for i, op_bool in enumerate(ops_bool):\n if op_bool == 0:\n continue\n elif op_bool == 1:\n prog += Program(X(qubits[i]))\n else:\n raise ValueError(\"ops_bool should only consist of 0s and/or 1s\")\n return prog"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _stats_from_measurements(bs_results: np.ndarray, qubit_index_map: Dict,\n setting: ExperimentSetting, n_shots: int,\n coeff: float = 1.0) -> Tuple[float]:\n \"\"\"\n :param bs_results: results from running `qc.run`\n :param qubit_index_map: dict mapping qubit to classical register index\n :param setting: ExperimentSetting\n :param n_shots: number of shots in the measurement process\n :param coeff: coefficient of the operator being estimated\n :return: tuple specifying (mean, variance)\n \"\"\"\n # Identify classical register indices to select\n idxs = [qubit_index_map[q] for q, _ in setting.out_operator]\n # Pick columns corresponding to qubits with a non-identity out_operation\n obs_strings = bs_results[:, idxs]\n # Transform bits to eigenvalues; ie (+1, -1)\n my_obs_strings = 1 - 2 * obs_strings\n # Multiply row-wise to get operator values. Do statistics. Return result.\n obs_vals = coeff * np.prod(my_obs_strings, axis=1)\n obs_mean = np.mean(obs_vals)\n obs_var = np.var(obs_vals) / n_shots\n\n return obs_mean, obs_var", "response": "Compute statistics from measurements."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef ratio_variance(a: Union[float, np.ndarray],\n var_a: Union[float, np.ndarray],\n b: Union[float, np.ndarray],\n var_b: Union[float, np.ndarray]) -> Union[float, np.ndarray]:\n r\"\"\"\n Given random variables 'A' and 'B', compute the variance on the ratio Y = A/B. Denote the\n mean of the random variables as a = E[A] and b = E[B] while the variances are var_a = Var[A]\n and var_b = Var[B] and the covariance as Cov[A,B]. The following expression approximates the\n variance of Y\n\n Var[Y] \\approx (a/b) ^2 * ( var_a /a^2 + var_b / b^2 - 2 * Cov[A,B]/(a*b) )\n\n We assume the covariance of A and B is negligible, resting on the assumption that A and B\n are independently measured. The expression above rests on the assumption that B is non-zero,\n an assumption which we expect to hold true in most cases, but makes no such assumptions\n about A. If we allow E[A] = 0, then calculating the expression above via numpy would complain\n about dividing by zero. Instead, we can re-write the above expression as\n\n Var[Y] \\approx var_a /b^2 + (a^2 * var_b) / b^4\n\n where we have dropped the covariance term as noted above.\n\n See the following for more details:\n - https://doi.org/10.1002/(SICI)1097-0320(20000401)39:4<300::AID-CYTO8>3.0.CO;2-O\n - http://www.stat.cmu.edu/~hseltman/files/ratio.pdf\n - https://en.wikipedia.org/wiki/Taylor_expansions_for_the_moments_of_functions_of_random_variables\n\n :param a: Mean of 'A', to be used as the numerator in a ratio.\n :param var_a: Variance in 'A'\n :param b: Mean of 'B', to be used as the numerator in a ratio.\n :param var_b: Variance in 'B'\n \"\"\"\n return var_a / b**2 + (a**2 * var_b) / b**4", "response": "r Computes the variance of a random variable and returns the ratio of the variance of the random variables A and B."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _exhaustive_symmetrization(qc: QuantumComputer, qubits: List[int],\n shots: int, prog: Program) -> (np.ndarray, Dict):\n \"\"\"\n Perform exhaustive symmetrization\n\n :param qc: A QuantumComputer which can run quantum programs\n :param qubits: qubits on which the symmetrization program runs\n :param shots: number of shots in the symmetrized program\n :prog: program to symmetrize\n :return: - the equivalent of a `run` output, but with exhaustive symmetrization\n - dict keyed by qubit, valued by index of the numpy array containing\n bitstring results\n \"\"\"\n # Symmetrize -- flip qubits pre-measurement\n n_shots_symm = int(round(np.ceil(shots / 2**len(qubits))))\n if n_shots_symm * 2**len(qubits) > shots:\n warnings.warn(f\"Symmetrization increasing number of shots from {shots} to {round(n_shots_symm * 2**len(qubits))}\")\n list_bitstrings_symm = []\n for ops_bool in itertools.product([0, 1], repeat=len(qubits)):\n total_prog_symm = prog.copy()\n prog_symm = _ops_bool_to_prog(ops_bool, qubits)\n total_prog_symm += prog_symm\n # Run the experiment\n dict_qub_idx = {}\n ro = total_prog_symm.declare('ro', 'BIT', len(qubits))\n for i, q in enumerate(qubits):\n total_prog_symm += MEASURE(q, ro[i])\n # Keep track of qubit-classical register mapping via dict\n dict_qub_idx[q] = i\n total_prog_symm.wrap_in_numshots_loop(n_shots_symm)\n total_prog_symm_native = qc.compiler.quil_to_native_quil(total_prog_symm)\n total_prog_symm_bin = qc.compiler.native_quil_to_executable(total_prog_symm_native)\n bitstrings_symm = qc.run(total_prog_symm_bin)\n # Flip the results post-measurement\n bitstrings_symm = bitstrings_symm ^ ops_bool\n # Gather together the symmetrized results into list\n list_bitstrings_symm.append(bitstrings_symm)\n\n # Gather together all the symmetrized results\n bitstrings = reduce(lambda x, y: np.vstack((x, y)), list_bitstrings_symm)\n return bitstrings, dict_qub_idx", "response": "Perform an exhaustive symmetrization on a list of qubits."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _calibration_program(qc: QuantumComputer, tomo_experiment: TomographyExperiment,\n setting: ExperimentSetting) -> Program:\n \"\"\"\n Program required for calibration in a tomography-like experiment.\n\n :param tomo_experiment: A suite of tomographic observables\n :param ExperimentSetting: The particular tomographic observable to measure\n :param symmetrize_readout: Method used to symmetrize the readout errors (see docstring for\n `measure_observables` for more details)\n :param cablir_shots: number of shots to take in the measurement process\n :return: Program performing the calibration\n \"\"\"\n # Inherit any noisy attributes from main Program, including gate definitions\n # and applications which can be handy in creating simulating noisy channels\n calibr_prog = Program()\n # Inherit readout errro instructions from main Program\n readout_povm_instruction = [i for i in tomo_experiment.program.out().split('\\n') if 'PRAGMA READOUT-POVM' in i]\n calibr_prog += readout_povm_instruction\n # Inherit any definitions of noisy gates from main Program\n kraus_instructions = [i for i in tomo_experiment.program.out().split('\\n') if 'PRAGMA ADD-KRAUS' in i]\n calibr_prog += kraus_instructions\n # Prepare the +1 eigenstate for the out operator\n for q, op in setting.out_operator.operations_as_set():\n calibr_prog += _one_q_pauli_prep(label=op, index=0, qubit=q)\n # Measure the out operator in this state\n for q, op in setting.out_operator.operations_as_set():\n calibr_prog += _local_pauli_eig_meas(op, q)\n\n return calibr_prog", "response": "Generates the calibration program required for a tomographic observable."} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nget the value of the environment variable or config file key or default.", "response": "def _env_or_config_or_default(self, env=None, file=None, section=None, name=None, default=None):\n \"\"\"\n Get the value of the environment variable or config file value.\n The environment variable takes precedence.\n\n :param env: The environment variable name.\n :param name: The config file key.\n :return: The value or None if not found\n \"\"\"\n env_val = environ.get(env)\n if env_val is not None:\n return env_val\n\n try:\n return self.configparsers[file].get(section, name)\n except (NoSectionError, NoOptionError, KeyError):\n return default"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef instance(cls, size):\n if not getattr(cls, \"_instance\", None):\n cls._instance = {}\n if size not in cls._instance:\n cls._instance[size] = ThreadPool(size)\n return cls._instance[size]", "response": "Return a threadpool of size size."} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nrun thumbor server with the specified arguments.", "response": "def main(arguments=None):\n '''Runs thumbor server with the specified arguments.'''\n if arguments is None:\n arguments = sys.argv[1:]\n\n server_parameters = get_server_parameters(arguments)\n config = get_config(server_parameters.config_path, server_parameters.use_environment)\n configure_log(config, server_parameters.log_level.upper())\n\n validate_config(config, server_parameters)\n\n importer = get_importer(config)\n\n with get_context(server_parameters, config, importer) as context:\n application = get_application(context)\n server = run_server(application, context)\n setup_signal_handler(server, config)\n logging.debug('thumbor running at %s:%d' % (context.server.ip, context.server.port))\n tornado.ioloop.IOLoop.instance().start()"} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns the mime type of the buffer.", "response": "def mime(self):\n '''\n Retrieves mime metadata if available\n :return:\n '''\n return self.metadata['ContentType'] if 'ContentType' in self.metadata else BaseEngine.get_mimetype(self.buffer)"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncache statsd client so it doesn t do DNS lookup over and over", "response": "def client(cls, config):\n \"\"\"\n Cache statsd client so it doesn't do a DNS lookup\n over and over\n \"\"\"\n if not hasattr(cls, \"_client\"):\n cls._client = statsd.StatsClient(config.STATSD_HOST, config.STATSD_PORT, config.STATSD_PREFIX)\n return cls._client"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef handle_padding(self, padding):\n '''Pads the image with transparent pixels if necessary.'''\n left = padding[0]\n top = padding[1]\n right = padding[2]\n bottom = padding[3]\n\n offset_x = 0\n offset_y = 0\n new_width = self.engine.size[0]\n new_height = self.engine.size[1]\n\n if left > 0:\n offset_x = left\n new_width += left\n if top > 0:\n offset_y = top\n new_height += top\n if right > 0:\n new_width += right\n if bottom > 0:\n new_height += bottom\n new_engine = self.context.modules.engine.__class__(self.context)\n new_engine.image = new_engine.gen_image((new_width, new_height), '#fff')\n new_engine.enable_alpha()\n new_engine.paste(self.engine, (offset_x, offset_y))\n self.engine.image = new_engine.image", "response": "Pads the image with transparent pixels if necessary."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn the target dimensions and calculates them if necessary.", "response": "def get_target_dimensions(self):\n \"\"\"\n Returns the target dimensions and calculates them if necessary.\n The target dimensions are display independent.\n :return: Target dimensions as a tuple (width, height)\n :rtype: (int, int)\n \"\"\"\n if self.target_height is None:\n self._calculate_target_dimensions()\n return int(self.target_width), int(self.target_height)"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ngetting the image orientation of the buffer image or None if it is undefined.", "response": "def get_orientation(self):\n \"\"\"\n Returns the image orientation of the buffer image or None\n if it is undefined. Gets the original value from the Exif tag.\n If the buffer has been rotated, then the value is adjusted to 1.\n :return: Orientation value (1 - 8)\n :rtype: int or None\n \"\"\"\n exif_dict = self._get_exif_segment()\n if exif_dict and piexif.ImageIFD.Orientation in exif_dict[\"0th\"]:\n return exif_dict[\"0th\"][piexif.ImageIFD.Orientation]\n return None"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef reorientate(self, override_exif=True):\n orientation = self.get_orientation()\n\n if orientation is None:\n return\n\n if orientation == 2:\n self.flip_horizontally()\n elif orientation == 3:\n self.rotate(180)\n elif orientation == 4:\n self.flip_vertically()\n elif orientation == 5:\n # Horizontal Mirror + Rotation 270 CCW\n self.flip_vertically()\n self.rotate(270)\n elif orientation == 6:\n self.rotate(270)\n elif orientation == 7:\n # Vertical Mirror + Rotation 270 CCW\n self.flip_horizontally()\n self.rotate(270)\n elif orientation == 8:\n self.rotate(90)\n\n if orientation != 1 and override_exif:\n exif_dict = self._get_exif_segment()\n if exif_dict and piexif.ImageIFD.Orientation in exif_dict[\"0th\"]:\n exif_dict[\"0th\"][piexif.ImageIFD.Orientation] = 1\n try:\n self.exif = piexif.dump(exif_dict)\n except Exception as e:\n msg = \"\"\"[piexif] %s\"\"\" % e\n logger.error(msg)", "response": "Rotates the image in the buffer so that it is oriented correctly."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_image(self):\n try:\n result = yield self._fetch(\n self.context.request.image_url\n )\n\n if not result.successful:\n if result.loader_error == LoaderResult.ERROR_NOT_FOUND:\n self._error(404)\n return\n elif result.loader_error == LoaderResult.ERROR_UPSTREAM:\n # Return a Bad Gateway status if the error came from upstream\n self._error(502)\n return\n elif result.loader_error == LoaderResult.ERROR_TIMEOUT:\n # Return a Gateway Timeout status if upstream timed out (i.e. 599)\n self._error(504)\n return\n elif isinstance(result.loader_error, int):\n self._error(result.loader_error)\n return\n elif hasattr(result, 'engine_error') and result.engine_error == EngineResult.COULD_NOT_LOAD_IMAGE:\n self._error(400)\n return\n else:\n self._error(500)\n return\n\n except Exception as e:\n msg = '[BaseHandler] get_image failed for url `{url}`. error: `{error}`'.format(\n url=self.context.request.image_url,\n error=e\n )\n\n self.log_exception(*sys.exc_info())\n\n if 'cannot identify image file' in e.message:\n logger.warning(msg)\n self._error(400)\n else:\n logger.error(msg)\n self._error(500)\n return\n\n normalized = result.normalized\n buffer = result.buffer\n engine = result.engine\n\n req = self.context.request\n\n if engine is None:\n if buffer is None:\n self._error(504)\n return\n\n engine = self.context.request.engine\n try:\n engine.load(buffer, self.context.request.extension)\n except Exception:\n self._error(504)\n return\n\n self.context.transformer = Transformer(self.context)\n\n def transform():\n self.normalize_crops(normalized, req, engine)\n\n if req.meta:\n self.context.transformer.engine = \\\n self.context.request.engine = \\\n JSONEngine(engine, req.image_url, req.meta_callback)\n\n self.context.transformer.transform(self.after_transform)\n\n self.filters_runner.apply_filters(thumbor.filters.PHASE_AFTER_LOAD, transform)", "response": "This function returns the image from the server."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef main(arguments=None):\n '''Converts a given url with the specified arguments.'''\n\n parsed_options, arguments = get_options(arguments)\n\n image_url = arguments[0]\n image_url = quote(image_url)\n\n try:\n config = Config.load(None)\n except Exception:\n config = None\n\n if not parsed_options.key and not config:\n sys.stdout.write('Error: The -k or --key argument is mandatory. For more information type thumbor-url -h\\n')\n return\n\n security_key, thumbor_params = get_thumbor_params(image_url, parsed_options, config)\n\n crypto = CryptoURL(key=security_key)\n url = crypto.generate(**thumbor_params)\n sys.stdout.write('URL:\\n')\n sys.stdout.write('%s\\n' % url)\n\n return url", "response": "Converts a given url with the specified arguments."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncheck numpy images and correct intensity range etc.", "response": "def checkImages(images):\n \"\"\" checkImages(images)\n Check numpy images and correct intensity range etc.\n The same for all movie formats.\n \"\"\"\n # Init results\n images2 = []\n\n for im in images:\n if PIL and isinstance(im, PIL.Image.Image):\n # We assume PIL images are allright\n images2.append(im)\n\n elif np and isinstance(im, np.ndarray):\n # Check and convert dtype\n if im.dtype == np.uint8:\n images2.append(im) # Ok\n elif im.dtype in [np.float32, np.float64]:\n im = im.copy()\n im[im < 0] = 0\n im[im > 1] = 1\n im *= 255\n images2.append(im.astype(np.uint8))\n else:\n im = im.astype(np.uint8)\n images2.append(im)\n # Check size\n if im.ndim == 2:\n pass # ok\n elif im.ndim == 3:\n if im.shape[2] not in [3, 4]:\n raise ValueError('This array can not represent an image.')\n else:\n raise ValueError('This array can not represent an image.')\n else:\n raise ValueError('Invalid image type: ' + str(type(im)))\n\n # Done\n return images2"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nconverts an integer to two bytes.", "response": "def intToBin(i):\n \"\"\" Integer to two bytes \"\"\"\n # devide in two parts (bytes)\n i1 = i % 256\n i2 = int(i / 256)\n # make string (little endian)\n return chr(i1) + chr(i2)"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nwrites an animated gif from the specified images.", "response": "def writeGif(\n filename, images, duration=0.1, repeat=True, dither=False,\n nq=0, subRectangles=True, dispose=None):\n \"\"\" writeGif(filename, images, duration=0.1, repeat=True, dither=False,\n nq=0, subRectangles=True, dispose=None)\n\n Write an animated gif from the specified images.\n\n Parameters\n ----------\n filename : string\n The name of the file to write the image to.\n images : list\n Should be a list consisting of PIL images or numpy arrays.\n The latter should be between 0 and 255 for integer types, and\n between 0 and 1 for float types.\n duration : scalar or list of scalars\n The duration for all frames, or (if a list) for each frame.\n repeat : bool or integer\n The amount of loops. If True, loops infinitetely.\n dither : bool\n Whether to apply dithering\n nq : integer\n If nonzero, applies the NeuQuant quantization algorithm to create\n the color palette. This algorithm is superior, but slower than\n the standard PIL algorithm. The value of nq is the quality\n parameter. 1 represents the best quality. 10 is in general a\n good tradeoff between quality and speed. When using this option,\n better results are usually obtained when subRectangles is False.\n subRectangles : False, True, or a list of 2-element tuples\n Whether to use sub-rectangles. If True, the minimal rectangle that\n is required to update each frame is automatically detected. This\n can give significant reductions in file size, particularly if only\n a part of the image changes. One can also give a list of x-y\n coordinates if you want to do the cropping yourself. The default\n is True.\n dispose : int\n How to dispose each frame. 1 means that each frame is to be left\n in place. 2 means the background color should be restored after\n each frame. 3 means the decoder should restore the previous frame.\n If subRectangles==False, the default is 2, otherwise it is 1.\n\n \"\"\"\n\n # Check PIL\n if PIL is None:\n raise RuntimeError(\"Need PIL to write animated gif files.\")\n\n # Check images\n images = checkImages(images)\n\n # Instantiate writer object\n gifWriter = GifWriter()\n\n # Check loops\n if repeat is False:\n loops = 1\n elif repeat is True:\n loops = 0 # zero means infinite\n else:\n loops = int(repeat)\n\n # Check duration\n if hasattr(duration, '__len__'):\n if len(duration) == len(images):\n duration = [d for d in duration]\n else:\n raise ValueError(\"len(duration) doesn't match amount of images.\")\n else:\n duration = [duration for im in images]\n\n # Check subrectangles\n if subRectangles:\n images, xy = gifWriter.handleSubRectangles(images, subRectangles)\n defaultDispose = 1 # Leave image in place\n else:\n # Normal mode\n xy = [(0, 0) for im in images]\n defaultDispose = 2 # Restore to background color.\n\n # Check dispose\n if dispose is None:\n dispose = defaultDispose\n if hasattr(dispose, '__len__'):\n if len(dispose) != len(images):\n raise ValueError(\"len(xy) doesn't match amount of images.\")\n else:\n dispose = [dispose for im in images]\n\n # Make images in a format that we can write easy\n images = gifWriter.convertImagesToPIL(images, dither, nq)\n\n # Write\n fp = open(filename, 'wb')\n try:\n gifWriter.writeGifToFile(fp, images, duration, loops, xy, dispose)\n finally:\n fp.close()"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreading an animated GIF file into a list of numpy arrays.", "response": "def readGif(filename, asNumpy=True):\n \"\"\" readGif(filename, asNumpy=True)\n\n Read images from an animated GIF file. Returns a list of numpy\n arrays, or, if asNumpy is false, a list if PIL images.\n\n \"\"\"\n\n # Check PIL\n if PIL is None:\n raise RuntimeError(\"Need PIL to read animated gif files.\")\n\n # Check Numpy\n if np is None:\n raise RuntimeError(\"Need Numpy to read animated gif files.\")\n\n # Check whether it exists\n if not os.path.isfile(filename):\n raise IOError('File not found: ' + str(filename))\n\n # Load file using PIL\n pilIm = PIL.Image.open(filename)\n pilIm.seek(0)\n\n # Read all images inside\n images = []\n try:\n while True:\n # Get image as numpy array\n tmp = pilIm.convert() # Make without palette\n a = np.asarray(tmp)\n if len(a.shape) == 0:\n raise MemoryError(\"Too little memory to convert PIL image to array\")\n # Store, and next\n images.append(a)\n pilIm.seek(pilIm.tell() + 1)\n except EOFError:\n pass\n\n # Convert to normal PIL images if needed\n if not asNumpy:\n images2 = images\n images = []\n for im in images2:\n images.append(PIL.Image.fromarray(im))\n\n # Done\n return images"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef getheaderAnim(self, im):\n bb = \"GIF89a\"\n bb += intToBin(im.size[0])\n bb += intToBin(im.size[1])\n bb += \"\\x87\\x00\\x00\"\n return bb", "response": "Get animation header. To replace PILs getheaderAnim"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn the image descriptor for the given image and xy.", "response": "def getImageDescriptor(self, im, xy=None):\n \"\"\" getImageDescriptor(im, xy=None)\n\n Used for the local color table properties per image.\n Otherwise global color table applies to all frames irrespective of\n whether additional colors comes in play that require a redefined\n palette. Still a maximum of 256 color per frame, obviously.\n\n Written by Ant1 on 2010-08-22\n Modified by Alex Robinson in Janurari 2011 to implement subrectangles.\n\n \"\"\"\n\n # Defaule use full image and place at upper left\n if xy is None:\n xy = (0, 0)\n\n # Image separator,\n bb = '\\x2C'\n\n # Image position and size\n bb += intToBin(xy[0]) # Left position\n bb += intToBin(xy[1]) # Top position\n bb += intToBin(im.size[0]) # image width\n bb += intToBin(im.size[1]) # image height\n\n # packed field: local color table flag1, interlace0, sorted table0,\n # reserved00, lct size111=7=2^(7+1)=256.\n\n bb += '\\x87'\n\n # LZW minimum size code now comes later, begining of [image data] blocks\n return bb"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef getAppExt(self, loops=float('inf')):\n\n if loops == 0 or loops == float('inf'):\n loops = 2 ** 16 - 1\n # bb = \"\" application extension should not be used\n # (the extension interprets zero loops\n # to mean an infinite number of loops)\n # Mmm, does not seem to work\n if True:\n bb = \"\\x21\\xFF\\x0B\" # application extension\n bb += \"NETSCAPE2.0\"\n bb += \"\\x03\\x01\"\n bb += intToBin(loops)\n bb += '\\x00' # end\n return bb", "response": "getAppExt - Returns the application extension."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef getGraphicsControlExt(self, duration=0.1, dispose=2):\n\n bb = '\\x21\\xF9\\x04'\n bb += chr((dispose & 3) << 2) # low bit 1 == transparency,\n # 2nd bit 1 == user input , next 3 bits, the low two of which are used,\n # are dispose.\n bb += intToBin(int(duration * 100)) # in 100th of seconds\n bb += '\\x00' # no transparant color\n bb += '\\x00' # end\n return bb", "response": "Returns the string that represents the Graphics Control Extension."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nhandling the sub - rectangles.", "response": "def handleSubRectangles(self, images, subRectangles):\n \"\"\" handleSubRectangles(images)\n\n Handle the sub-rectangle stuff. If the rectangles are given by the\n user, the values are checked. Otherwise the subrectangles are\n calculated automatically.\n\n \"\"\"\n\n if isinstance(subRectangles, (tuple, list)):\n # xy given directly\n\n # Check xy\n xy = subRectangles\n if xy is None:\n xy = (0, 0)\n if hasattr(xy, '__len__'):\n if len(xy) == len(images):\n xy = [xxyy for xxyy in xy]\n else:\n raise ValueError(\"len(xy) doesn't match amount of images.\")\n else:\n xy = [xy for im in images]\n xy[0] = (0, 0)\n\n else:\n # Calculate xy using some basic image processing\n\n # Check Numpy\n if np is None:\n raise RuntimeError(\"Need Numpy to use auto-subRectangles.\")\n\n # First make numpy arrays if required\n for i in range(len(images)):\n im = images[i]\n if isinstance(im, Image.Image):\n tmp = im.convert() # Make without palette\n a = np.asarray(tmp)\n if len(a.shape) == 0:\n raise MemoryError(\"Too little memory to convert PIL image to array\")\n images[i] = a\n\n # Determine the sub rectangles\n images, xy = self.getSubRectangles(images)\n\n # Done\n return images, xy"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncalculate the minimal rectangles that need updating each frame.", "response": "def getSubRectangles(self, ims):\n \"\"\" getSubRectangles(ims)\n\n Calculate the minimal rectangles that need updating each frame.\n Returns a two-element tuple containing the cropped images and a\n list of x-y positions.\n\n Calculating the subrectangles takes extra time, obviously. However,\n if the image sizes were reduced, the actual writing of the GIF\n goes faster. In some cases applying this method produces a GIF faster.\n\n \"\"\"\n\n # Check image count\n if len(ims) < 2:\n return ims, [(0, 0) for i in ims]\n\n # We need numpy\n if np is None:\n raise RuntimeError(\"Need Numpy to calculate sub-rectangles. \")\n\n # Prepare\n ims2 = [ims[0]]\n xy = [(0, 0)]\n # t0 = time.time()\n\n # Iterate over images\n prev = ims[0]\n for im in ims[1:]:\n\n # Get difference, sum over colors\n diff = np.abs(im - prev)\n if diff.ndim == 3:\n diff = diff.sum(2)\n # Get begin and end for both dimensions\n X = np.argwhere(diff.sum(0))\n Y = np.argwhere(diff.sum(1))\n # Get rect coordinates\n if X.size and Y.size:\n x0, x1 = X[0], X[-1] + 1\n y0, y1 = Y[0], Y[-1] + 1\n else: # No change ... make it minimal\n x0, x1 = 0, 2\n y0, y1 = 0, 2\n\n # Cut out and store\n im2 = im[y0:y1, x0:x1]\n prev = im\n ims2.append(im2)\n xy.append((x0, y0))\n\n # Done\n # print('%1.2f seconds to determine subrectangles of %i images' %\n # (time.time()-t0, len(ims2)) )\n return ims2, xy"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nconverting images to Paletted PIL images which can then be written to a single animaged GIF.", "response": "def convertImagesToPIL(self, images, dither, nq=0):\n \"\"\" convertImagesToPIL(images, nq=0)\n\n Convert images to Paletted PIL images, which can then be\n written to a single animaged GIF.\n\n \"\"\"\n\n # Convert to PIL images\n images2 = []\n for im in images:\n if isinstance(im, Image.Image):\n images2.append(im)\n elif np and isinstance(im, np.ndarray):\n if im.ndim == 3 and im.shape[2] == 3:\n im = Image.fromarray(im, 'RGB')\n elif im.ndim == 3 and im.shape[2] == 4:\n im = Image.fromarray(im[:, :, :3], 'RGB')\n elif im.ndim == 2:\n im = Image.fromarray(im, 'L')\n images2.append(im)\n\n # Convert to paletted PIL images\n images, images2 = images2, []\n\n # Adaptive PIL algorithm\n AD = Image.ADAPTIVE\n for im in images:\n im = im.convert('P', palette=AD, dither=dither)\n images2.append(im)\n\n # Done\n return images2"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef writeGifToFile(self, fp, images, durations, loops, xys, disposes):\n # Obtain palette for all images and count each occurance\n palettes, occur = [], []\n for im in images:\n header, usedPaletteColors = getheader(im)\n palettes.append(header[-1]) # Last part of the header is the frame palette\n for palette in palettes:\n occur.append(palettes.count(palette))\n\n # Select most-used palette as the global one (or first in case no max)\n globalPalette = palettes[occur.index(max(occur))]\n\n # Init\n frames = 0\n firstFrame = True\n\n for im, palette in zip(images, palettes):\n\n if firstFrame:\n # Write header\n\n # Gather info\n header = self.getheaderAnim(im)\n appext = self.getAppExt(loops)\n\n # Write\n fp.write(header)\n fp.write(globalPalette)\n fp.write(appext)\n\n # Next frame is not the first\n firstFrame = False\n\n if True:\n # Write palette and image data\n\n # Gather info\n data = getdata(im)\n\n imdes, data = b''.join(data[:-2]), data[-2:]\n graphext = self.getGraphicsControlExt(durations[frames], disposes[frames])\n # Make image descriptor suitable for using 256 local color palette\n lid = self.getImageDescriptor(im, xys[frames])\n\n # Write local header\n if (palette != globalPalette) or (disposes[frames] != 2):\n # Use local color palette\n fp.write(graphext)\n fp.write(lid) # write suitable image descriptor\n fp.write(palette) # write local color table\n fp.write('\\x08') # LZW minimum size code\n else:\n # Use global color palette\n fp.write(graphext)\n fp.write(imdes) # write suitable image descriptor\n\n for d in data:\n fp.write(d)\n\n # Prepare for next round\n frames = frames + 1\n\n fp.write(\";\") # end gif\n return frames", "response": "Writes the Gif to file."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ngenerating an application string based on some of the given information that can be pulled from the test object.", "response": "def generate_application_string(cls, test):\n \"\"\" Generate an application string based on some of the given information\n that can be pulled from the test object: app_env, start_time. \"\"\"\n\n app_env = 'test'\n if hasattr(test, 'env'):\n app_env = test.env\n elif hasattr(test, 'environment'):\n app_env = test.environment\n\n start_time = int(time.time() * 1000)\n\n return \"%s.%s\" % (app_env, start_time)"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _filter_xpath_grouping(xpath):\n\n # First remove the first open parentheses\n xpath = xpath[1:]\n\n # Next remove the last closed parentheses\n index = xpath.rfind(')')\n if index == -1:\n raise XpathException(\"Invalid or unsupported Xpath: %s\" % xpath)\n xpath = xpath[:index] + xpath[index + 1:]\n return xpath", "response": "This method removes the outer parentheses for xpath grouping."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncreates a Chrome Proxy ZIP file.", "response": "def create_proxy_zip(proxy_string, proxy_user, proxy_pass):\n \"\"\" Implementation of https://stackoverflow.com/a/35293284 for\n https://stackoverflow.com/questions/12848327/\n (Run Selenium on a proxy server that requires authentication.)\n Solution involves creating & adding a Chrome extension on the fly.\n * CHROME-ONLY for now! *\n \"\"\"\n proxy_host = proxy_string.split(':')[0]\n proxy_port = proxy_string.split(':')[1]\n background_js = (\n \"\"\"var config = {\\n\"\"\"\n \"\"\" mode: \"fixed_servers\",\\n\"\"\"\n \"\"\" rules: {\\n\"\"\"\n \"\"\" singleProxy: {\\n\"\"\"\n \"\"\" scheme: \"http\",\\n\"\"\"\n \"\"\" host: \"%s\",\\n\"\"\"\n \"\"\" port: parseInt(\"%s\")\\n\"\"\"\n \"\"\" },\\n\"\"\"\n \"\"\" }\\n\"\"\"\n \"\"\" };\\n\"\"\"\n \"\"\"chrome.proxy.settings.set(\"\"\"\n \"\"\"{value: config, scope: \"regular\"}, function() {\"\"\"\n \"\"\"});\\n\"\"\"\n \"\"\"function callbackFn(details) {\\n\"\"\"\n \"\"\" return {\\n\"\"\"\n \"\"\" authCredentials: {\\n\"\"\"\n \"\"\" username: \"%s\",\\n\"\"\"\n \"\"\" password: \"%s\"\\n\"\"\"\n \"\"\" }\\n\"\"\"\n \"\"\" };\\n\"\"\"\n \"\"\"}\\n\"\"\"\n \"\"\"chrome.webRequest.onAuthRequired.addListener(\\n\"\"\"\n \"\"\" callbackFn,\\n\"\"\"\n \"\"\" {urls: [\"\"]},\\n\"\"\"\n \"\"\" ['blocking']\\n\"\"\"\n \"\"\");\"\"\" % (proxy_host, proxy_port, proxy_user, proxy_pass))\n manifest_json = (\n '''{\\n'''\n '''\"version\": \"1.0.0\",\\n'''\n '''\"manifest_version\": 2,\\n'''\n '''\"name\": \"Chrome Proxy\",\\n'''\n '''\"permissions\": [\\n'''\n ''' \"proxy\",\\n'''\n ''' \"tabs\",\\n'''\n ''' \"unlimitedStorage\",\\n'''\n ''' \"storage\",\\n'''\n ''' \"\",\\n'''\n ''' \"webRequest\",\\n'''\n ''' \"webRequestBlocking\"\\n'''\n '''],\\n'''\n '''\"background\": {\\n'''\n ''' \"scripts\": [\"background.js\"]\\n'''\n '''},\\n'''\n '''\"minimum_chrome_version\":\"22.0.0\"\\n'''\n '''}''')\n lock = threading.RLock() # Support multi-threaded test runs with Pytest\n with lock:\n try:\n zf = zipfile.ZipFile(PROXY_ZIP_PATH, mode='w')\n except IOError:\n # Handle \"Permission denied\" on the default proxy.zip path\n abs_path = os.path.abspath('.')\n downloads_path = os.path.join(abs_path, DOWNLOADS_DIR)\n if not os.path.exists(downloads_path):\n os.mkdir(downloads_path)\n zf = zipfile.ZipFile(PROXY_ZIP_PATH_2, mode='w')\n zf.writestr(\"background.js\", background_js)\n zf.writestr(\"manifest.json\", manifest_json)\n zf.close()"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef remove_proxy_zip_if_present():\n try:\n if os.path.exists(PROXY_ZIP_PATH):\n os.remove(PROXY_ZIP_PATH)\n elif os.path.exists(PROXY_ZIP_PATH_2):\n os.remove(PROXY_ZIP_PATH_2)\n except Exception:\n pass", "response": "Remove the Chrome extension zip file used for proxy server authentication."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef click_chain(self, selectors_list, by=By.CSS_SELECTOR,\n timeout=settings.SMALL_TIMEOUT, spacing=0):\n \"\"\" This method clicks on a list of elements in succession.\n 'spacing' is the amount of time to wait between clicks. (sec) \"\"\"\n if self.timeout_multiplier and timeout == settings.SMALL_TIMEOUT:\n timeout = self.__get_new_timeout(timeout)\n for selector in selectors_list:\n self.click(selector, by=by, timeout=timeout)\n if spacing > 0:\n time.sleep(spacing)", "response": "This method clicks on a list of elements in succession."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef is_link_text_present(self, link_text):\n soup = self.get_beautiful_soup()\n html_links = soup.find_all('a')\n for html_link in html_links:\n if html_link.text.strip() == link_text.strip():\n return True\n return False", "response": "Returns True if the link text appears in the HTML of the page."} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nfind a link by link text and then returns the attribute s value.", "response": "def get_link_attribute(self, link_text, attribute, hard_fail=True):\n \"\"\" Finds a link by link text and then returns the attribute's value.\n If the link text or attribute cannot be found, an exception will\n get raised if hard_fail is True (otherwise None is returned). \"\"\"\n soup = self.get_beautiful_soup()\n html_links = soup.find_all('a')\n for html_link in html_links:\n if html_link.text.strip() == link_text.strip():\n if html_link.has_attr(attribute):\n attribute_value = html_link.get(attribute)\n return attribute_value\n if hard_fail:\n raise Exception(\n 'Unable to find attribute {%s} from link text {%s}!'\n % (attribute, link_text))\n else:\n return None\n if hard_fail:\n raise Exception(\"Link text {%s} was not found!\" % link_text)\n else:\n return None"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef click_link_text(self, link_text, timeout=settings.SMALL_TIMEOUT):\n # If using phantomjs, might need to extract and open the link directly\n if self.timeout_multiplier and timeout == settings.SMALL_TIMEOUT:\n timeout = self.__get_new_timeout(timeout)\n if self.browser == 'phantomjs':\n if self.is_link_text_visible(link_text):\n element = self.wait_for_link_text_visible(\n link_text, timeout=timeout)\n element.click()\n return\n self.open(self.__get_href_from_link_text(link_text))\n return\n if not self.is_link_text_present(link_text):\n self.wait_for_link_text_present(link_text, timeout=timeout)\n pre_action_url = self.get_current_url()\n try:\n element = self.wait_for_link_text_visible(\n link_text, timeout=0.2)\n self.__demo_mode_highlight_if_active(link_text, by=By.LINK_TEXT)\n try:\n element.click()\n except (StaleElementReferenceException, ENI_Exception):\n self.wait_for_ready_state_complete()\n time.sleep(0.05)\n element = self.wait_for_link_text_visible(\n link_text, timeout=timeout)\n element.click()\n except Exception:\n found_css = False\n text_id = self.get_link_attribute(link_text, \"id\", False)\n if text_id:\n link_css = '[id=\"%s\"]' % link_text\n found_css = True\n\n if not found_css:\n href = self.__get_href_from_link_text(link_text, False)\n if href:\n if href.startswith('/') or page_utils.is_valid_url(href):\n link_css = '[href=\"%s\"]' % href\n found_css = True\n\n if not found_css:\n ngclick = self.get_link_attribute(link_text, \"ng-click\", False)\n if ngclick:\n link_css = '[ng-click=\"%s\"]' % ngclick\n found_css = True\n\n if not found_css:\n onclick = self.get_link_attribute(link_text, \"onclick\", False)\n if onclick:\n link_css = '[onclick=\"%s\"]' % onclick\n found_css = True\n\n success = False\n if found_css:\n if self.is_element_visible(link_css):\n self.click(link_css)\n success = True\n else:\n # The link text might be hidden under a dropdown menu\n success = self.__click_dropdown_link_text(\n link_text, link_css)\n\n if not success:\n element = self.wait_for_link_text_visible(\n link_text, timeout=settings.MINI_TIMEOUT)\n element.click()\n\n if settings.WAIT_FOR_RSC_ON_CLICKS:\n self.wait_for_ready_state_complete()\n if self.demo_mode:\n if self.driver.current_url != pre_action_url:\n self.__demo_mode_pause_if_active()\n else:\n self.__demo_mode_pause_if_active(tiny=True)", "response": "This method clicks the link text on a page."} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nclicks a link text.", "response": "def click_link(self, link_text, timeout=settings.SMALL_TIMEOUT):\n \"\"\" Same as self.click_link_text() \"\"\"\n self.click_link_text(link_text, timeout=timeout)"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef click_partial_link_text(self, partial_link_text,\n timeout=settings.SMALL_TIMEOUT):\n \"\"\" This method clicks the partial link text on a page. \"\"\"\n # If using phantomjs, might need to extract and open the link directly\n if self.timeout_multiplier and timeout == settings.SMALL_TIMEOUT:\n timeout = self.__get_new_timeout(timeout)\n if self.browser == 'phantomjs':\n if self.is_partial_link_text_visible(partial_link_text):\n element = self.wait_for_partial_link_text(partial_link_text)\n element.click()\n return\n soup = self.get_beautiful_soup()\n html_links = soup.fetch('a')\n for html_link in html_links:\n if partial_link_text in html_link.text:\n for html_attribute in html_link.attrs:\n if html_attribute[0] == 'href':\n href = html_attribute[1]\n if href.startswith('//'):\n link = \"http:\" + href\n elif href.startswith('/'):\n url = self.driver.current_url\n domain_url = self.get_domain_url(url)\n link = domain_url + href\n else:\n link = href\n self.open(link)\n return\n raise Exception(\n 'Could not parse link from partial link_text '\n '{%s}' % partial_link_text)\n raise Exception(\n \"Partial link text {%s} was not found!\" % partial_link_text)\n # Not using phantomjs\n element = self.wait_for_partial_link_text(\n partial_link_text, timeout=timeout)\n self.__demo_mode_highlight_if_active(\n partial_link_text, by=By.PARTIAL_LINK_TEXT)\n pre_action_url = self.driver.current_url\n try:\n element.click()\n except (StaleElementReferenceException, ENI_Exception):\n self.wait_for_ready_state_complete()\n time.sleep(0.05)\n element = self.wait_for_partial_link_text(\n partial_link_text, timeout=timeout)\n element.click()\n if settings.WAIT_FOR_RSC_ON_CLICKS:\n self.wait_for_ready_state_complete()\n if self.demo_mode:\n if self.driver.current_url != pre_action_url:\n self.__demo_mode_pause_if_active()\n else:\n self.__demo_mode_pause_if_active(tiny=True)", "response": "This method clicks the partial link text on a page."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning the value of a page element s computed style property.", "response": "def get_property_value(self, selector, property, by=By.CSS_SELECTOR,\n timeout=settings.SMALL_TIMEOUT):\n \"\"\" Returns the property value of a page element's computed style.\n Example:\n opacity = self.get_property_value(\"html body a\", \"opacity\")\n self.assertTrue(float(opacity) > 0, \"Element not visible!\") \"\"\"\n if self.timeout_multiplier and timeout == settings.SMALL_TIMEOUT:\n timeout = self.__get_new_timeout(timeout)\n if page_utils.is_xpath_selector(selector):\n by = By.XPATH\n if page_utils.is_link_text_selector(selector):\n selector = page_utils.get_link_text_from_selector(selector)\n by = By.LINK_TEXT\n self.wait_for_ready_state_complete()\n page_actions.wait_for_element_present(\n self.driver, selector, by, timeout)\n try:\n selector = self.convert_to_css_selector(selector, by=by)\n except Exception:\n # Don't run action if can't convert to CSS_Selector for JavaScript\n raise Exception(\n \"Exception: Could not convert {%s}(by=%s) to CSS_SELECTOR!\" % (\n selector, by))\n selector = re.escape(selector)\n selector = self.__escape_quotes_if_needed(selector)\n script = (\"\"\"var $elm = document.querySelector('%s');\n $val = window.getComputedStyle($elm).getPropertyValue('%s');\n return $val;\"\"\"\n % (selector, property))\n value = self.execute_script(script)\n if value is not None:\n return value\n else:\n return \"\""} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nextracts the URL from an image element on the page.", "response": "def get_image_url(self, selector, by=By.CSS_SELECTOR,\n timeout=settings.SMALL_TIMEOUT):\n \"\"\" Extracts the URL from an image element on the page. \"\"\"\n if self.timeout_multiplier and timeout == settings.SMALL_TIMEOUT:\n timeout = self.__get_new_timeout(timeout)\n return self.get_attribute(selector,\n attribute='src', by=by, timeout=timeout)"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef add_text(self, selector, new_value, by=By.CSS_SELECTOR,\n timeout=settings.LARGE_TIMEOUT):\n \"\"\" The more-reliable version of driver.send_keys()\n Similar to update_text(), but won't clear the text field first. \"\"\"\n if self.timeout_multiplier and timeout == settings.LARGE_TIMEOUT:\n timeout = self.__get_new_timeout(timeout)\n if page_utils.is_xpath_selector(selector):\n by = By.XPATH\n element = self.wait_for_element_visible(\n selector, by=by, timeout=timeout)\n self.__demo_mode_highlight_if_active(selector, by)\n if not self.demo_mode:\n self.__scroll_to_element(element)\n pre_action_url = self.driver.current_url\n try:\n if not new_value.endswith('\\n'):\n element.send_keys(new_value)\n else:\n new_value = new_value[:-1]\n element.send_keys(new_value)\n element.send_keys(Keys.RETURN)\n if settings.WAIT_FOR_RSC_ON_PAGE_LOADS:\n self.wait_for_ready_state_complete()\n except (StaleElementReferenceException, ENI_Exception):\n self.wait_for_ready_state_complete()\n time.sleep(0.06)\n element = self.wait_for_element_visible(\n selector, by=by, timeout=timeout)\n if not new_value.endswith('\\n'):\n element.send_keys(new_value)\n else:\n new_value = new_value[:-1]\n element.send_keys(new_value)\n element.send_keys(Keys.RETURN)\n if settings.WAIT_FOR_RSC_ON_PAGE_LOADS:\n self.wait_for_ready_state_complete()\n except Exception:\n exc_message = self.__get_improved_exception_message()\n raise Exception(exc_message)\n if self.demo_mode:\n if self.driver.current_url != pre_action_url:\n self.__demo_mode_pause_if_active()\n else:\n self.__demo_mode_pause_if_active(tiny=True)", "response": "This method adds a text field to the element."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef send_keys(self, selector, new_value, by=By.CSS_SELECTOR,\n timeout=settings.LARGE_TIMEOUT):\n \"\"\" Same as add_text() -> more reliable, but less name confusion. \"\"\"\n if self.timeout_multiplier and timeout == settings.LARGE_TIMEOUT:\n timeout = self.__get_new_timeout(timeout)\n if page_utils.is_xpath_selector(selector):\n by = By.XPATH\n self.add_text(selector, new_value, by=by, timeout=timeout)", "response": "Send a keypress to the page."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef update_text(self, selector, new_value, by=By.CSS_SELECTOR,\n timeout=settings.LARGE_TIMEOUT, retry=False):\n \"\"\" The shorter version of update_text_value(), which\n clears existing text and adds new text into the text field.\n We want to keep the old version for backward compatibility. \"\"\"\n if self.timeout_multiplier and timeout == settings.LARGE_TIMEOUT:\n timeout = self.__get_new_timeout(timeout)\n if page_utils.is_xpath_selector(selector):\n by = By.XPATH\n self.update_text_value(selector, new_value, by=by,\n timeout=timeout, retry=retry)", "response": "This method updates the text field of the current object."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef find_elements(self, selector, by=By.CSS_SELECTOR, limit=0):\n self.wait_for_ready_state_complete()\n if page_utils.is_xpath_selector(selector):\n by = By.XPATH\n if page_utils.is_link_text_selector(selector):\n selector = page_utils.get_link_text_from_selector(selector)\n by = By.LINK_TEXT\n elements = self.driver.find_elements(by=by, value=selector)\n if limit and limit > 0 and len(elements) > limit:\n elements = elements[:limit]\n return elements", "response": "Returns a list of matching WebElements."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef find_visible_elements(self, selector, by=By.CSS_SELECTOR, limit=0):\n self.wait_for_ready_state_complete()\n if page_utils.is_xpath_selector(selector):\n by = By.XPATH\n if page_utils.is_link_text_selector(selector):\n selector = page_utils.get_link_text_from_selector(selector)\n by = By.LINK_TEXT\n v_elems = page_actions.find_visible_elements(self.driver, selector, by)\n if limit and limit > 0 and len(v_elems) > limit:\n v_elems = v_elems[:limit]\n return v_elems", "response": "Returns a list of matching WebElements that are visible.\n If \"limit\" is set and > 0, will only return that many elements."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nfinding all matching page elements and clicks visible ones.", "response": "def click_visible_elements(self, selector, by=By.CSS_SELECTOR, limit=0):\n \"\"\" Finds all matching page elements and clicks visible ones in order.\n If a click reloads or opens a new page, the clicking will stop.\n Works best for actions such as clicking all checkboxes on a page.\n Example: self.click_visible_elements('input[type=\"checkbox\"]')\n If \"limit\" is set and > 0, will only click that many elements. \"\"\"\n elements = self.find_elements(selector, by=by)\n count = 0\n click_count = 0\n for element in elements:\n if limit and limit > 0 and click_count >= limit:\n return\n count += 1\n if count == 1:\n self.wait_for_ready_state_complete()\n if self.is_element_visible(selector, by=by):\n self.click(selector, by=by)\n click_count += 1\n else:\n self.wait_for_ready_state_complete()\n try:\n if element.is_displayed():\n self.__scroll_to_element(element)\n element.click()\n click_count += 1\n except (StaleElementReferenceException, ENI_Exception):\n self.wait_for_ready_state_complete()\n time.sleep(0.05)\n try:\n if element.is_displayed():\n self.__scroll_to_element(element)\n element.click()\n click_count += 1\n except (StaleElementReferenceException, ENI_Exception):\n return"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef is_element_in_an_iframe(self, selector, by=By.CSS_SELECTOR):\n selector, by = self.__recalculate_selector(selector, by)\n if self.is_element_present(selector, by=by):\n return False\n soup = self.get_beautiful_soup()\n iframe_list = soup.select('iframe')\n for iframe in iframe_list:\n iframe_identifier = None\n if iframe.has_attr('name') and len(iframe['name']) > 0:\n iframe_identifier = iframe['name']\n elif iframe.has_attr('id') and len(iframe['id']) > 0:\n iframe_identifier = iframe['id']\n else:\n continue\n self.switch_to_frame(iframe_identifier)\n if self.is_element_present(selector, by=by):\n self.switch_to_default_content()\n return True\n self.switch_to_default_content()\n return False", "response": "Returns True if the selector s element is located in an iframe otherwise returns False."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef create_tour(self, name=None, theme=None):\n if not name:\n name = \"default\"\n\n if theme:\n if theme.lower() == \"bootstrap\":\n self.create_bootstrap_tour(name)\n return\n elif theme.lower() == \"hopscotch\":\n self.create_hopscotch_tour(name)\n return\n elif theme.lower() == \"intro\":\n self.create_introjs_tour(name)\n return\n elif theme.lower() == \"introjs\":\n self.create_introjs_tour(name)\n return\n elif theme.lower() == \"shepherd\":\n self.create_shepherd_tour(name, theme=\"light\")\n return\n else:\n self.create_shepherd_tour(name, theme)\n else:\n self.create_shepherd_tour(name, theme=\"light\")", "response": "Creates a new tour for a website."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef create_shepherd_tour(self, name=None, theme=None):\n\n shepherd_theme = \"shepherd-theme-arrows\"\n if theme:\n if theme.lower() == \"default\":\n shepherd_theme = \"shepherd-theme-default\"\n elif theme.lower() == \"dark\":\n shepherd_theme = \"shepherd-theme-dark\"\n elif theme.lower() == \"light\":\n shepherd_theme = \"shepherd-theme-arrows\"\n elif theme.lower() == \"arrows\":\n shepherd_theme = \"shepherd-theme-arrows\"\n elif theme.lower() == \"square\":\n shepherd_theme = \"shepherd-theme-square\"\n elif theme.lower() == \"square-dark\":\n shepherd_theme = \"shepherd-theme-square-dark\"\n\n if not name:\n name = \"default\"\n\n new_tour = (\n \"\"\"\n // Shepherd Tour\n var tour = new Shepherd.Tour({\n defaults: {\n classes: '%s',\n scrollTo: true\n }\n });\n var allButtons = {\n skip: {\n text: \"Skip\",\n action: tour.cancel,\n classes: 'shepherd-button-secondary tour-button-left'\n },\n back: {\n text: \"Back\",\n action: tour.back,\n classes: 'shepherd-button-secondary'\n },\n next: {\n text: \"Next\",\n action: tour.next,\n classes: 'shepherd-button-primary tour-button-right'\n },\n };\n var firstStepButtons = [allButtons.skip, allButtons.next];\n var midTourButtons = [allButtons.back, allButtons.next];\n \"\"\" % shepherd_theme)\n\n self._tour_steps[name] = []\n self._tour_steps[name].append(new_tour)", "response": "Creates a new Shepherd tour."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef create_bootstrap_tour(self, name=None):\n if not name:\n name = \"default\"\n\n new_tour = (\n \"\"\"\n // Bootstrap Tour\n var tour = new Tour({\n });\n tour.addSteps([\n \"\"\")\n\n self._tour_steps[name] = []\n self._tour_steps[name].append(new_tour)", "response": "Creates a Bootstrap tour for a website."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef create_hopscotch_tour(self, name=None):\n if not name:\n name = \"default\"\n\n new_tour = (\n \"\"\"\n // Hopscotch Tour\n var tour = {\n id: \"hopscotch_tour\",\n steps: [\n \"\"\")\n\n self._tour_steps[name] = []\n self._tour_steps[name].append(new_tour)", "response": "Creates a Hopscotch tour for a website."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef create_introjs_tour(self, name=None):\n if not name:\n name = \"default\"\n\n new_tour = (\n \"\"\"\n // IntroJS Tour\n function startIntro(){\n var intro = introJs();\n intro.setOptions({\n steps: [\n \"\"\")\n\n self._tour_steps[name] = []\n self._tour_steps[name].append(new_tour)", "response": "Creates an IntroJS tour for a website."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nadding a new tour step to the page.", "response": "def add_tour_step(self, message, selector=None, name=None,\n title=None, theme=None, alignment=None, duration=None):\n \"\"\" Allows the user to add tour steps for a website.\n @Params\n message - The message to display.\n selector - The CSS Selector of the Element to attach to.\n name - If creating multiple tours at the same time,\n use this to select the tour you wish to add steps to.\n title - Additional header text that appears above the message.\n theme - (NON-Bootstrap Tours ONLY) The styling of the tour step.\n Choose from \"light\"/\"arrows\", \"dark\", \"default\", \"square\",\n and \"square-dark\". (\"arrows\" is used if None is selected.)\n alignment - Choose from \"top\", \"bottom\", \"left\", and \"right\".\n (\"top\" is the default alignment).\n duration - (Bootstrap Tours ONLY) The amount of time, in seconds,\n before automatically advancing to the next tour step.\n \"\"\"\n if not selector:\n selector = \"html\"\n if page_utils.is_xpath_selector(selector):\n selector = self.convert_to_css_selector(selector, By.XPATH)\n selector = self.__escape_quotes_if_needed(selector)\n\n if not name:\n name = \"default\"\n if name not in self._tour_steps:\n # By default, will create an IntroJS tour if no tours exist\n self.create_tour(name=name, theme=\"introjs\")\n\n if not title:\n title = \"\"\n title = self.__escape_quotes_if_needed(title)\n\n if message:\n message = self.__escape_quotes_if_needed(message)\n else:\n message = \"\"\n\n if not alignment or (\n alignment not in [\"top\", \"bottom\", \"left\", \"right\"]):\n if \"Hopscotch\" not in self._tour_steps[name][0]:\n alignment = \"top\"\n else:\n alignment = \"bottom\"\n\n if \"Bootstrap\" in self._tour_steps[name][0]:\n self.__add_bootstrap_tour_step(\n message, selector=selector, name=name, title=title,\n alignment=alignment, duration=duration)\n elif \"Hopscotch\" in self._tour_steps[name][0]:\n self.__add_hopscotch_tour_step(\n message, selector=selector, name=name, title=title,\n alignment=alignment)\n elif \"IntroJS\" in self._tour_steps[name][0]:\n self.__add_introjs_tour_step(\n message, selector=selector, name=name, title=title,\n alignment=alignment)\n else:\n self.__add_shepherd_tour_step(\n message, selector=selector, name=name, title=title,\n theme=theme, alignment=alignment)"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef __add_shepherd_tour_step(self, message, selector=None, name=None,\n title=None, theme=None, alignment=None):\n \"\"\" Allows the user to add tour steps for a website.\n @Params\n message - The message to display.\n selector - The CSS Selector of the Element to attach to.\n name - If creating multiple tours at the same time,\n use this to select the tour you wish to add steps to.\n title - Additional header text that appears above the message.\n theme - (NON-Bootstrap Tours ONLY) The styling of the tour step.\n Choose from \"light\"/\"arrows\", \"dark\", \"default\", \"square\",\n and \"square-dark\". (\"arrows\" is used if None is selected.)\n alignment - Choose from \"top\", \"bottom\", \"left\", and \"right\".\n (\"top\" is the default alignment).\n \"\"\"\n if theme == \"default\":\n shepherd_theme = \"shepherd-theme-default\"\n elif theme == \"dark\":\n shepherd_theme = \"shepherd-theme-dark\"\n elif theme == \"light\":\n shepherd_theme = \"shepherd-theme-arrows\"\n elif theme == \"arrows\":\n shepherd_theme = \"shepherd-theme-arrows\"\n elif theme == \"square\":\n shepherd_theme = \"shepherd-theme-square\"\n elif theme == \"square-dark\":\n shepherd_theme = \"shepherd-theme-square-dark\"\n else:\n shepherd_base_theme = re.search(\n r\"[\\S\\s]+classes: '([\\S\\s]+)',[\\S\\s]+\",\n self._tour_steps[name][0]).group(1)\n shepherd_theme = shepherd_base_theme\n\n shepherd_classes = shepherd_theme\n if selector == \"html\":\n shepherd_classes += \" shepherd-orphan\"\n buttons = \"firstStepButtons\"\n if len(self._tour_steps[name]) > 1:\n buttons = \"midTourButtons\"\n\n step = (\"\"\"\n tour.addStep('%s', {\n title: '%s',\n classes: '%s',\n text: '%s',\n attachTo: {element: '%s', on: '%s'},\n buttons: %s,\n advanceOn: '.docs-link click'\n });\"\"\" % (\n name, title, shepherd_classes, message, selector, alignment,\n buttons))\n\n self._tour_steps[name].append(step)", "response": "Adds a new tour step to the website."} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nadd a bootstrap tour step to the list of available tours.", "response": "def __add_bootstrap_tour_step(self, message, selector=None, name=None,\n title=None, alignment=None, duration=None):\n \"\"\" Allows the user to add tour steps for a website.\n @Params\n message - The message to display.\n selector - The CSS Selector of the Element to attach to.\n name - If creating multiple tours at the same time,\n use this to select the tour you wish to add steps to.\n title - Additional header text that appears above the message.\n alignment - Choose from \"top\", \"bottom\", \"left\", and \"right\".\n (\"top\" is the default alignment).\n duration - (Bootstrap Tours ONLY) The amount of time, in seconds,\n before automatically advancing to the next tour step.\n \"\"\"\n if selector != \"html\":\n selector = self.__make_css_match_first_element_only(selector)\n element_row = \"element: '%s',\" % selector\n else:\n element_row = \"\"\n if not duration:\n duration = \"0\"\n else:\n duration = str(float(duration) * 1000.0)\n\n step = (\"\"\"{\n %s\n title: '%s',\n content: '%s',\n orphan: true,\n placement: 'auto %s',\n smartPlacement: true,\n duration: %s,\n },\"\"\" % (element_row, title, message, alignment, duration))\n\n self._tour_steps[name].append(step)"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef __add_hopscotch_tour_step(self, message, selector=None, name=None,\n title=None, alignment=None):\n \"\"\" Allows the user to add tour steps for a website.\n @Params\n message - The message to display.\n selector - The CSS Selector of the Element to attach to.\n name - If creating multiple tours at the same time,\n use this to select the tour you wish to add steps to.\n title - Additional header text that appears above the message.\n alignment - Choose from \"top\", \"bottom\", \"left\", and \"right\".\n (\"bottom\" is the default alignment).\n \"\"\"\n arrow_offset_row = None\n if not selector or selector == \"html\":\n selector = \"head\"\n alignment = \"bottom\"\n arrow_offset_row = \"arrowOffset: '200',\"\n else:\n arrow_offset_row = \"\"\n\n step = (\"\"\"{\n target: '%s',\n title: '%s',\n content: '%s',\n %s\n showPrevButton: 'true',\n scrollDuration: '550',\n placement: '%s'},\n \"\"\" % (selector, title, message, arrow_offset_row, alignment))\n\n self._tour_steps[name].append(step)", "response": "Adds a tour step to the list of available tours."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef __add_introjs_tour_step(self, message, selector=None, name=None,\n title=None, alignment=None):\n \"\"\" Allows the user to add tour steps for a website.\n @Params\n message - The message to display.\n selector - The CSS Selector of the Element to attach to.\n name - If creating multiple tours at the same time,\n use this to select the tour you wish to add steps to.\n title - Additional header text that appears above the message.\n alignment - Choose from \"top\", \"bottom\", \"left\", and \"right\".\n (\"top\" is the default alignment).\n \"\"\"\n if selector != \"html\":\n element_row = \"element: '%s',\" % selector\n else:\n element_row = \"\"\n\n if title:\n message = \"
\" + title + \"
\" + message\n\n message = '' + message + ''\n\n step = (\"\"\"{%s\n intro: '%s',\n position: '%s'},\n \"\"\" % (element_row, message, alignment))\n\n self._tour_steps[name].append(step)", "response": "Adds a tour step to the list of introjs."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef play_tour(self, name=None, interval=0):\n if self.headless:\n return # Tours should not run in headless mode.\n\n if not name:\n name = \"default\"\n if name not in self._tour_steps:\n raise Exception(\"Tour {%s} does not exist!\" % name)\n\n if \"Bootstrap\" in self._tour_steps[name][0]:\n tour_helper.play_bootstrap_tour(\n self.driver, self._tour_steps, self.browser,\n self.message_duration, name=name, interval=interval)\n elif \"Hopscotch\" in self._tour_steps[name][0]:\n tour_helper.play_hopscotch_tour(\n self.driver, self._tour_steps, self.browser,\n self.message_duration, name=name, interval=interval)\n elif \"IntroJS\" in self._tour_steps[name][0]:\n tour_helper.play_introjs_tour(\n self.driver, self._tour_steps, self.browser,\n self.message_duration, name=name, interval=interval)\n else:\n # \"Shepherd\"\n tour_helper.play_shepherd_tour(\n self.driver, self._tour_steps,\n self.message_duration, name=name, interval=interval)", "response": "Plays a single tour on the current website."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nexporting a tour as a JavaScript file.", "response": "def export_tour(self, name=None, filename=\"my_tour.js\", url=None):\n \"\"\" Exports a tour as a JS file.\n You can call self.export_tour() anywhere where you would\n normally use self.play_tour() to play a tour.\n It will include necessary resources as well, such as jQuery.\n You'll be able to copy the tour directly into the Console of\n any web browser to play the tour outside of SeleniumBase runs.\n @Params\n name - If creating multiple tours at the same time,\n use this to select the tour you wish to add steps to.\n filename - The name of the JavaScript file that you wish to\n save the tour to. \"\"\"\n if not url:\n url = self.get_current_url()\n tour_helper.export_tour(\n self._tour_steps, name=name, filename=filename, url=url)"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef set_messenger_theme(self, theme=\"default\", location=\"default\",\n max_messages=\"default\"):\n \"\"\" Sets a theme for posting messages.\n Themes: [\"flat\", \"future\", \"block\", \"air\", \"ice\"]\n Locations: [\"top_left\", \"top_center\", \"top_right\",\n \"bottom_left\", \"bottom_center\", \"bottom_right\"]\n max_messages is the limit of concurrent messages to display. \"\"\"\n if not theme:\n theme = \"default\" # \"future\"\n if not location:\n location = \"default\" # \"bottom_right\"\n if not max_messages:\n max_messages = \"default\" # \"8\"\n js_utils.set_messenger_theme(\n self.driver, theme=theme,\n location=location, max_messages=max_messages)", "response": "Sets a theme for posting messages."} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nposts a message on the screen with Messenger.", "response": "def post_message(self, message, duration=None, pause=True, style=\"info\"):\n \"\"\" Post a message on the screen with Messenger.\n Arguments:\n message: The message to display.\n duration: The time until the message vanishes. (Default: 2.55s)\n pause: If True, the program waits until the message completes.\n style: \"info\", \"success\", or \"error\".\n\n You can also post messages by using =>\n self.execute_script('Messenger().post(\"My Message\")')\n \"\"\"\n if not duration:\n if not self.message_duration:\n duration = settings.DEFAULT_MESSAGE_DURATION\n else:\n duration = self.message_duration\n js_utils.post_message(\n self.driver, message, duration, style=style)\n if pause:\n duration = float(duration) + 0.15\n time.sleep(float(duration))"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nbringing the element to the front of the page.", "response": "def bring_to_front(self, selector, by=By.CSS_SELECTOR):\n \"\"\" Updates the Z-index of a page element to bring it into view.\n Useful when getting a WebDriverException, such as the one below:\n { Element is not clickable at point (#, #).\n Other element would receive the click: ... } \"\"\"\n if page_utils.is_xpath_selector(selector):\n by = By.XPATH\n self.find_element(selector, by=by, timeout=settings.SMALL_TIMEOUT)\n try:\n selector = self.convert_to_css_selector(selector, by=by)\n except Exception:\n # Don't run action if can't convert to CSS_Selector for JavaScript\n return\n selector = re.escape(selector)\n selector = self.__escape_quotes_if_needed(selector)\n script = (\"\"\"document.querySelector('%s').style.zIndex = '9999';\"\"\"\n % selector)\n self.execute_script(script)"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef highlight(self, selector, by=By.CSS_SELECTOR,\n loops=settings.HIGHLIGHTS, scroll=True):\n \"\"\" This method uses fancy JavaScript to highlight an element.\n Used during demo_mode.\n @Params\n selector - the selector of the element to find\n by - the type of selector to search by (Default: CSS)\n loops - # of times to repeat the highlight animation\n (Default: 4. Each loop lasts for about 0.18s)\n scroll - the option to scroll to the element first (Default: True)\n \"\"\"\n selector, by = self.__recalculate_selector(selector, by)\n element = self.find_element(\n selector, by=by, timeout=settings.SMALL_TIMEOUT)\n if scroll:\n self.__slow_scroll_to_element(element)\n try:\n selector = self.convert_to_css_selector(selector, by=by)\n except Exception:\n # Don't highlight if can't convert to CSS_SELECTOR\n return\n\n if self.highlights:\n loops = self.highlights\n if self.browser == 'ie':\n loops = 1 # Override previous setting because IE is slow\n loops = int(loops)\n\n o_bs = '' # original_box_shadow\n style = element.get_attribute('style')\n if style:\n if 'box-shadow: ' in style:\n box_start = style.find('box-shadow: ')\n box_end = style.find(';', box_start) + 1\n original_box_shadow = style[box_start:box_end]\n o_bs = original_box_shadow\n\n if \":contains\" not in selector and \":first\" not in selector:\n selector = re.escape(selector)\n selector = self.__escape_quotes_if_needed(selector)\n self.__highlight_with_js(selector, loops, o_bs)\n else:\n selector = self.__make_css_match_first_element_only(selector)\n selector = re.escape(selector)\n selector = self.__escape_quotes_if_needed(selector)\n try:\n self.__highlight_with_jquery(selector, loops, o_bs)\n except Exception:\n pass # JQuery probably couldn't load. Skip highlighting.\n time.sleep(0.065)", "response": "This method uses fancy JavaScript to highlight an element."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nfast scroll to destination", "response": "def scroll_to(self, selector, by=By.CSS_SELECTOR,\n timeout=settings.SMALL_TIMEOUT):\n ''' Fast scroll to destination '''\n if self.demo_mode:\n self.slow_scroll_to(selector, by=by, timeout=timeout)\n return\n if self.timeout_multiplier and timeout == settings.SMALL_TIMEOUT:\n timeout = self.__get_new_timeout(timeout)\n element = self.wait_for_element_visible(\n selector, by=by, timeout=timeout)\n try:\n self.__scroll_to_element(element)\n except (StaleElementReferenceException, ENI_Exception):\n self.wait_for_ready_state_complete()\n time.sleep(0.05)\n element = self.wait_for_element_visible(\n selector, by=by, timeout=timeout)\n self.__scroll_to_element(element)"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nslow motion scroll to destination", "response": "def slow_scroll_to(self, selector, by=By.CSS_SELECTOR,\n timeout=settings.SMALL_TIMEOUT):\n ''' Slow motion scroll to destination '''\n if self.timeout_multiplier and timeout == settings.SMALL_TIMEOUT:\n timeout = self.__get_new_timeout(timeout)\n element = self.wait_for_element_visible(\n selector, by=by, timeout=timeout)\n self.__slow_scroll_to_element(element)"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nclicking an element using pure JavaScript. Does not use jQuery.", "response": "def js_click(self, selector, by=By.CSS_SELECTOR):\n \"\"\" Clicks an element using pure JS. Does not use jQuery. \"\"\"\n selector, by = self.__recalculate_selector(selector, by)\n if by == By.LINK_TEXT:\n message = (\n \"Pure JavaScript doesn't support clicking by Link Text. \"\n \"You may want to use self.jquery_click() instead, which \"\n \"allows this with :contains(), assuming jQuery isn't blocked. \"\n \"For now, self.js_click() will use a regular WebDriver click.\")\n logging.debug(message)\n self.click(selector, by=by)\n return\n element = self.wait_for_element_present(\n selector, by=by, timeout=settings.SMALL_TIMEOUT)\n if self.is_element_visible(selector, by=by):\n self.__demo_mode_highlight_if_active(selector, by)\n if not self.demo_mode:\n self.__scroll_to_element(element)\n css_selector = self.convert_to_css_selector(selector, by=by)\n css_selector = re.escape(css_selector)\n css_selector = self.__escape_quotes_if_needed(css_selector)\n self.__js_click(selector, by=by) # The real \"magic\" happens here\n self.__demo_mode_pause_if_active()"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nclick an element using jQuery. Different from pure JS. Different from using pure JS.", "response": "def jquery_click(self, selector, by=By.CSS_SELECTOR):\n \"\"\" Clicks an element using jQuery. Different from using pure JS. \"\"\"\n selector, by = self.__recalculate_selector(selector, by)\n self.wait_for_element_present(\n selector, by=by, timeout=settings.SMALL_TIMEOUT)\n if self.is_element_visible(selector, by=by):\n self.__demo_mode_highlight_if_active(selector, by)\n selector = self.convert_to_css_selector(selector, by=by)\n selector = self.__make_css_match_first_element_only(selector)\n click_script = \"\"\"jQuery('%s')[0].click()\"\"\" % selector\n self.safe_execute_script(click_script)\n self.__demo_mode_pause_if_active()"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef submit(self, selector, by=By.CSS_SELECTOR):\n if page_utils.is_xpath_selector(selector):\n by = By.XPATH\n element = self.wait_for_element_visible(\n selector, by=by, timeout=settings.SMALL_TIMEOUT)\n element.submit()\n self.__demo_mode_pause_if_active()", "response": "Alternative to find_element_by_*[ selector ]. submit"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nhiding the first element on the page that matches the selector.", "response": "def hide_element(self, selector, by=By.CSS_SELECTOR):\n \"\"\" Hide the first element on the page that matches the selector. \"\"\"\n selector, by = self.__recalculate_selector(selector, by)\n selector = self.convert_to_css_selector(selector, by=by)\n selector = self.__make_css_match_first_element_only(selector)\n hide_script = \"\"\"jQuery('%s').hide()\"\"\" % selector\n self.safe_execute_script(hide_script)"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef hide_elements(self, selector, by=By.CSS_SELECTOR):\n selector, by = self.__recalculate_selector(selector, by)\n selector = self.convert_to_css_selector(selector, by=by)\n hide_script = \"\"\"jQuery('%s').hide()\"\"\" % selector\n self.safe_execute_script(hide_script)", "response": "Hide all elements on the page that match the selector."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nshowing the first element on the page that matches the selector.", "response": "def show_element(self, selector, by=By.CSS_SELECTOR):\n \"\"\" Show the first element on the page that matches the selector. \"\"\"\n selector, by = self.__recalculate_selector(selector, by)\n selector = self.convert_to_css_selector(selector, by=by)\n selector = self.__make_css_match_first_element_only(selector)\n show_script = \"\"\"jQuery('%s').show(0)\"\"\" % selector\n self.safe_execute_script(show_script)"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef show_elements(self, selector, by=By.CSS_SELECTOR):\n selector, by = self.__recalculate_selector(selector, by)\n selector = self.convert_to_css_selector(selector, by=by)\n show_script = \"\"\"jQuery('%s').show(0)\"\"\" % selector\n self.safe_execute_script(show_script)", "response": "Show all elements on the page that match the selector."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef remove_element(self, selector, by=By.CSS_SELECTOR):\n selector, by = self.__recalculate_selector(selector, by)\n selector = self.convert_to_css_selector(selector, by=by)\n selector = self.__make_css_match_first_element_only(selector)\n remove_script = \"\"\"jQuery('%s').remove()\"\"\" % selector\n self.safe_execute_script(remove_script)", "response": "Removes the first element on the page that matches the selector."} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nremove all elements on the page that match the selector.", "response": "def remove_elements(self, selector, by=By.CSS_SELECTOR):\n \"\"\" Remove all elements on the page that match the selector. \"\"\"\n selector, by = self.__recalculate_selector(selector, by)\n selector = self.convert_to_css_selector(selector, by=by)\n remove_script = \"\"\"jQuery('%s').remove()\"\"\" % selector\n self.safe_execute_script(remove_script)"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_beautiful_soup(self, source=None):\n from bs4 import BeautifulSoup\n if not source:\n self.wait_for_ready_state_complete()\n source = self.get_page_source()\n soup = BeautifulSoup(source, \"html.parser\")\n return soup", "response": "Returns a BeautifulSoup object for the given HTML document."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_unique_links(self):\n page_url = self.get_current_url()\n soup = self.get_beautiful_soup(self.get_page_source())\n links = page_utils._get_unique_links(page_url, soup)\n return links", "response": "Get all unique links in the html of the current page."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_link_status_code(self, link, allow_redirects=False, timeout=5):\n status_code = page_utils._get_link_status_code(\n link, allow_redirects=allow_redirects, timeout=timeout)\n return status_code", "response": "Get the status code of a link."} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nasserting no 404 errors from page links obtained from the page links obtained from the page links and script links.", "response": "def assert_no_404_errors(self, multithreaded=True):\n \"\"\" Assert no 404 errors from page links obtained from:\n \"a\"->\"href\", \"img\"->\"src\", \"link\"->\"href\", and \"script\"->\"src\". \"\"\"\n links = self.get_unique_links()\n if multithreaded:\n from multiprocessing.dummy import Pool as ThreadPool\n pool = ThreadPool(10)\n pool.map(self.assert_link_status_code_is_not_404, links)\n pool.close()\n pool.join()\n else:\n for link in links:\n self.assert_link_status_code_is_not_404(link)"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef print_unique_links_with_status_codes(self):\n page_url = self.get_current_url()\n soup = self.get_beautiful_soup(self.get_page_source())\n page_utils._print_unique_links_with_status_codes(page_url, soup)", "response": "Print out all unique links in the html of the page source\n and all status codes."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef safe_execute_script(self, script):\n try:\n self.execute_script(script)\n except Exception:\n # The likely reason this fails is because: \"jQuery is not defined\"\n self.activate_jquery() # It's a good thing we can define it here\n self.execute_script(script)", "response": "This method will execute a script that contains a jQuery command."} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncreates a folder of the given name if it doesn t already exist.", "response": "def create_folder(self, folder):\n \"\"\" Creates a folder of the given name if it doesn't already exist. \"\"\"\n if folder.endswith(\"/\"):\n folder = folder[:-1]\n if len(folder) < 1:\n raise Exception(\"Minimum folder name length = 1.\")\n if not os.path.exists(folder):\n try:\n os.makedirs(folder)\n except Exception:\n pass"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef save_element_as_image_file(self, selector, file_name, folder=None):\n element = self.find_element(selector)\n element_png = element.screenshot_as_png\n if len(file_name.split('.')[0]) < 1:\n raise Exception(\"Error: file_name length must be > 0.\")\n if not file_name.endswith(\".png\"):\n file_name = file_name + \".png\"\n image_file_path = None\n if folder:\n if folder.endswith(\"/\"):\n folder = folder[:-1]\n if len(folder) > 0:\n self.create_folder(folder)\n image_file_path = \"%s/%s\" % (folder, file_name)\n if not image_file_path:\n image_file_path = file_name\n with open(image_file_path, \"wb\") as file:\n file.write(element_png)", "response": "Take a screenshot of an element and save it as an image file."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef download_file(self, file_url, destination_folder=None):\n if not destination_folder:\n destination_folder = constants.Files.DOWNLOADS_FOLDER\n page_utils._download_file_to(file_url, destination_folder)", "response": "Downloads the file from the url to the destination folder."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef save_file_as(self, file_url, new_file_name, destination_folder=None):\n if not destination_folder:\n destination_folder = constants.Files.DOWNLOADS_FOLDER\n page_utils._download_file_to(\n file_url, destination_folder, new_file_name)", "response": "Downloads the file from the URL and saves it to the new file name."} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nsaves the data specified to a file of the specified name.", "response": "def save_data_as(self, data, file_name, destination_folder=None):\n \"\"\" Saves the data specified to a file of the name specified.\n If no destination folder is specified, the default one is used.\n (The default downloads folder = \"./downloaded_files\") \"\"\"\n if not destination_folder:\n destination_folder = constants.Files.DOWNLOADS_FOLDER\n page_utils._save_data_as(data, destination_folder, file_name)"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nasserting that there are no JavaScript errors.", "response": "def assert_no_js_errors(self):\n \"\"\" Asserts that there are no JavaScript \"SEVERE\"-level page errors.\n Works ONLY for Chrome (non-headless) and Chrome-based browsers.\n Does NOT work on Firefox, Edge, IE, and some other browsers:\n * See https://github.com/SeleniumHQ/selenium/issues/1161\n Based on the following Stack Overflow solution:\n * https://stackoverflow.com/a/41150512/7058266 \"\"\"\n try:\n browser_logs = self.driver.get_log('browser')\n except (ValueError, WebDriverException):\n # If unable to get browser logs, skip the assert and return.\n return\n\n messenger_library = \"//cdnjs.cloudflare.com/ajax/libs/messenger\"\n errors = []\n for entry in browser_logs:\n if entry['level'] == 'SEVERE':\n if messenger_library not in entry['message']:\n # Add errors if not caused by SeleniumBase dependencies\n errors.append(entry)\n if len(errors) > 0:\n current_url = self.get_current_url()\n raise Exception(\n \"JavaScript errors found on %s => %s\" % (current_url, errors))"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get_google_auth_password(self, totp_key=None):\n import pyotp\n if not totp_key:\n totp_key = settings.TOTP_KEY\n\n epoch_interval = time.time() / 30.0\n cycle_lifespan = float(epoch_interval) - int(epoch_interval)\n if float(cycle_lifespan) > 0.95:\n # Password expires in the next 1.5 seconds. Wait for a new one.\n for i in range(30):\n time.sleep(0.05)\n epoch_interval = time.time() / 30.0\n cycle_lifespan = float(epoch_interval) - int(epoch_interval)\n if not float(cycle_lifespan) > 0.95:\n # The new password cycle has begun\n break\n\n totp = pyotp.TOTP(totp_key)\n return str(totp.now())", "response": "Returns a time - based one - time password based on the Google Authenticator password algorithm."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef set_value(self, selector, new_value, by=By.CSS_SELECTOR,\n timeout=settings.LARGE_TIMEOUT):\n \"\"\" This method uses JavaScript to update a text field. \"\"\"\n if self.timeout_multiplier and timeout == settings.LARGE_TIMEOUT:\n timeout = self.__get_new_timeout(timeout)\n if page_utils.is_xpath_selector(selector):\n by = By.XPATH\n orginal_selector = selector\n css_selector = self.convert_to_css_selector(selector, by=by)\n self.__demo_mode_highlight_if_active(orginal_selector, by)\n if not self.demo_mode:\n self.scroll_to(orginal_selector, by=by, timeout=timeout)\n value = re.escape(new_value)\n value = self.__escape_quotes_if_needed(value)\n css_selector = re.escape(css_selector)\n css_selector = self.__escape_quotes_if_needed(css_selector)\n script = (\"\"\"document.querySelector('%s').value='%s';\"\"\"\n % (css_selector, value))\n self.execute_script(script)\n if new_value.endswith('\\n'):\n element = self.wait_for_element_present(\n orginal_selector, by=by, timeout=timeout)\n element.send_keys(Keys.RETURN)\n if settings.WAIT_FOR_RSC_ON_PAGE_LOADS:\n self.wait_for_ready_state_complete()\n self.__demo_mode_pause_if_active()", "response": "This method updates the value of the element identified by the selector."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef js_update_text(self, selector, new_value, by=By.CSS_SELECTOR,\n timeout=settings.LARGE_TIMEOUT):\n \"\"\" Same as self.set_value() \"\"\"\n if self.timeout_multiplier and timeout == settings.LARGE_TIMEOUT:\n timeout = self.__get_new_timeout(timeout)\n self.set_value(\n selector, new_value, by=by, timeout=timeout)", "response": "Updates the value of the js element."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef jquery_update_text(self, selector, new_value, by=By.CSS_SELECTOR,\n timeout=settings.LARGE_TIMEOUT):\n \"\"\" The shorter version of self.jquery_update_text_value()\n (The longer version remains for backwards compatibility.) \"\"\"\n if self.timeout_multiplier and timeout == settings.LARGE_TIMEOUT:\n timeout = self.__get_new_timeout(timeout)\n self.jquery_update_text_value(\n selector, new_value, by=by, timeout=timeout)", "response": "Updates the value of the element in the text field with the given selector."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef __select_option(self, dropdown_selector, option,\n dropdown_by=By.CSS_SELECTOR, option_by=\"text\",\n timeout=settings.SMALL_TIMEOUT):\n \"\"\" Selects an HTML option by text.", "response": "def select_option_by_text(self, dropdown_selector, option,\n dropdown_by=By.CSS_SELECTOR,\n timeout=settings.SMALL_TIMEOUT):\n \"\"\" Selects an HTML selector\n option - the text of the option \"\"\"\n if self.timeout_multiplier and timeout == settings.SMALL_TIMEOUT:\n timeout = self.__get_new_timeout(timeout)\n self.__select_option(dropdown_selector, option,\n dropdown_by=dropdown_by, option_by=\"text\",\n timeout=timeout)"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ngenerating traffic from start_page to destination_page.", "response": "def generate_traffic(self, start_page, destination_page, loops=1):\n \"\"\" Similar to generate_referral(), but can do multiple loops. \"\"\"\n for loop in range(loops):\n self.generate_referral(start_page, destination_page)\n time.sleep(0.05)"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef generate_referral_chain(self, pages):\n if not type(pages) is tuple and not type(pages) is list:\n raise Exception(\n \"Exception: Expecting a list of website pages for chaining!\")\n if len(pages) < 2:\n raise Exception(\n \"Exception: At least two website pages required for chaining!\")\n for page in pages:\n # Find out if any of the web pages are invalid before continuing\n if not page_utils.is_valid_url(page):\n raise Exception(\n \"Exception: Website page {%s} is not a valid URL!\" % page)\n for page in pages:\n self.generate_referral(None, page)", "response": "This method generates the referral chain for the given list of website pages."} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ngenerates the traffic chain for a set of pages.", "response": "def generate_traffic_chain(self, pages, loops=1):\n \"\"\" Similar to generate_referral_chain(), but for multiple loops. \"\"\"\n for loop in range(loops):\n self.generate_referral_chain(pages)\n time.sleep(0.05)"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nasserting that an element is present in the HTML of a page.", "response": "def assert_element_present(self, selector, by=By.CSS_SELECTOR,\n timeout=settings.SMALL_TIMEOUT):\n \"\"\" Similar to wait_for_element_present(), but returns nothing.\n Waits for an element to appear in the HTML of a page.\n The element does not need be visible (it may be hidden).\n Returns True if successful. Default timeout = SMALL_TIMEOUT. \"\"\"\n if self.timeout_multiplier and timeout == settings.SMALL_TIMEOUT:\n timeout = self.__get_new_timeout(timeout)\n self.wait_for_element_present(selector, by=by, timeout=timeout)\n return True"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef wait_for_element_visible(self, selector, by=By.CSS_SELECTOR,\n timeout=settings.LARGE_TIMEOUT):\n \"\"\" Waits for an element to appear in the HTML of a page.\n The element must be visible (it cannot be hidden). \"\"\"\n if page_utils.is_xpath_selector(selector):\n by = By.XPATH\n if page_utils.is_link_text_selector(selector):\n selector = page_utils.get_link_text_from_selector(selector)\n by = By.LINK_TEXT\n return page_actions.wait_for_element_visible(\n self.driver, selector, by, timeout)", "response": "Waits for an element to appear in the HTML of a page."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef wait_for_element(self, selector, by=By.CSS_SELECTOR,\n timeout=settings.LARGE_TIMEOUT):\n \"\"\" The shorter version of wait_for_element_visible() \"\"\"\n if self.timeout_multiplier and timeout == settings.LARGE_TIMEOUT:\n timeout = self.__get_new_timeout(timeout)\n return self.wait_for_element_visible(selector, by=by, timeout=timeout)", "response": "This method waits for an element to be visible."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef assert_element_visible(self, selector, by=By.CSS_SELECTOR,\n timeout=settings.SMALL_TIMEOUT):\n \"\"\" Same as self.assert_element()\n As above, will raise an exception if nothing can be found. \"\"\"\n if self.timeout_multiplier and timeout == settings.SMALL_TIMEOUT:\n timeout = self.__get_new_timeout(timeout)\n self.assert_element(selector, by=by, timeout=timeout)\n return True", "response": "Same as assert_element but will raise an exception if the element is not visible."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef assert_text_visible(self, text, selector=\"html\", by=By.CSS_SELECTOR,\n timeout=settings.SMALL_TIMEOUT):\n \"\"\" Same as assert_text() \"\"\"\n if self.timeout_multiplier and timeout == settings.SMALL_TIMEOUT:\n timeout = self.__get_new_timeout(timeout)\n return self.assert_text(text, selector, by=by, timeout=timeout)", "response": "Same as assert_text but returns True if the text is visible."} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nasserts that the text is exactly the given one.", "response": "def assert_exact_text(self, text, selector=\"html\", by=By.CSS_SELECTOR,\n timeout=settings.SMALL_TIMEOUT):\n \"\"\" Similar to assert_text(), but the text must be exact, rather than\n exist as a subset of the full text.\n (Extra whitespace at the beginning or the end doesn't count.)\n Raises an exception if the element or the text is not found.\n Returns True if successful. Default timeout = SMALL_TIMEOUT. \"\"\"\n if self.timeout_multiplier and timeout == settings.SMALL_TIMEOUT:\n timeout = self.__get_new_timeout(timeout)\n self.wait_for_exact_text_visible(\n text, selector, by=by, timeout=timeout)\n\n if self.demo_mode:\n if page_utils.is_xpath_selector(selector):\n by = By.XPATH\n if page_utils.is_link_text_selector(selector):\n selector = page_utils.get_link_text_from_selector(selector)\n by = By.LINK_TEXT\n messenger_post = (\"ASSERT TEXT {%s} in %s: %s\"\n % (text, by, selector))\n self.__highlight_with_assert_success(messenger_post, selector, by)\n return True"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef wait_for_link_text(self, link_text, timeout=settings.LARGE_TIMEOUT):\n if self.timeout_multiplier and timeout == settings.LARGE_TIMEOUT:\n timeout = self.__get_new_timeout(timeout)\n return self.wait_for_link_text_visible(link_text, timeout=timeout)", "response": "Wait for a link text to be visible."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef assert_link_text(self, link_text, timeout=settings.SMALL_TIMEOUT):\n if self.timeout_multiplier and timeout == settings.SMALL_TIMEOUT:\n timeout = self.__get_new_timeout(timeout)\n self.wait_for_link_text_visible(link_text, timeout=timeout)\n if self.demo_mode:\n messenger_post = (\"ASSERT LINK TEXT {%s}.\" % link_text)\n self.__highlight_with_assert_success(\n messenger_post, link_text, by=By.LINK_TEXT)\n return True", "response": "Assert that the given link text is visible."} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns the element that contains the given partial link text.", "response": "def find_partial_link_text(self, partial_link_text,\n timeout=settings.LARGE_TIMEOUT):\n \"\"\" Same as wait_for_partial_link_text() - returns the element \"\"\"\n if self.timeout_multiplier and timeout == settings.LARGE_TIMEOUT:\n timeout = self.__get_new_timeout(timeout)\n return self.wait_for_partial_link_text(\n partial_link_text, timeout=timeout)"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef assert_partial_link_text(self, partial_link_text,\n timeout=settings.SMALL_TIMEOUT):\n \"\"\" Similar to wait_for_partial_link_text(), but returns nothing.\n As above, will raise an exception if nothing can be found.\n Returns True if successful. Default timeout = SMALL_TIMEOUT. \"\"\"\n if self.timeout_multiplier and timeout == settings.SMALL_TIMEOUT:\n timeout = self.__get_new_timeout(timeout)\n self.wait_for_partial_link_text(partial_link_text, timeout=timeout)\n return True", "response": "Assert that partial_link_text is found in the resource table."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nwaiting for an element to be absent in the HTML of a page.", "response": "def wait_for_element_absent(self, selector, by=By.CSS_SELECTOR,\n timeout=settings.LARGE_TIMEOUT):\n \"\"\" Waits for an element to no longer appear in the HTML of a page.\n A hidden element still counts as appearing in the page HTML.\n If an element with \"hidden\" status is acceptable,\n use wait_for_element_not_visible() instead. \"\"\"\n if self.timeout_multiplier and timeout == settings.LARGE_TIMEOUT:\n timeout = self.__get_new_timeout(timeout)\n if page_utils.is_xpath_selector(selector):\n by = By.XPATH\n return page_actions.wait_for_element_absent(\n self.driver, selector, by, timeout)"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef assert_element_absent(self, selector, by=By.CSS_SELECTOR,\n timeout=settings.SMALL_TIMEOUT):\n \"\"\" Similar to wait_for_element_absent() - returns nothing.\n As above, will raise an exception if the element stays present.\n Returns True if successful. Default timeout = SMALL_TIMEOUT. \"\"\"\n if self.timeout_multiplier and timeout == settings.SMALL_TIMEOUT:\n timeout = self.__get_new_timeout(timeout)\n self.wait_for_element_absent(selector, by=by, timeout=timeout)\n return True", "response": "Assert that the element is absent."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef assert_element_not_visible(self, selector, by=By.CSS_SELECTOR,\n timeout=settings.SMALL_TIMEOUT):\n \"\"\" Similar to wait_for_element_not_visible() - returns nothing.\n As above, will raise an exception if the element stays visible.\n Returns True if successful. Default timeout = SMALL_TIMEOUT. \"\"\"\n if self.timeout_multiplier and timeout == settings.SMALL_TIMEOUT:\n timeout = self.__get_new_timeout(timeout)\n self.wait_for_element_not_visible(selector, by=by, timeout=timeout)\n return True", "response": "Assert that the element is not visible."} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nswitch the driver control to the specified browser frame.", "response": "def switch_to_frame(self, frame, timeout=settings.SMALL_TIMEOUT):\n \"\"\" Sets driver control to the specified browser frame. \"\"\"\n if self.timeout_multiplier and timeout == settings.SMALL_TIMEOUT:\n timeout = self.__get_new_timeout(timeout)\n page_actions.switch_to_frame(self.driver, frame, timeout)"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nopening a new browser tab or window and switches to it by default.", "response": "def open_new_window(self, switch_to=True):\n \"\"\" Opens a new browser tab/window and switches to it by default. \"\"\"\n self.driver.execute_script(\"window.open('');\")\n time.sleep(0.01)\n if switch_to:\n self.switch_to_window(len(self.driver.window_handles) - 1)"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef check_window(self, name=\"default\", level=0, baseline=False):\n if level == \"0\":\n level = 0\n if level == \"1\":\n level = 1\n if level == \"2\":\n level = 2\n if level == \"3\":\n level = 3\n if level != 0 and level != 1 and level != 2 and level != 3:\n raise Exception('Parameter \"level\" must be set to 0, 1, 2, or 3!')\n\n module = self.__class__.__module__\n if '.' in module and len(module.split('.')[-1]) > 1:\n module = module.split('.')[-1]\n test_id = \"%s.%s\" % (module, self._testMethodName)\n if not name or len(name) < 1:\n name = \"default\"\n name = str(name)\n visual_helper.visual_baseline_folder_setup()\n baseline_dir = constants.VisualBaseline.STORAGE_FOLDER\n visual_baseline_path = baseline_dir + \"/\" + test_id + \"/\" + name\n page_url_file = visual_baseline_path + \"/page_url.txt\"\n screenshot_file = visual_baseline_path + \"/screenshot.png\"\n level_1_file = visual_baseline_path + \"/tags_level_1.txt\"\n level_2_file = visual_baseline_path + \"/tags_level_2.txt\"\n level_3_file = visual_baseline_path + \"/tags_level_3.txt\"\n\n set_baseline = False\n if baseline or self.visual_baseline:\n set_baseline = True\n if not os.path.exists(visual_baseline_path):\n set_baseline = True\n try:\n os.makedirs(visual_baseline_path)\n except Exception:\n pass # Only reachable during multi-threaded test runs\n if not os.path.exists(page_url_file):\n set_baseline = True\n if not os.path.exists(screenshot_file):\n set_baseline = True\n if not os.path.exists(level_1_file):\n set_baseline = True\n if not os.path.exists(level_2_file):\n set_baseline = True\n if not os.path.exists(level_3_file):\n set_baseline = True\n\n page_url = self.get_current_url()\n soup = self.get_beautiful_soup()\n html_tags = soup.body.find_all()\n level_1 = [[tag.name] for tag in html_tags]\n level_1 = json.loads(json.dumps(level_1)) # Tuples become lists\n level_2 = [[tag.name, sorted(tag.attrs.keys())] for tag in html_tags]\n level_2 = json.loads(json.dumps(level_2)) # Tuples become lists\n level_3 = [[tag.name, sorted(tag.attrs.items())] for tag in html_tags]\n level_3 = json.loads(json.dumps(level_3)) # Tuples become lists\n\n if set_baseline:\n self.save_screenshot(\"screenshot.png\", visual_baseline_path)\n out_file = codecs.open(page_url_file, \"w+\")\n out_file.writelines(page_url)\n out_file.close()\n out_file = codecs.open(level_1_file, \"w+\")\n out_file.writelines(json.dumps(level_1))\n out_file.close()\n out_file = codecs.open(level_2_file, \"w+\")\n out_file.writelines(json.dumps(level_2))\n out_file.close()\n out_file = codecs.open(level_3_file, \"w+\")\n out_file.writelines(json.dumps(level_3))\n out_file.close()\n\n if not set_baseline:\n f = open(page_url_file, 'r')\n page_url_data = f.read().strip()\n f.close()\n f = open(level_1_file, 'r')\n level_1_data = json.loads(f.read())\n f.close()\n f = open(level_2_file, 'r')\n level_2_data = json.loads(f.read())\n f.close()\n f = open(level_3_file, 'r')\n level_3_data = json.loads(f.read())\n f.close()\n\n domain_fail = (\n \"Page Domain Mismatch Failure: \"\n \"Current Page Domain doesn't match the Page Domain of the \"\n \"Baseline! Can't compare two completely different sites! \"\n \"Run with --visual_baseline to reset the baseline!\")\n level_1_failure = (\n \"\\n\\n*** Exception: Visual Diff Failure:\\n\"\n \"* HTML tags don't match the baseline!\")\n level_2_failure = (\n \"\\n\\n*** Exception: Visual Diff Failure:\\n\"\n \"* HTML tag attributes don't match the baseline!\")\n level_3_failure = (\n \"\\n\\n*** Exception: Visual Diff Failure:\\n\"\n \"* HTML tag attribute values don't match the baseline!\")\n\n page_domain = self.get_domain_url(page_url)\n page_data_domain = self.get_domain_url(page_url_data)\n unittest.TestCase.maxDiff = 1000\n if level == 1 or level == 2 or level == 3:\n self.assert_equal(page_domain, page_data_domain, domain_fail)\n self.assert_equal(level_1, level_1_data, level_1_failure)\n unittest.TestCase.maxDiff = None\n if level == 2 or level == 3:\n self.assert_equal(level_2, level_2_data, level_2_failure)\n if level == 3:\n self.assert_equal(level_3, level_3_data, level_3_failure)\n if level == 0:\n try:\n unittest.TestCase.maxDiff = 1000\n self.assert_equal(\n page_domain, page_data_domain, domain_fail)\n self.assert_equal(level_1, level_1_data, level_1_failure)\n unittest.TestCase.maxDiff = None\n self.assert_equal(level_2, level_2_data, level_2_failure)\n self.assert_equal(level_3, level_3_data, level_3_failure)\n except Exception as e:\n print(e)", "response": "This method is used to check the current window and check if the current window matches the current window."} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nsave screenshot to folder.", "response": "def save_screenshot(self, name, folder=None):\n \"\"\" The screenshot will be in PNG format. \"\"\"\n return page_actions.save_screenshot(self.driver, name, folder)"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef __get_new_timeout(self, timeout):\n try:\n timeout_multiplier = float(self.timeout_multiplier)\n if timeout_multiplier <= 0.5:\n timeout_multiplier = 0.5\n timeout = int(math.ceil(timeout_multiplier * timeout))\n return timeout\n except Exception:\n # Wrong data type for timeout_multiplier (expecting int or float)\n return timeout", "response": "Get new timeout value based on timeout_multiplier"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning the original exception message for an exception.", "response": "def __get_improved_exception_message(self):\n \"\"\"\n If Chromedriver is out-of-date, make it clear!\n Given the high popularity of the following StackOverflow article:\n https://stackoverflow.com/questions/49162667/unknown-error-\n call-function-result-missing-value-for-selenium-send-keys-even\n ... the original error message was not helpful. Tell people directly.\n (Only expected when using driver.send_keys() with an old Chromedriver.)\n \"\"\"\n exc_message = self.__get_exception_message()\n maybe_using_old_chromedriver = False\n if \"unknown error: call function result missing\" in exc_message:\n maybe_using_old_chromedriver = True\n if self.browser == 'chrome' and maybe_using_old_chromedriver:\n update = (\"Your version of ChromeDriver may be out-of-date! \"\n \"Please go to \"\n \"https://sites.google.com/a/chromium.org/chromedriver/ \"\n \"and download the latest version to your system PATH! \"\n \"Or use: ``seleniumbase install chromedriver`` . \"\n \"Original Exception Message: %s\" % exc_message)\n exc_message = update\n return exc_message"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nadds a delayed_assert failure into the list for future processing.", "response": "def __add_delayed_assert_failure(self):\n \"\"\" Add a delayed_assert failure into a list for future processing. \"\"\"\n current_url = self.driver.current_url\n message = self.__get_exception_message()\n self.__delayed_assert_failures.append(\n \"CHECK #%s: (%s)\\n %s\" % (\n self.__delayed_assert_count, current_url, message))"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef process_delayed_asserts(self, print_only=False):\n if self.__delayed_assert_failures:\n exception_output = ''\n exception_output += \"\\n*** DELAYED ASSERTION FAILURES FOR: \"\n exception_output += \"%s\\n\" % self.id()\n all_failing_checks = self.__delayed_assert_failures\n self.__delayed_assert_failures = []\n for tb in all_failing_checks:\n exception_output += \"%s\\n\" % tb\n if print_only:\n print(exception_output)\n else:\n raise Exception(exception_output)", "response": "This method is used to process delayed asserts that are not terminated by any test that uses delayed_asserts which only raise exceptions\n after this method is called."} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nclicks an element using pure JS. Does not use jQuery.", "response": "def __js_click(self, selector, by=By.CSS_SELECTOR):\n \"\"\" Clicks an element using pure JS. Does not use jQuery. \"\"\"\n selector, by = self.__recalculate_selector(selector, by)\n css_selector = self.convert_to_css_selector(selector, by=by)\n css_selector = re.escape(css_selector)\n css_selector = self.__escape_quotes_if_needed(css_selector)\n script = (\"\"\"var simulateClick = function (elem) {\n var evt = new MouseEvent('click', {\n bubbles: true,\n cancelable: true,\n view: window\n });\n var canceled = !elem.dispatchEvent(evt);\n };\n var someLink = document.querySelector('%s');\n simulateClick(someLink);\"\"\"\n % css_selector)\n self.execute_script(script)"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef __click_dropdown_link_text(self, link_text, link_css):\n soup = self.get_beautiful_soup()\n drop_down_list = soup.select('[class*=dropdown]')\n for item in soup.select('[class*=HeaderMenu]'):\n drop_down_list.append(item)\n for item in soup.select('[class*=menu-item]'):\n drop_down_list.append(item)\n for item in soup.select('[class*=chevron]'):\n drop_down_list.append(item)\n csstype = link_css.split('[')[1].split('=')[0]\n for item in drop_down_list:\n if link_text in item.text.split('\\n') and csstype in item.decode():\n dropdown_css = \"\"\n for css_class in item['class']:\n dropdown_css += '.'\n dropdown_css += css_class\n dropdown_css = item.name + dropdown_css\n matching_dropdowns = self.find_visible_elements(dropdown_css)\n for dropdown in matching_dropdowns:\n # The same class names might be used for multiple dropdowns\n try:\n page_actions.hover_element_and_click(\n self.driver, dropdown, link_text,\n click_by=By.LINK_TEXT, timeout=0.2)\n return True\n except Exception:\n pass\n return False", "response": "Clicks a dropdown link text."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef setUp(self, masterqa_mode=False):\n self.masterqa_mode = masterqa_mode\n self.is_pytest = None\n try:\n # This raises an exception if the test is not coming from pytest\n self.is_pytest = sb_config.is_pytest\n except Exception:\n # Not using pytest (probably nosetests)\n self.is_pytest = False\n if self.is_pytest:\n # pytest-specific code\n test_id = \"%s.%s.%s\" % (self.__class__.__module__,\n self.__class__.__name__,\n self._testMethodName)\n self.browser = sb_config.browser\n self.data = sb_config.data\n self.demo_mode = sb_config.demo_mode\n self.demo_sleep = sb_config.demo_sleep\n self.highlights = sb_config.highlights\n self.environment = sb_config.environment\n self.env = self.environment # Add a shortened version\n self.with_selenium = sb_config.with_selenium # Should be True\n self.headless = sb_config.headless\n self.headless_active = False\n self.log_path = sb_config.log_path\n self.with_testing_base = sb_config.with_testing_base\n self.with_basic_test_info = sb_config.with_basic_test_info\n self.with_screen_shots = sb_config.with_screen_shots\n self.with_page_source = sb_config.with_page_source\n self.with_db_reporting = sb_config.with_db_reporting\n self.with_s3_logging = sb_config.with_s3_logging\n self.servername = sb_config.servername\n self.port = sb_config.port\n self.proxy_string = sb_config.proxy_string\n self.user_agent = sb_config.user_agent\n self.cap_file = sb_config.cap_file\n self.database_env = sb_config.database_env\n self.message_duration = sb_config.message_duration\n self.js_checking_on = sb_config.js_checking_on\n self.ad_block_on = sb_config.ad_block_on\n self.verify_delay = sb_config.verify_delay\n self.disable_csp = sb_config.disable_csp\n self.save_screenshot_after_test = sb_config.save_screenshot\n self.visual_baseline = sb_config.visual_baseline\n self.timeout_multiplier = sb_config.timeout_multiplier\n self.pytest_html_report = sb_config.pytest_html_report\n self.report_on = False\n if self.pytest_html_report:\n self.report_on = True\n self.use_grid = False\n if self.servername != \"localhost\":\n # Use Selenium Grid (Use --server=127.0.0.1 for localhost Grid)\n self.use_grid = True\n if self.with_db_reporting:\n from seleniumbase.core.application_manager import (\n ApplicationManager)\n from seleniumbase.core.testcase_manager import (\n ExecutionQueryPayload)\n import getpass\n self.execution_guid = str(uuid.uuid4())\n self.testcase_guid = None\n self.execution_start_time = 0\n self.case_start_time = 0\n self.application = None\n self.testcase_manager = None\n self.error_handled = False\n self.testcase_manager = TestcaseManager(self.database_env)\n #\n exec_payload = ExecutionQueryPayload()\n exec_payload.execution_start_time = int(time.time() * 1000)\n self.execution_start_time = exec_payload.execution_start_time\n exec_payload.guid = self.execution_guid\n exec_payload.username = getpass.getuser()\n self.testcase_manager.insert_execution_data(exec_payload)\n #\n data_payload = TestcaseDataPayload()\n self.testcase_guid = str(uuid.uuid4())\n data_payload.guid = self.testcase_guid\n data_payload.execution_guid = self.execution_guid\n if self.with_selenium:\n data_payload.browser = self.browser\n else:\n data_payload.browser = \"N/A\"\n data_payload.test_address = test_id\n application = ApplicationManager.generate_application_string(\n self._testMethodName)\n data_payload.env = application.split('.')[0]\n data_payload.start_time = application.split('.')[1]\n data_payload.state = constants.State.NOTRUN\n self.testcase_manager.insert_testcase_data(data_payload)\n self.case_start_time = int(time.time() * 1000)\n if self.headless:\n try:\n from pyvirtualdisplay import Display\n self.display = Display(visible=0, size=(1440, 1080))\n self.display.start()\n self.headless_active = True\n except Exception:\n # pyvirtualdisplay might not be necessary anymore because\n # Chrome and Firefox now have built-in headless displays\n pass\n\n # Launch WebDriver for both Pytest and Nosetests\n if not hasattr(self, \"browser\"):\n raise Exception(\"\"\"SeleniumBase plugins did not load! \"\"\"\n \"\"\"Please reinstall using:\\n\"\"\"\n \"\"\" >>> \"python setup.py install\" <<< \"\"\")\n self.driver = self.get_new_driver(browser=self.browser,\n headless=self.headless,\n servername=self.servername,\n port=self.port,\n proxy=self.proxy_string,\n agent=self.user_agent,\n switch_to=True,\n cap_file=self.cap_file,\n disable_csp=self.disable_csp)\n self._default_driver = self.driver", "response": "This method is called by the base class to set up the internal properties of the base class."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef __set_last_page_screenshot(self):\n if not self.__last_page_screenshot and (\n not self.__last_page_screenshot_png):\n try:\n element = self.driver.find_element_by_tag_name('body')\n if self.is_pytest and self.report_on:\n self.__last_page_screenshot_png = (\n self.driver.get_screenshot_as_png())\n self.__last_page_screenshot = element.screenshot_as_base64\n else:\n self.__last_page_screenshot_png = element.screenshot_as_png\n except Exception:\n if not self.__last_page_screenshot:\n if self.is_pytest and self.report_on:\n try:\n self.__last_page_screenshot = (\n self.driver.get_screenshot_as_base64())\n except Exception:\n pass\n if not self.__last_page_screenshot_png:\n try:\n self.__last_page_screenshot_png = (\n self.driver.get_screenshot_as_png())\n except Exception:\n pass", "response": "Sets the last screenshot of the page."} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nis careful if a subclass of BaseCase overrides setUp() You'll need to add the following line to the subclass's tearDown(): super(SubClassOfBaseCase, self).tearDown()", "response": "def tearDown(self):\n \"\"\"\n Be careful if a subclass of BaseCase overrides setUp()\n You'll need to add the following line to the subclass's tearDown():\n super(SubClassOfBaseCase, self).tearDown()\n \"\"\"\n has_exception = False\n if sys.version.startswith('3') and hasattr(self, '_outcome'):\n if hasattr(self._outcome, 'errors') and self._outcome.errors:\n has_exception = True\n else:\n has_exception = sys.exc_info()[1] is not None\n if self.__delayed_assert_failures:\n print(\n \"\\nWhen using self.delayed_assert_*() methods in your tests, \"\n \"remember to call self.process_delayed_asserts() afterwards. \"\n \"Now calling in tearDown()...\\nFailures Detected:\")\n if not has_exception:\n self.process_delayed_asserts()\n else:\n self.process_delayed_asserts(print_only=True)\n self.is_pytest = None\n try:\n # This raises an exception if the test is not coming from pytest\n self.is_pytest = sb_config.is_pytest\n except Exception:\n # Not using pytest (probably nosetests)\n self.is_pytest = False\n if self.is_pytest:\n # pytest-specific code\n test_id = \"%s.%s.%s\" % (self.__class__.__module__,\n self.__class__.__name__,\n self._testMethodName)\n try:\n with_selenium = self.with_selenium\n except Exception:\n sub_class_name = str(\n self.__class__.__bases__[0]).split('.')[-1].split(\"'\")[0]\n sub_file_name = str(self.__class__.__bases__[0]).split('.')[-2]\n sub_file_name = sub_file_name + \".py\"\n class_name = str(self.__class__).split('.')[-1].split(\"'\")[0]\n file_name = str(self.__class__).split('.')[-2] + \".py\"\n class_name_used = sub_class_name\n file_name_used = sub_file_name\n if sub_class_name == \"BaseCase\":\n class_name_used = class_name\n file_name_used = file_name\n fix_setup = \"super(%s, self).setUp()\" % class_name_used\n fix_teardown = \"super(%s, self).tearDown()\" % class_name_used\n message = (\"You're overriding SeleniumBase's BaseCase setUp() \"\n \"method with your own setUp() method, which breaks \"\n \"SeleniumBase. You can fix this by going to your \"\n \"%s class located in your %s file and adding the \"\n \"following line of code AT THE BEGINNING of your \"\n \"setUp() method:\\n%s\\n\\nAlso make sure \"\n \"you have added the following line of code AT THE \"\n \"END of your tearDown() method:\\n%s\\n\"\n % (class_name_used, file_name_used,\n fix_setup, fix_teardown))\n raise Exception(message)\n if with_selenium:\n # Save a screenshot if logging is on when an exception occurs\n if has_exception:\n self.__add_pytest_html_extra()\n if self.with_testing_base and not has_exception and (\n self.save_screenshot_after_test):\n test_logpath = self.log_path + \"/\" + test_id\n if not os.path.exists(test_logpath):\n try:\n os.makedirs(test_logpath)\n except Exception:\n pass # Only reachable during multi-threaded runs\n if not self.__last_page_screenshot_png:\n self.__set_last_page_screenshot()\n log_helper.log_screenshot(\n test_logpath,\n self.driver,\n self.__last_page_screenshot_png)\n self.__add_pytest_html_extra()\n if self.with_testing_base and has_exception:\n test_logpath = self.log_path + \"/\" + test_id\n if not os.path.exists(test_logpath):\n try:\n os.makedirs(test_logpath)\n except Exception:\n pass # Only reachable during multi-threaded runs\n if ((not self.with_screen_shots) and (\n not self.with_basic_test_info) and (\n not self.with_page_source)):\n # Log everything if nothing specified (if testing_base)\n if not self.__last_page_screenshot_png:\n self.__set_last_page_screenshot()\n log_helper.log_screenshot(\n test_logpath,\n self.driver,\n self.__last_page_screenshot_png)\n log_helper.log_test_failure_data(\n self, test_logpath, self.driver, self.browser)\n log_helper.log_page_source(test_logpath, self.driver)\n else:\n if self.with_screen_shots:\n if not self.__last_page_screenshot_png:\n self.__set_last_page_screenshot()\n log_helper.log_screenshot(\n test_logpath,\n self.driver,\n self.__last_page_screenshot_png)\n if self.with_basic_test_info:\n log_helper.log_test_failure_data(\n self, test_logpath, self.driver, self.browser)\n if self.with_page_source:\n log_helper.log_page_source(\n test_logpath, self.driver)\n # (Pytest) Finally close all open browser windows\n self.__quit_all_drivers()\n if self.headless:\n if self.headless_active:\n self.display.stop()\n self.display = None\n if self.with_db_reporting:\n if has_exception:\n self.__insert_test_result(constants.State.ERROR, True)\n else:\n self.__insert_test_result(constants.State.PASS, False)\n runtime = int(time.time() * 1000) - self.execution_start_time\n self.testcase_manager.update_execution_data(\n self.execution_guid, runtime)\n if self.with_s3_logging and has_exception:\n \"\"\" If enabled, upload logs to S3 during test exceptions. \"\"\"\n from seleniumbase.core.s3_manager import S3LoggingBucket\n s3_bucket = S3LoggingBucket()\n guid = str(uuid.uuid4().hex)\n path = \"%s/%s\" % (self.log_path, test_id)\n uploaded_files = []\n for logfile in os.listdir(path):\n logfile_name = \"%s/%s/%s\" % (guid,\n test_id,\n logfile.split(path)[-1])\n s3_bucket.upload_file(logfile_name,\n \"%s/%s\" % (path, logfile))\n uploaded_files.append(logfile_name)\n s3_bucket.save_uploaded_file_names(uploaded_files)\n index_file = s3_bucket.upload_index_file(test_id, guid)\n print(\"\\n\\n*** Log files uploaded: ***\\n%s\\n\" % index_file)\n logging.error(\n \"\\n\\n*** Log files uploaded: ***\\n%s\\n\" % index_file)\n if self.with_db_reporting:\n self.testcase_manager = TestcaseManager(self.database_env)\n data_payload = TestcaseDataPayload()\n data_payload.guid = self.testcase_guid\n data_payload.logURL = index_file\n self.testcase_manager.update_testcase_log_url(data_payload)\n else:\n # (Nosetests)\n if has_exception:\n test_id = \"%s.%s.%s\" % (self.__class__.__module__,\n self.__class__.__name__,\n self._testMethodName)\n test_logpath = \"latest_logs/\" + test_id\n if not os.path.exists(test_logpath):\n try:\n os.makedirs(test_logpath)\n except Exception:\n pass # Only reachable during multi-threaded runs\n log_helper.log_test_failure_data(\n self, test_logpath, self.driver, self.browser)\n if len(self._drivers_list) > 0:\n if not self.__last_page_screenshot_png:\n self.__set_last_page_screenshot()\n log_helper.log_screenshot(\n test_logpath,\n self.driver,\n self.__last_page_screenshot_png)\n log_helper.log_page_source(test_logpath, self.driver)\n elif self.save_screenshot_after_test:\n test_id = \"%s.%s.%s\" % (self.__class__.__module__,\n self.__class__.__name__,\n self._testMethodName)\n test_logpath = \"latest_logs/\" + test_id\n if not os.path.exists(test_logpath):\n try:\n os.makedirs(test_logpath)\n except Exception:\n pass # Only reachable during multi-threaded runs\n if not self.__last_page_screenshot_png:\n self.__set_last_page_screenshot()\n log_helper.log_screenshot(\n test_logpath,\n self.driver,\n self.__last_page_screenshot_png)\n if self.report_on:\n self._last_page_screenshot = self.__last_page_screenshot_png\n try:\n self._last_page_url = self.get_current_url()\n except Exception:\n self._last_page_url = \"(Error: Unknown URL)\"\n # Finally close all open browser windows\n self.__quit_all_drivers()"} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nupload a given file to the bucket with the new name and path file_name.", "response": "def upload_file(self, file_name, file_path):\n \"\"\" Upload a given file from the file_path to the bucket\n with the new name/path file_name. \"\"\"\n upload_key = Key(bucket=self.bucket, name=file_name)\n content_type = \"text/plain\"\n if file_name.endswith(\".html\"):\n content_type = \"text/html\"\n elif file_name.endswith(\".jpg\"):\n content_type = \"image/jpeg\"\n elif file_name.endswith(\".png\"):\n content_type = \"image/png\"\n upload_key.set_contents_from_filename(\n file_path,\n headers={\"Content-Type\": content_type})\n upload_key.url = \\\n upload_key.generate_url(expires_in=3600).split(\"?\")[0]\n try:\n upload_key.make_public()\n except Exception:\n pass"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef upload_index_file(self, test_address, timestamp):\n global already_uploaded_files\n already_uploaded_files = list(set(already_uploaded_files))\n already_uploaded_files.sort()\n file_name = \"%s/%s/index.html\" % (test_address, timestamp)\n index = self.get_key(file_name)\n index_str = []\n for completed_file in already_uploaded_files:\n index_str.append(\"%s\" % (completed_file, completed_file))\n index.set_contents_from_string(\n \"
\".join(index_str),\n headers={\"Content-Type\": \"text/html\"})\n index.make_public()\n return \"%s%s\" % (self.bucket_url, file_name)", "response": "Create an index. html file with links to all the log files that were just uploaded."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef activate_bootstrap(driver):\n bootstrap_tour_css = constants.BootstrapTour.MIN_CSS\n bootstrap_tour_js = constants.BootstrapTour.MIN_JS\n\n verify_script = (\"\"\"// Verify Bootstrap Tour activated\n var tour2 = new Tour({\n });\"\"\")\n\n backdrop_style = style_sheet.bt_backdrop_style\n js_utils.add_css_style(driver, backdrop_style)\n js_utils.wait_for_ready_state_complete(driver)\n js_utils.wait_for_angularjs(driver)\n for x in range(4):\n js_utils.activate_jquery(driver)\n js_utils.add_css_link(driver, bootstrap_tour_css)\n js_utils.add_js_link(driver, bootstrap_tour_js)\n time.sleep(0.1)\n for x in range(int(settings.MINI_TIMEOUT * 2.0)):\n # Bootstrap needs a small amount of time to load & activate.\n try:\n driver.execute_script(verify_script)\n time.sleep(0.05)\n return\n except Exception:\n time.sleep(0.15)\n raise_unable_to_load_jquery_exception(driver)", "response": "Activates Bootstrap Tours with SeleniumBase\n "} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef activate_hopscotch(driver):\n hopscotch_css = constants.Hopscotch.MIN_CSS\n hopscotch_js = constants.Hopscotch.MIN_JS\n backdrop_style = style_sheet.hops_backdrop_style\n\n verify_script = (\"\"\"// Verify Hopscotch activated\n var hops = hopscotch.isActive;\n \"\"\")\n\n activate_bootstrap(driver)\n js_utils.wait_for_ready_state_complete(driver)\n js_utils.wait_for_angularjs(driver)\n js_utils.add_css_style(driver, backdrop_style)\n for x in range(4):\n js_utils.activate_jquery(driver)\n js_utils.add_css_link(driver, hopscotch_css)\n js_utils.add_js_link(driver, hopscotch_js)\n time.sleep(0.1)\n for x in range(int(settings.MINI_TIMEOUT * 2.0)):\n # Hopscotch needs a small amount of time to load & activate.\n try:\n driver.execute_script(verify_script)\n js_utils.wait_for_ready_state_complete(driver)\n js_utils.wait_for_angularjs(driver)\n time.sleep(0.05)\n return\n except Exception:\n time.sleep(0.15)\n raise_unable_to_load_jquery_exception(driver)", "response": "Activates Hopscotch Tours with Hopscotch Tours with SeleniumBase\n SARL."} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nactivates IntroJS Tours with SeleniumBase .", "response": "def activate_introjs(driver):\n \"\"\" Allows you to use IntroJS Tours with SeleniumBase\n https://introjs.com/\n \"\"\"\n intro_css = constants.IntroJS.MIN_CSS\n intro_js = constants.IntroJS.MIN_JS\n\n verify_script = (\"\"\"// Verify IntroJS activated\n var intro2 = introJs();\n \"\"\")\n\n activate_bootstrap(driver)\n js_utils.wait_for_ready_state_complete(driver)\n js_utils.wait_for_angularjs(driver)\n for x in range(4):\n js_utils.activate_jquery(driver)\n js_utils.add_css_link(driver, intro_css)\n js_utils.add_js_link(driver, intro_js)\n time.sleep(0.1)\n for x in range(int(settings.MINI_TIMEOUT * 2.0)):\n # IntroJS needs a small amount of time to load & activate.\n try:\n driver.execute_script(verify_script)\n js_utils.wait_for_ready_state_complete(driver)\n js_utils.wait_for_angularjs(driver)\n time.sleep(0.05)\n return\n except Exception:\n time.sleep(0.15)\n raise_unable_to_load_jquery_exception(driver)"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nactivate the shepherd theme", "response": "def activate_shepherd(driver):\n \"\"\" Allows you to use Shepherd Tours with SeleniumBase\n http://github.hubspot.com/shepherd/docs/welcome/\n \"\"\"\n shepherd_js = constants.Shepherd.MIN_JS\n sh_theme_arrows_css = constants.Shepherd.THEME_ARROWS_CSS\n sh_theme_arrows_fix_css = constants.Shepherd.THEME_ARR_FIX_CSS\n sh_theme_default_css = constants.Shepherd.THEME_DEFAULT_CSS\n sh_theme_dark_css = constants.Shepherd.THEME_DARK_CSS\n sh_theme_sq_css = constants.Shepherd.THEME_SQ_CSS\n sh_theme_sq_dark_css = constants.Shepherd.THEME_SQ_DK_CSS\n tether_js = constants.Tether.MIN_JS\n spinner_css = constants.Messenger.SPINNER_CSS\n sh_style = style_sheet.sh_style_test\n backdrop_style = style_sheet.sh_backdrop_style\n\n activate_bootstrap(driver)\n js_utils.wait_for_ready_state_complete(driver)\n js_utils.wait_for_angularjs(driver)\n js_utils.add_css_style(driver, backdrop_style)\n js_utils.wait_for_ready_state_complete(driver)\n js_utils.wait_for_angularjs(driver)\n for x in range(4):\n js_utils.add_css_link(driver, spinner_css)\n js_utils.add_css_link(driver, sh_theme_arrows_css)\n js_utils.add_css_link(driver, sh_theme_arrows_fix_css)\n js_utils.add_css_link(driver, sh_theme_default_css)\n js_utils.add_css_link(driver, sh_theme_dark_css)\n js_utils.add_css_link(driver, sh_theme_sq_css)\n js_utils.add_css_link(driver, sh_theme_sq_dark_css)\n js_utils.add_js_link(driver, tether_js)\n js_utils.add_js_link(driver, shepherd_js)\n time.sleep(0.1)\n for x in range(int(settings.MINI_TIMEOUT * 2.0)):\n # Shepherd needs a small amount of time to load & activate.\n try:\n driver.execute_script(sh_style) # Verify Shepherd has loaded\n js_utils.wait_for_ready_state_complete(driver)\n js_utils.wait_for_angularjs(driver)\n driver.execute_script(sh_style) # Need it twice for ordering\n js_utils.wait_for_ready_state_complete(driver)\n js_utils.wait_for_angularjs(driver)\n time.sleep(0.05)\n return\n except Exception:\n time.sleep(0.15)\n raise_unable_to_load_jquery_exception(driver)"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef play_shepherd_tour(driver, tour_steps, msg_dur, name=None, interval=0):\n instructions = \"\"\n for tour_step in tour_steps[name]:\n instructions += tour_step\n instructions += (\"\"\"\n // Start the tour\n tour.start();\n $tour = tour;\"\"\")\n autoplay = False\n if interval and interval > 0:\n autoplay = True\n interval = float(interval)\n if interval < 0.5:\n interval = 0.5\n\n if not is_shepherd_activated(driver):\n activate_shepherd(driver)\n\n if len(tour_steps[name]) > 1:\n try:\n selector = re.search(\n r\"[\\S\\s]+{element: '([\\S\\s]+)', on: [\\S\\s]+\",\n tour_steps[name][1]).group(1)\n selector = selector.replace('\\\\', '')\n page_actions.wait_for_element_present(\n driver, selector, by=By.CSS_SELECTOR,\n timeout=settings.SMALL_TIMEOUT)\n except Exception:\n js_utils.post_messenger_error_message(\n driver, \"Tour Error: {'%s'} was not found!\" % selector,\n msg_dur)\n raise Exception(\n \"Tour Error: {'%s'} was not found! \"\n \"Exiting due to failure on first tour step!\"\n \"\" % selector)\n driver.execute_script(instructions)\n tour_on = True\n if autoplay:\n start_ms = time.time() * 1000.0\n stop_ms = start_ms + (interval * 1000.0)\n latest_element = None\n latest_text = None\n while tour_on:\n try:\n time.sleep(0.01)\n result = driver.execute_script(\n \"return Shepherd.activeTour.currentStep.isOpen()\")\n except Exception:\n tour_on = False\n result = None\n if result:\n tour_on = True\n if autoplay:\n try:\n element = driver.execute_script(\n \"return Shepherd.activeTour.currentStep\"\n \".options.attachTo.element\")\n shep_text = driver.execute_script(\n \"return Shepherd.activeTour.currentStep\"\n \".options.text\")\n except Exception:\n continue\n if element != latest_element or shep_text != latest_text:\n latest_element = element\n latest_text = shep_text\n start_ms = time.time() * 1000.0\n stop_ms = start_ms + (interval * 1000.0)\n now_ms = time.time() * 1000.0\n if now_ms >= stop_ms:\n if ((element == latest_element) and (\n shep_text == latest_text)):\n driver.execute_script(\"Shepherd.activeTour.next()\")\n try:\n latest_element = driver.execute_script(\n \"return Shepherd.activeTour.currentStep\"\n \".options.attachTo.element\")\n latest_text = driver.execute_script(\n \"return Shepherd.activeTour.currentStep\"\n \".options.text\")\n start_ms = time.time() * 1000.0\n stop_ms = start_ms + (interval * 1000.0)\n except Exception:\n pass\n continue\n else:\n try:\n time.sleep(0.01)\n selector = driver.execute_script(\n \"return Shepherd.activeTour\"\n \".currentStep.options.attachTo.element\")\n try:\n js_utils.wait_for_css_query_selector(\n driver, selector, timeout=settings.SMALL_TIMEOUT)\n except Exception:\n remove_script = (\n \"jQuery('%s').remove()\" % \"div.shepherd-content\")\n driver.execute_script(remove_script)\n js_utils.post_messenger_error_message(\n driver, \"Tour Error: {'%s'} was not found!\" % selector,\n msg_dur)\n time.sleep(0.1)\n driver.execute_script(\"Shepherd.activeTour.next()\")\n if autoplay:\n start_ms = time.time() * 1000.0\n stop_ms = start_ms + (interval * 1000.0)\n tour_on = True\n except Exception:\n tour_on = False\n time.sleep(0.1)", "response": "Plays a Shepherd tour on the current website."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef play_bootstrap_tour(\n driver, tour_steps, browser, msg_dur, name=None, interval=0):\n \"\"\" Plays a Bootstrap tour on the current website. \"\"\"\n instructions = \"\"\n for tour_step in tour_steps[name]:\n instructions += tour_step\n instructions += (\n \"\"\"]);\n // Initialize the tour\n tour.init();\n // Start the tour\n tour.start();\n // Fix timing issue by restarting tour immediately\n tour.restart();\n // Save for later\n $tour = tour;\"\"\")\n\n if interval and interval > 0:\n if interval < 1:\n interval = 1\n interval = str(float(interval) * 1000.0)\n instructions = instructions.replace(\n 'duration: 0,', 'duration: %s,' % interval)\n\n if not is_bootstrap_activated(driver):\n activate_bootstrap(driver)\n\n if len(tour_steps[name]) > 1:\n try:\n if \"element: \" in tour_steps[name][1]:\n selector = re.search(\n r\"[\\S\\s]+element: '([\\S\\s]+)',[\\S\\s]+title: '\",\n tour_steps[name][1]).group(1)\n selector = selector.replace('\\\\', '').replace(':first', '')\n page_actions.wait_for_element_present(\n driver, selector, by=By.CSS_SELECTOR,\n timeout=settings.SMALL_TIMEOUT)\n else:\n selector = \"html\"\n except Exception:\n js_utils.post_messenger_error_message(\n driver, \"Tour Error: {'%s'} was not found!\" % selector,\n msg_dur)\n raise Exception(\n \"Tour Error: {'%s'} was not found! \"\n \"Exiting due to failure on first tour step!\"\n \"\" % selector)\n\n driver.execute_script(instructions)\n tour_on = True\n while tour_on:\n try:\n time.sleep(0.01)\n if browser != \"firefox\":\n result = driver.execute_script(\n \"return $tour.ended()\")\n else:\n page_actions.wait_for_element_present(\n driver, \".tour-tour\", by=By.CSS_SELECTOR, timeout=0.4)\n result = False\n except Exception:\n tour_on = False\n result = None\n if result is False:\n tour_on = True\n else:\n try:\n time.sleep(0.01)\n if browser != \"firefox\":\n result = driver.execute_script(\n \"return $tour.ended()\")\n else:\n page_actions.wait_for_element_present(\n driver, \".tour-tour\", by=By.CSS_SELECTOR, timeout=0.4)\n result = False\n if result is False:\n time.sleep(0.1)\n continue\n else:\n return\n except Exception:\n tour_on = False\n time.sleep(0.1)", "response": "Plays a Bootstrap tour on the current website."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef play_introjs_tour(\n driver, tour_steps, browser, msg_dur, name=None, interval=0):\n \"\"\" Plays an IntroJS tour on the current website. \"\"\"\n instructions = \"\"\n for tour_step in tour_steps[name]:\n instructions += tour_step\n instructions += (\n \"\"\"]\n });\n intro.setOption(\"disableInteraction\", true);\n intro.setOption(\"overlayOpacity\", .29);\n intro.setOption(\"scrollToElement\", true);\n intro.setOption(\"keyboardNavigation\", true);\n intro.setOption(\"exitOnEsc\", false);\n intro.setOption(\"exitOnOverlayClick\", false);\n intro.setOption(\"showStepNumbers\", false);\n intro.setOption(\"showProgress\", false);\n intro.start();\n $tour = intro;\n };\n // Start the tour\n startIntro();\n \"\"\")\n autoplay = False\n if interval and interval > 0:\n autoplay = True\n interval = float(interval)\n if interval < 0.5:\n interval = 0.5\n\n if not is_introjs_activated(driver):\n activate_introjs(driver)\n\n if len(tour_steps[name]) > 1:\n try:\n if \"element: \" in tour_steps[name][1]:\n selector = re.search(\n r\"[\\S\\s]+element: '([\\S\\s]+)',[\\S\\s]+intro: '\",\n tour_steps[name][1]).group(1)\n selector = selector.replace('\\\\', '')\n page_actions.wait_for_element_present(\n driver, selector, by=By.CSS_SELECTOR,\n timeout=settings.SMALL_TIMEOUT)\n else:\n selector = \"html\"\n except Exception:\n js_utils.post_messenger_error_message(\n driver, \"Tour Error: {'%s'} was not found!\" % selector,\n msg_dur)\n raise Exception(\n \"Tour Error: {'%s'} was not found! \"\n \"Exiting due to failure on first tour step!\"\n \"\" % selector)\n driver.execute_script(instructions)\n tour_on = True\n if autoplay:\n start_ms = time.time() * 1000.0\n stop_ms = start_ms + (interval * 1000.0)\n latest_step = 0\n while tour_on:\n try:\n time.sleep(0.01)\n if browser != \"firefox\":\n result = driver.execute_script(\n \"return $tour._currentStep\")\n else:\n page_actions.wait_for_element_present(\n driver, \".introjs-tooltip\",\n by=By.CSS_SELECTOR, timeout=0.4)\n result = True\n except Exception:\n tour_on = False\n result = None\n if result is not None:\n tour_on = True\n if autoplay:\n try:\n current_step = driver.execute_script(\n \"return $tour._currentStep\")\n except Exception:\n continue\n if current_step != latest_step:\n latest_step = current_step\n start_ms = time.time() * 1000.0\n stop_ms = start_ms + (interval * 1000.0)\n now_ms = time.time() * 1000.0\n if now_ms >= stop_ms:\n if current_step == latest_step:\n driver.execute_script(\"return $tour.nextStep()\")\n try:\n latest_step = driver.execute_script(\n \"return $tour._currentStep\")\n start_ms = time.time() * 1000.0\n stop_ms = start_ms + (interval * 1000.0)\n except Exception:\n pass\n continue\n else:\n try:\n time.sleep(0.01)\n if browser != \"firefox\":\n result = driver.execute_script(\n \"return $tour._currentStep\")\n else:\n page_actions.wait_for_element_present(\n driver, \".introjs-tooltip\",\n by=By.CSS_SELECTOR, timeout=0.4)\n result = True\n if result is not None:\n time.sleep(0.1)\n continue\n else:\n return\n except Exception:\n tour_on = False\n time.sleep(0.1)", "response": "Plays an IntroJS tour on the current website."} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nexports a tour as a JavaScript file.", "response": "def export_tour(tour_steps, name=None, filename=\"my_tour.js\", url=None):\n \"\"\" Exports a tour as a JS file.\n It will include necessary resources as well, such as jQuery.\n You'll be able to copy the tour directly into the Console of\n any web browser to play the tour outside of SeleniumBase runs. \"\"\"\n if not name:\n name = \"default\"\n if name not in tour_steps:\n raise Exception(\"Tour {%s} does not exist!\" % name)\n if not filename.endswith('.js'):\n raise Exception('Tour file must end in \".js\"!')\n if not url:\n url = \"data:,\"\n\n tour_type = None\n if \"Bootstrap\" in tour_steps[name][0]:\n tour_type = \"bootstrap\"\n elif \"Hopscotch\" in tour_steps[name][0]:\n tour_type = \"hopscotch\"\n elif \"IntroJS\" in tour_steps[name][0]:\n tour_type = \"introjs\"\n elif \"Shepherd\" in tour_steps[name][0]:\n tour_type = \"shepherd\"\n else:\n raise Exception('Unknown tour type!')\n\n instructions = (\n '''//////// Load Tour Start Page (if not there now) ////////\\n\\n'''\n '''if (window.location.href != \"%s\") {\\n'''\n ''' window.location.href=\"%s\";\\n'''\n '''}\\n\\n'''\n '''//////// Resources ////////\\n\\n'''\n '''function injectCSS(css_link) {'''\n '''var head = document.getElementsByTagName(\"head\")[0];'''\n '''var link = document.createElement(\"link\");'''\n '''link.rel = \"stylesheet\";'''\n '''link.type = \"text/css\";'''\n '''link.href = css_link;'''\n '''link.crossorigin = \"anonymous\";'''\n '''head.appendChild(link);'''\n '''};\\n'''\n '''function injectJS(js_link) {'''\n '''var head = document.getElementsByTagName(\"head\")[0];'''\n '''var script = document.createElement(\"script\");'''\n '''script.src = js_link;'''\n '''script.defer;'''\n '''script.type=\"text/javascript\";'''\n '''script.crossorigin = \"anonymous\";'''\n '''script.onload = function() { null };'''\n '''head.appendChild(script);'''\n '''};\\n'''\n '''function injectStyle(css) {'''\n '''var head = document.getElementsByTagName(\"head\")[0];'''\n '''var style = document.createElement(\"style\");'''\n '''style.type = \"text/css\";'''\n '''style.appendChild(document.createTextNode(css));'''\n '''head.appendChild(style);'''\n '''};\\n''' % (url, url))\n\n if tour_type == \"bootstrap\":\n jquery_js = constants.JQuery.MIN_JS\n bootstrap_tour_css = constants.BootstrapTour.MIN_CSS\n bootstrap_tour_js = constants.BootstrapTour.MIN_JS\n backdrop_style = style_sheet.bt_backdrop_style\n backdrop_style = backdrop_style.replace('\\n', '')\n backdrop_style = js_utils.escape_quotes_if_needed(backdrop_style)\n instructions += 'injectJS(\"%s\");' % jquery_js\n instructions += '\\n\\n//////// Resources - Load 2 ////////\\n\\n'\n instructions += 'injectCSS(\"%s\");\\n' % bootstrap_tour_css\n instructions += 'injectStyle(\"%s\");\\n' % backdrop_style\n instructions += 'injectJS(\"%s\");' % bootstrap_tour_js\n\n elif tour_type == \"hopscotch\":\n hopscotch_css = constants.Hopscotch.MIN_CSS\n hopscotch_js = constants.Hopscotch.MIN_JS\n backdrop_style = style_sheet.hops_backdrop_style\n backdrop_style = backdrop_style.replace('\\n', '')\n backdrop_style = js_utils.escape_quotes_if_needed(backdrop_style)\n instructions += 'injectCSS(\"%s\");\\n' % hopscotch_css\n instructions += 'injectStyle(\"%s\");\\n' % backdrop_style\n instructions += 'injectJS(\"%s\");' % hopscotch_js\n\n elif tour_type == \"introjs\":\n intro_css = constants.IntroJS.MIN_CSS\n intro_js = constants.IntroJS.MIN_JS\n instructions += 'injectCSS(\"%s\");\\n' % intro_css\n instructions += 'injectJS(\"%s\");' % intro_js\n\n elif tour_type == \"shepherd\":\n jquery_js = constants.JQuery.MIN_JS\n shepherd_js = constants.Shepherd.MIN_JS\n sh_theme_arrows_css = constants.Shepherd.THEME_ARROWS_CSS\n sh_theme_arrows_fix_css = constants.Shepherd.THEME_ARR_FIX_CSS\n sh_theme_default_css = constants.Shepherd.THEME_DEFAULT_CSS\n sh_theme_dark_css = constants.Shepherd.THEME_DARK_CSS\n sh_theme_sq_css = constants.Shepherd.THEME_SQ_CSS\n sh_theme_sq_dark_css = constants.Shepherd.THEME_SQ_DK_CSS\n tether_js = constants.Tether.MIN_JS\n spinner_css = constants.Messenger.SPINNER_CSS\n backdrop_style = style_sheet.sh_backdrop_style\n backdrop_style = backdrop_style.replace('\\n', '')\n backdrop_style = js_utils.escape_quotes_if_needed(backdrop_style)\n instructions += 'injectCSS(\"%s\");\\n' % spinner_css\n instructions += 'injectJS(\"%s\");\\n' % jquery_js\n instructions += 'injectJS(\"%s\");' % tether_js\n instructions += '\\n\\n//////// Resources - Load 2 ////////\\n\\n'\n instructions += 'injectCSS(\"%s\");' % sh_theme_arrows_css\n instructions += 'injectCSS(\"%s\");' % sh_theme_arrows_fix_css\n instructions += 'injectCSS(\"%s\");' % sh_theme_default_css\n instructions += 'injectCSS(\"%s\");' % sh_theme_dark_css\n instructions += 'injectCSS(\"%s\");' % sh_theme_sq_css\n instructions += 'injectCSS(\"%s\");\\n' % sh_theme_sq_dark_css\n instructions += 'injectStyle(\"%s\");\\n' % backdrop_style\n instructions += 'injectJS(\"%s\");' % shepherd_js\n\n instructions += '\\n\\n//////// Tour Code ////////\\n\\n'\n for tour_step in tour_steps[name]:\n instructions += tour_step\n\n if tour_type == \"bootstrap\":\n instructions += (\n \"\"\"]);\n // Initialize the tour\n tour.init();\n // Start the tour\n tour.start();\n $tour = tour;\n $tour.restart();\\n\"\"\")\n elif tour_type == \"hopscotch\":\n instructions += (\n \"\"\"]\n };\n // Start the tour!\n hopscotch.startTour(tour);\n $tour = hopscotch;\\n\"\"\")\n elif tour_type == \"introjs\":\n instructions += (\n \"\"\"]\n });\n intro.setOption(\"disableInteraction\", true);\n intro.setOption(\"overlayOpacity\", .29);\n intro.setOption(\"scrollToElement\", true);\n intro.setOption(\"keyboardNavigation\", true);\n intro.setOption(\"exitOnEsc\", false);\n intro.setOption(\"exitOnOverlayClick\", false);\n intro.setOption(\"showStepNumbers\", false);\n intro.setOption(\"showProgress\", false);\n intro.start();\n $tour = intro;\n };\n startIntro();\\n\"\"\")\n elif tour_type == \"shepherd\":\n instructions += (\n \"\"\"\n tour.start();\n $tour = tour;\\n\"\"\")\n else:\n pass\n\n exported_tours_folder = EXPORTED_TOURS_FOLDER\n if exported_tours_folder.endswith(\"/\"):\n exported_tours_folder = exported_tours_folder[:-1]\n if not os.path.exists(exported_tours_folder):\n try:\n os.makedirs(exported_tours_folder)\n except Exception:\n pass\n import codecs\n file_path = exported_tours_folder + \"/\" + filename\n out_file = codecs.open(file_path, \"w+\")\n out_file.writelines(instructions)\n out_file.close()\n print('\\n>>> [%s] was saved!\\n' % file_path)"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef reset_downloads_folder():\n ''' Clears the downloads folder.\n If settings.ARCHIVE_EXISTING_DOWNLOADS is set to True, archives it. '''\n if os.path.exists(downloads_path) and not os.listdir(downloads_path) == []:\n archived_downloads_folder = os.path.join(downloads_path, '..',\n ARCHIVE_DIR)\n if not \"\".join(sys.argv) == \"-c\":\n # Only move files if the test run is not multi-threaded.\n # (Running tests with \"-n NUM\" will create threads that only\n # have \"-c\" in the sys.argv list. Easy to catch.)\n reset_downloads_folder_assistant(archived_downloads_folder)", "response": "Clears the downloads folder."} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns whether the specified element selector is present on the page.", "response": "def is_element_present(driver, selector, by=By.CSS_SELECTOR):\n \"\"\"\n Returns whether the specified element selector is present on the page.\n @Params\n driver - the webdriver object (required)\n selector - the locator that is used (required)\n by - the method to search for the locator (Default: By.CSS_SELECTOR)\n @Returns\n Boolean (is element present)\n \"\"\"\n try:\n driver.find_element(by=by, value=selector)\n return True\n except Exception:\n return False"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef is_element_visible(driver, selector, by=By.CSS_SELECTOR):\n try:\n element = driver.find_element(by=by, value=selector)\n return element.is_displayed()\n except Exception:\n return False", "response": "Returns whether the specified element selector is visible on the page."} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn whether the specified text is visible in the specified selector.", "response": "def is_text_visible(driver, text, selector, by=By.CSS_SELECTOR):\n \"\"\"\n Returns whether the specified text is visible in the specified selector.\n @Params\n driver - the webdriver object (required)\n text - the text string to search for\n selector - the locator that is used (required)\n by - the method to search for the locator (Default: By.CSS_SELECTOR)\n @Returns\n Boolean (is text visible)\n \"\"\"\n try:\n element = driver.find_element(by=by, value=selector)\n return element.is_displayed() and text in element.text\n except Exception:\n return False"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef hover_on_element(driver, selector, by=By.CSS_SELECTOR):\n element = driver.find_element(by=by, value=selector)\n hover = ActionChains(driver).move_to_element(element)\n hover.perform()", "response": "Fires the hover event for the specified element by the given selector."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef hover_element_and_click(driver, element, click_selector,\n click_by=By.CSS_SELECTOR,\n timeout=settings.SMALL_TIMEOUT):\n \"\"\"\n Similar to hover_and_click(), but assumes top element is already found.\n \"\"\"\n start_ms = time.time() * 1000.0\n stop_ms = start_ms + (timeout * 1000.0)\n hover = ActionChains(driver).move_to_element(element)\n hover.perform()\n for x in range(int(timeout * 10)):\n try:\n element = driver.find_element(by=click_by, value=click_selector)\n element.click()\n return element\n except Exception:\n now_ms = time.time() * 1000.0\n if now_ms >= stop_ms:\n break\n time.sleep(0.1)\n raise NoSuchElementException(\n \"Element {%s} was not present after %s seconds!\" %\n (click_selector, timeout))", "response": "Hovers an element and clicks it."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nsearch for the specified element by the given selector and returns the element object if it is present on the page.", "response": "def wait_for_element_present(driver, selector, by=By.CSS_SELECTOR,\n timeout=settings.LARGE_TIMEOUT):\n \"\"\"\n Searches for the specified element by the given selector. Returns the\n element object if the element is present on the page. The element can be\n invisible. Raises an exception if the element does not appear in the\n specified timeout.\n @Params\n driver - the webdriver object\n selector - the locator that is used (required)\n by - the method to search for the locator (Default: By.CSS_SELECTOR)\n timeout - the time to wait for elements in seconds\n @Returns\n A web element object\n \"\"\"\n\n element = None\n start_ms = time.time() * 1000.0\n stop_ms = start_ms + (timeout * 1000.0)\n for x in range(int(timeout * 10)):\n try:\n element = driver.find_element(by=by, value=selector)\n return element\n except Exception:\n now_ms = time.time() * 1000.0\n if now_ms >= stop_ms:\n break\n time.sleep(0.1)\n if not element:\n raise NoSuchElementException(\n \"Element {%s} was not present after %s seconds!\" % (\n selector, timeout))"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nsearches for the specified element by the given selector and returns the element object if it is visible on the page.", "response": "def wait_for_element_visible(driver, selector, by=By.CSS_SELECTOR,\n timeout=settings.LARGE_TIMEOUT):\n \"\"\"\n Searches for the specified element by the given selector. Returns the\n element object if the element is present and visible on the page.\n Raises an exception if the element does not appear in the\n specified timeout.\n @Params\n driver - the webdriver object (required)\n selector - the locator that is used (required)\n by - the method to search for the locator (Default: By.CSS_SELECTOR)\n timeout - the time to wait for elements in seconds\n\n @Returns\n A web element object\n \"\"\"\n\n element = None\n start_ms = time.time() * 1000.0\n stop_ms = start_ms + (timeout * 1000.0)\n for x in range(int(timeout * 10)):\n try:\n element = driver.find_element(by=by, value=selector)\n if element.is_displayed():\n return element\n else:\n element = None\n raise Exception()\n except Exception:\n now_ms = time.time() * 1000.0\n if now_ms >= stop_ms:\n break\n time.sleep(0.1)\n plural = \"s\"\n if timeout == 1:\n plural = \"\"\n if not element and by != By.LINK_TEXT:\n raise ElementNotVisibleException(\n \"Element {%s} was not visible after %s second%s!\" % (\n selector, timeout, plural))\n if not element and by == By.LINK_TEXT:\n raise ElementNotVisibleException(\n \"Link text {%s} was not visible after %s second%s!\" % (\n selector, timeout, plural))"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef find_visible_elements(driver, selector, by=By.CSS_SELECTOR):\n elements = driver.find_elements(by=by, value=selector)\n return [element for element in elements if element.is_displayed()]", "response": "Finds all WebElements that match a selector and are visible."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef save_screenshot(driver, name, folder=None):\n if \".\" not in name:\n name = name + \".png\"\n if folder:\n abs_path = os.path.abspath('.')\n file_path = abs_path + \"/%s\" % folder\n if not os.path.exists(file_path):\n os.makedirs(file_path)\n screenshot_path = \"%s/%s\" % (file_path, name)\n else:\n screenshot_path = name\n try:\n element = driver.find_element_by_tag_name('body')\n element_png = element.screenshot_as_png\n with open(screenshot_path, \"wb\") as file:\n file.write(element_png)\n except Exception:\n if driver:\n driver.get_screenshot_as_file(screenshot_path)\n else:\n pass", "response": "Saves a screenshot to the current directory or to a subfolder if provided."} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nwaits for and accept an alert. Returns the text from the alert.", "response": "def wait_for_and_accept_alert(driver, timeout=settings.LARGE_TIMEOUT):\n \"\"\"\n Wait for and accept an alert. Returns the text from the alert.\n @Params\n driver - the webdriver object (required)\n timeout - the time to wait for the alert in seconds\n \"\"\"\n alert = wait_for_and_switch_to_alert(driver, timeout)\n alert_text = alert.text\n alert.accept()\n return alert_text"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nwaiting for and dismiss an alert. Returns the text from the alert.", "response": "def wait_for_and_dismiss_alert(driver, timeout=settings.LARGE_TIMEOUT):\n \"\"\"\n Wait for and dismiss an alert. Returns the text from the alert.\n @Params\n driver - the webdriver object (required)\n timeout - the time to wait for the alert in seconds\n \"\"\"\n alert = wait_for_and_switch_to_alert(driver, timeout)\n alert_text = alert.text\n alert.dismiss()\n return alert_text"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nwait for an alert to appear and switch to it.", "response": "def wait_for_and_switch_to_alert(driver, timeout=settings.LARGE_TIMEOUT):\n \"\"\"\n Wait for a browser alert to appear, and switch to it. This should be usable\n as a drop-in replacement for driver.switch_to.alert when the alert box\n may not exist yet.\n @Params\n driver - the webdriver object (required)\n timeout - the time to wait for the alert in seconds\n \"\"\"\n\n start_ms = time.time() * 1000.0\n stop_ms = start_ms + (timeout * 1000.0)\n for x in range(int(timeout * 10)):\n try:\n alert = driver.switch_to.alert\n # Raises exception if no alert present\n dummy_variable = alert.text # noqa\n return alert\n except NoAlertPresentException:\n now_ms = time.time() * 1000.0\n if now_ms >= stop_ms:\n break\n time.sleep(0.1)\n raise Exception(\"Alert was not present after %s seconds!\" % timeout)"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nswitch to a frame element.", "response": "def switch_to_frame(driver, frame, timeout=settings.SMALL_TIMEOUT):\n \"\"\"\n Wait for an iframe to appear, and switch to it. This should be usable\n as a drop-in replacement for driver.switch_to.frame().\n @Params\n driver - the webdriver object (required)\n frame - the frame element, name, or index\n timeout - the time to wait for the alert in seconds\n \"\"\"\n\n start_ms = time.time() * 1000.0\n stop_ms = start_ms + (timeout * 1000.0)\n for x in range(int(timeout * 10)):\n try:\n driver.switch_to.frame(frame)\n return True\n except NoSuchFrameException:\n now_ms = time.time() * 1000.0\n if now_ms >= stop_ms:\n break\n time.sleep(0.1)\n raise Exception(\"Frame was not present after %s seconds!\" % timeout)"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nwaiting for a window to appear, and switch to it. This should be usable as a drop-in replacement for driver.switch_to.window(). @Params driver - the webdriver object (required) window - the window index or window handle timeout - the time to wait for the window in seconds", "response": "def switch_to_window(driver, window, timeout=settings.SMALL_TIMEOUT):\n \"\"\"\n Wait for a window to appear, and switch to it. This should be usable\n as a drop-in replacement for driver.switch_to.window().\n @Params\n driver - the webdriver object (required)\n window - the window index or window handle\n timeout - the time to wait for the window in seconds\n \"\"\"\n\n start_ms = time.time() * 1000.0\n stop_ms = start_ms + (timeout * 1000.0)\n if isinstance(window, int):\n for x in range(int(timeout * 10)):\n try:\n window_handle = driver.window_handles[window]\n driver.switch_to.window(window_handle)\n return True\n except IndexError:\n now_ms = time.time() * 1000.0\n if now_ms >= stop_ms:\n break\n time.sleep(0.1)\n raise Exception(\"Window was not present after %s seconds!\" % timeout)\n else:\n window_handle = window\n for x in range(int(timeout * 10)):\n try:\n driver.switch_to.window(window_handle)\n return True\n except NoSuchWindowException:\n now_ms = time.time() * 1000.0\n if now_ms >= stop_ms:\n break\n time.sleep(0.1)\n raise Exception(\"Window was not present after %s seconds!\" % timeout)"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nconverting a url like this : url = blog. xkcd. com", "response": "def get_domain_url(url):\n \"\"\"\n Use this to convert a url like this:\n https://blog.xkcd.com/2014/07/22/what-if-book-tour/\n Into this:\n https://blog.xkcd.com\n \"\"\"\n if \"http://\" not in url and \"https://\" not in url:\n return url\n url_header = url.split('://')[0]\n simple_url = url.split('://')[1]\n base_url = simple_url.split('/')[0]\n domain_url = url_header + '://' + base_url\n return domain_url"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef is_xpath_selector(selector):\n if (selector.startswith('/') or selector.startswith('./') or (\n selector.startswith('('))):\n return True\n return False", "response": "A basic method to determine if a selector is an xpath selector."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef get_link_text_from_selector(selector):\n if selector.startswith('link='):\n return selector.split('link=')[1]\n elif selector.startswith('link_text='):\n return selector.split('link_text=')[1]\n return selector", "response": "A basic method to get the link text from a link text selector."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _get_unique_links(page_url, soup):\n if \"http://\" not in page_url and \"https://\" not in page_url:\n return []\n prefix = 'http:'\n if page_url.startswith('https:'):\n prefix = 'https:'\n simple_url = page_url.split('://')[1]\n base_url = simple_url.split('/')[0]\n full_base_url = prefix + \"//\" + base_url\n\n raw_links = []\n raw_unique_links = []\n\n # Get \"href\" from all \"a\" tags\n links = soup.find_all('a')\n for link in links:\n raw_links.append(link.get('href'))\n\n # Get \"src\" from all \"img\" tags\n img_links = soup.find_all('img')\n for img_link in img_links:\n raw_links.append(img_link.get('src'))\n\n # Get \"href\" from all \"link\" tags\n links = soup.find_all('link')\n for link in links:\n raw_links.append(link.get('href'))\n\n # Get \"src\" from all \"script\" tags\n img_links = soup.find_all('script')\n for img_link in img_links:\n raw_links.append(img_link.get('src'))\n\n for link in raw_links:\n if link not in raw_unique_links:\n raw_unique_links.append(link)\n\n unique_links = []\n for link in raw_unique_links:\n if link and len(link) > 1:\n if link.startswith('//'):\n link = prefix + link\n elif link.startswith('/'):\n link = full_base_url + link\n elif link.startswith('./'):\n link = full_base_url + link[1:]\n elif link.startswith('#'):\n link = full_base_url + link\n elif '//' not in link:\n link = full_base_url + \"/\" + link\n else:\n pass\n unique_links.append(link)\n\n return unique_links", "response": "Get all unique links."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nget the status code of a link.", "response": "def _get_link_status_code(link, allow_redirects=False, timeout=5):\n \"\"\" Get the status code of a link.\n If the timeout is exceeded, will return a 404.\n For a list of available status codes, see:\n https://en.wikipedia.org/wiki/List_of_HTTP_status_codes\n \"\"\"\n status_code = None\n try:\n response = requests.get(\n link, allow_redirects=allow_redirects, timeout=timeout)\n status_code = response.status_code\n except Exception:\n status_code = 404\n return status_code"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _print_unique_links_with_status_codes(page_url, soup):\n links = _get_unique_links(page_url, soup)\n for link in links:\n status_code = _get_link_status_code(link)\n print(link, \" -> \", status_code)", "response": "Find all unique links in the html of the page and print out those links with their status codes."} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nconnects to the IMAP mailbox.", "response": "def imap_connect(self):\n \"\"\"\n Connect to the IMAP mailbox.\n \"\"\"\n self.mailbox = imaplib.IMAP4_SSL(self.imap_string, self.port)\n self.mailbox.login(self.uname, self.pwd)\n self.mailbox.select()"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nsearching for a query in the given IMAP criteria and returns a list of message numbers that match the given criteria.", "response": "def __imap_search(self, ** criteria_dict):\n \"\"\" Searches for query in the given IMAP criteria and returns\n the message numbers that match as a list of strings.\n\n Criteria without values (eg DELETED) should be keyword args\n with KEY=True, or else not passed. Criteria with values should\n be keyword args of the form KEY=\"VALUE\" where KEY is a valid\n IMAP key.\n\n IMAP default is to AND all criteria together. We don't support\n other logic quite yet.\n\n All valid keys: ALL, ANSWERED, BCC , BEFORE ,\n BODY , CC , DELETED, DRAFT, FLAGGED, FROM\n , HEADER (UNTESTED), KEYWORD\n , LARGER , NEW, NOT , OLD, ON ,\n OR (UNTESTED), RECENT, SEEN,\n SENTBEFORE , SENTON , SENTSINCE , SINCE ,\n SMALLER , SUBJECT , TEXT , TO ,\n UID , UNANSWERED, UNDELETED, UNDRAFT, UNFLAGGED,\n UNKEYWORD , UNSEEN.\n\n For details on keys and their values, see\n http://tools.ietf.org/html/rfc3501#section-6.4.4\n\n :param criteria_dict: dictionary of search criteria keywords\n :raises: EmailException if something in IMAP breaks\n :returns: List of message numbers as strings matched by given criteria\n \"\"\"\n self.imap_connect()\n\n criteria = []\n for key in criteria_dict:\n if criteria_dict[key] is True:\n criteria.append('(%s)' % key)\n else:\n criteria.append('(%s \"%s\")' % (key, criteria_dict[key]))\n\n # If any of these criteria are not valid IMAP keys, IMAP will tell us.\n status, msg_nums = self.mailbox.search('UTF-8', * criteria)\n self.imap_disconnect()\n\n if 0 == len(msg_nums):\n msg_nums = []\n\n if 'OK' in status:\n return self.__parse_imap_search_result(msg_nums)\n else:\n raise EmailException(\"IMAP status is \" + str(status))"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef __parse_imap_search_result(self, result):\n if isinstance(result, types.ListType):\n # Above is same as \"type(result) == types.ListType\"\n if len(result) == 1:\n return self.__parse_imap_search_result(result[0])\n else:\n return result\n elif isinstance(result, types.StringType):\n # Above is same as \"type(result) == types.StringType\"\n return result.split()\n else:\n # Fail silently assuming tests will fail if emails are not found\n return []", "response": "This takes the result of an IMAP search and returns SANE results\n "} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ngive a message number that we found with imap_search fetch the html content.", "response": "def fetch_html(self, msg_nums):\n \"\"\"\n Given a message number that we found with imap_search,\n get the text/html content.\n @Params\n msg_nums - message number to get html message for\n @Returns\n HTML content of message matched by message number\n \"\"\"\n if not msg_nums:\n raise Exception(\"Invalid Message Number!\")\n\n return self.__imap_fetch_content_type(msg_nums, self.HTML)"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef fetch_plaintext(self, msg_nums):\n if not msg_nums:\n raise Exception(\"Invalid Message Number!\")\n\n return self.__imap_fetch_content_type(msg_nums, self.PLAIN)", "response": "This method returns the plain text content of a message number."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef __imap_fetch_content_type(self, msg_nums, content_type):\n\n if not msg_nums:\n raise Exception(\"Invalid Message Number!\")\n if not content_type:\n raise Exception(\"Need a content type!\")\n\n contents = {}\n self.imap_connect()\n for num in msg_nums:\n status, data = self.mailbox.fetch(num, \"(RFC822)\")\n for response_part in data:\n if isinstance(response_part, tuple):\n msg = email.message_from_string(response_part[1])\n for part in msg.walk():\n if str(part.get_content_type()) == content_type:\n content = str(part.get_payload(decode=True))\n contents[int(num)] = content\n self.imap_disconnect()\n return contents", "response": "Fetch the contents of a message number that matches a content type."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ngetting the html of an email by subject.", "response": "def fetch_html_by_subject(self, email_name):\n \"\"\"\n Get the html of an email, searching by subject.\n @Params\n email_name - the subject to search for\n @Returns\n HTML content of the matched email\n \"\"\"\n if not email_name:\n raise EmailException(\"Subject cannot be null\")\n\n results = self.__imap_search(SUBJECT=email_name)\n sources = self.fetch_html(results)\n\n return sources"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef fetch_plaintext_by_subject(self, email_name):\n if not email_name:\n raise EmailException(\"Subject cannot be null\")\n\n results = self.__imap_search(SUBJECT=email_name)\n sources = self.fetch_plaintext(results)\n\n return sources", "response": "Returns the plain text of an email by subject."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef search_for_recipient(self, email, timeout=None, content_type=None):\n return self.search(timeout=timeout,\n content_type=content_type, TO=email)", "response": "Search for a specific recipient in the cache."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef search_for_subject(self, subject, timeout=None, content_type=None):\n return self.search(timeout=timeout,\n content_type=content_type, SUBJECT=subject)", "response": "Search for a specific subject in the cache."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef search_for_count(self, ** args):\n if \"timeout\" not in args.keys():\n timeout = self.TIMEOUT\n elif args[\"timeout\"]:\n timeout = args[\"timeout\"]\n args[\"timeout\"] = timeout / 15\n\n if \"count\" not in args.keys():\n raise EmailException(\"Count param not defined!\")\n else:\n count = int(args[\"count\"])\n del args[\"count\"]\n\n results = None\n timer = timeout\n count = 0\n while count < timer:\n try:\n results = self.search(** args)\n except EmailException:\n if count == 0:\n return []\n\n if results and len(results) == count:\n return results\n else:\n time.sleep(15)\n count += 15\n if count >= timer:\n raise EmailException(\"Failed to match criteria %s in %s minutes\" %\n (args, timeout / 60))", "response": "A search that keeps searching up to timeout for a specific number of matches to a search."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncheck an Email. Message object for the headers in email_headers.", "response": "def __check_msg_for_headers(self, msg, ** email_headers):\n \"\"\"\n Checks an Email.Message object for the headers in email_headers.\n\n Following are acceptable header names: ['Delivered-To',\n 'Received', 'Return-Path', 'Received-SPF',\n 'Authentication-Results', 'DKIM-Signature',\n 'DomainKey-Signature', 'From', 'To', 'Message-ID',\n 'Subject', 'MIME-Version', 'Content-Type', 'Date',\n 'X-Sendgrid-EID', 'Sender'].\n\n @Params\n msg - the Email.message object to check\n email_headers - list of headers to check against\n @Returns\n Boolean whether all the headers were found\n \"\"\"\n all_headers_found = False\n email_headers['Delivered-To'] = email_headers['To']\n email_headers.pop('To')\n all_headers_found = all(k in msg.keys() for k in email_headers)\n\n return all_headers_found"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef fetch_message(self, msgnum):\n self.imap_connect()\n status, data = self.mailbox.fetch(msgnum, \"(RFC822)\")\n self.imap_disconnect()\n\n for response_part in data:\n if isinstance(response_part, tuple):\n return email.message_from_string(response_part[1])", "response": "Given a message number return the Email. Message object."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_content_type(self, msg, content_type=\"HTML\"):\n if \"HTML\" in content_type.upper():\n content_type = self.HTML\n elif \"PLAIN\" in content_type.upper():\n content_type = self.PLAIN\n\n for part in msg.walk():\n if str(part.get_content_type()) == content_type:\n return str(part.get_payload(decode=True))", "response": "Gets the content - type payload for the given content - type."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nsearching email inbox for messages that match the criteria.", "response": "def search(self, ** args):\n \"\"\"\n Checks email inbox every 15 seconds that match the criteria\n up until timeout.\n\n Search criteria should be keyword args eg\n TO=\"selenium@gmail.com\". See __imap_search docstring for list\n of valid criteria. If content_type is not defined, will return\n a list of msg numbers.\n\n Options:\n - fetch: will return a dict of Message objects, keyed on msgnum,\n which can be used to look at headers and other parts of the complete\n message. (http://docs.python.org/library/email.message.html)\n - timeout: will replace the default module timeout with the\n value in SECONDS.\n - content_type: should be either \"PLAIN\" or\n \"HTML\". If defined returns the source of the matched messages\n as a dict of msgnum:content. If not defined we return a list\n of msg nums.\n \"\"\"\n\n if \"content_type\" not in args.keys():\n content_type = None\n elif \"HTML\" in args[\"content_type\"]:\n content_type = self.HTML\n del args[\"content_type\"]\n elif \"PLAIN\" in args[\"content_type\"]:\n content_type = self.PLAIN\n del args[\"content_type\"]\n elif args[\"content_type\"]:\n content_type = args['content_type']\n del args[\"content_type\"]\n\n if \"timeout\" not in args.keys():\n timeout = self.TIMEOUT\n elif \"timeout\" in args:\n timeout = args[\"timeout\"]\n del args[\"timeout\"]\n\n fetch = False\n if \"fetch\" in args.keys():\n fetch = True\n del args[\"fetch\"]\n\n results = None\n timer = timeout\n count = 0\n while count < timer:\n results = self.__imap_search(** args)\n if len(results) > 0:\n if fetch:\n msgs = {}\n for msgnum in results:\n msgs[msgnum] = self.fetch_message(msgnum)\n return msgs\n elif not content_type:\n return results\n else:\n return self.__imap_fetch_content_type(results,\n content_type)\n else:\n time.sleep(15)\n count += 15\n if count >= timer:\n raise EmailException(\n \"Failed to find message for criteria %s in %s minutes\" %\n (args, timeout / 60))"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef remove_whitespace(self, html):\n # Does python have a better way to do exactly this?\n clean_html = html\n for char in (\"\\r\", \"\\n\", \"\\t\"):\n clean_html = clean_html.replace(char, \"\")\n return clean_html", "response": "Removes whitespace from html."} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreplace entities with unicode characters in the given html source.", "response": "def replace_entities(self, html):\n \"\"\"\n Replace htmlentities with unicode characters\n @Params\n html - html source to replace entities in\n @Returns\n String html with entities replaced\n \"\"\"\n def fixup(text):\n \"\"\"replace the htmlentities in some text\"\"\"\n text = text.group(0)\n if text[:2] == \"&#\":\n # character reference\n try:\n if text[:3] == \"&#x\":\n return chr(int(text[3:-1], 16))\n else:\n return chr(int(text[2:-1]))\n except ValueError:\n pass\n else:\n # named entity\n try:\n text = chr(htmlentitydefs.name2codepoint[text[1:-1]])\n except KeyError:\n pass\n return text # leave as is\n return re.sub(r\"&#?\\w+;\", fixup, html)"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef wait_for_ready_state_complete(driver, timeout=settings.EXTREME_TIMEOUT):\n\n start_ms = time.time() * 1000.0\n stop_ms = start_ms + (timeout * 1000.0)\n for x in range(int(timeout * 10)):\n try:\n ready_state = driver.execute_script(\"return document.readyState\")\n except WebDriverException:\n # Bug fix for: [Permission denied to access property \"document\"]\n time.sleep(0.03)\n return True\n if ready_state == u'complete':\n time.sleep(0.01) # Better be sure everything is done loading\n return True\n else:\n now_ms = time.time() * 1000.0\n if now_ms >= stop_ms:\n break\n time.sleep(0.1)\n raise Exception(\n \"Page elements never fully loaded after %s seconds!\" % timeout)", "response": "Waits until the page elements are loaded."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef activate_jquery(driver):\n try:\n # Let's first find out if jQuery is already defined.\n driver.execute_script(\"jQuery('html')\")\n # Since that command worked, jQuery is defined. Let's return.\n return\n except Exception:\n # jQuery is not currently defined. Let's proceed by defining it.\n pass\n jquery_js = constants.JQuery.MIN_JS\n activate_jquery_script = (\n '''var script = document.createElement('script');'''\n '''script.src = \"%s\";document.getElementsByTagName('head')[0]'''\n '''.appendChild(script);''' % jquery_js)\n driver.execute_script(activate_jquery_script)\n for x in range(int(settings.MINI_TIMEOUT * 10.0)):\n # jQuery needs a small amount of time to activate.\n try:\n driver.execute_script(\"jQuery('html')\")\n return\n except Exception:\n time.sleep(0.1)\n # Since jQuery still isn't activating, give up and raise an exception\n raise Exception(\n '''Unable to load jQuery on \"%s\" due to a possible violation '''\n '''of the website's Content Security Policy directive. '''\n '''To override this policy, add \"--disable_csp\" on the '''\n '''command-line when running your tests.''' % driver.current_url)", "response": "Activates jQuery and returns the current version of the page."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef escape_quotes_if_needed(string):\n if are_quotes_escaped(string):\n if string.count(\"'\") != string.count(\"\\\\'\"):\n string = string.replace(\"'\", \"\\\\'\")\n if string.count('\"') != string.count('\\\\\"'):\n string = string.replace('\"', '\\\\\"')\n return string", "response": "Escape quotes in a string if needed."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef safe_execute_script(driver, script):\n try:\n driver.execute_script(script)\n except Exception:\n # The likely reason this fails is because: \"jQuery is not defined\"\n activate_jquery(driver) # It's a good thing we can define it here\n driver.execute_script(script)", "response": "This method will execute a script that contains a jQuery command."} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\npost a message on the screen with Messenger.", "response": "def post_message(driver, message, msg_dur, style=\"info\"):\n \"\"\" A helper method to post a message on the screen with Messenger.\n (Should only be called from post_message() in base_case.py) \"\"\"\n if not msg_dur:\n msg_dur = settings.DEFAULT_MESSAGE_DURATION\n msg_dur = float(msg_dur)\n message = re.escape(message)\n message = escape_quotes_if_needed(message)\n messenger_script = ('''Messenger().post({message: \"%s\", type: \"%s\", '''\n '''hideAfter: %s, hideOnNavigate: true});'''\n % (message, style, msg_dur))\n try:\n driver.execute_script(messenger_script)\n except Exception:\n activate_messenger(driver)\n set_messenger_theme(driver)\n try:\n driver.execute_script(messenger_script)\n except Exception:\n time.sleep(0.2)\n activate_messenger(driver)\n time.sleep(0.2)\n set_messenger_theme(driver)\n time.sleep(0.5)\n driver.execute_script(messenger_script)"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _jq_format(code):\n code = code.replace('\\\\', '\\\\\\\\').replace('\\t', '\\\\t').replace('\\n', '\\\\n')\n code = code.replace('\\\"', '\\\\\\\"').replace('\\'', '\\\\\\'')\n code = code.replace('\\v', '\\\\v').replace('\\a', '\\\\a').replace('\\f', '\\\\f')\n code = code.replace('\\b', '\\\\b').replace(r'\\u', '\\\\u').replace('\\r', '\\\\r')\n return code", "response": "Convert a raw javascript code into a JQ - style string."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef __log_all_options_if_none_specified(self, test):\n if ((not self.options.enable_plugin_basic_test_info) and (\n not self.options.enable_plugin_screen_shots) and (\n not self.options.enable_plugin_page_source)):\n test_logpath = self.options.log_path + \"/\" + test.id()\n log_helper.log_screenshot(test_logpath, test.driver)\n log_helper.log_test_failure_data(\n test, test_logpath, test.driver, test.browser)\n log_helper.log_page_source(test_logpath, test.driver)", "response": "Log all options if none are specified."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef addError(self, test, err, capt=None):\n if (err[0] == errors.BlockedTest or (\n err[0] == errors.SkipTest) or (\n err[0] == errors.DeprecatedTest)):\n print(err[1].__str__().split('''-------------------- >> '''\n '''begin captured logging'''\n ''' << --------------------''', 1)[0])\n else:\n # self.__log_all_options_if_none_specified(test)\n pass\n self.add_fails_or_errors(test)", "response": "Add an error to the nose log."} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nexecute a db query gets all the values", "response": "def query_fetch_all(self, query, values):\n \"\"\"\n Executes a db query, gets all the values, and closes the connection.\n \"\"\"\n self.cursor.execute(query, values)\n retval = self.cursor.fetchall()\n self.__close_db()\n return retval"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nexecute a db query gets the first value", "response": "def query_fetch_one(self, query, values):\n \"\"\"\n Executes a db query, gets the first value, and closes the connection.\n \"\"\"\n self.cursor.execute(query, values)\n retval = self.cursor.fetchone()\n self.__close_db()\n return retval"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef execute_query(self, query, values):\n retval = self.cursor.execute(query, values)\n self.__close_db()\n return retval", "response": "Executes a query to the test_db and closes the connection afterwards."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncombines the domain base href with the html source.", "response": "def get_html_source_with_base_href(driver, page_source):\n ''' Combines the domain base href with the html source.\n This is needed for the page html to render correctly. '''\n last_page = get_last_page(driver)\n if '://' in last_page:\n base_href_html = get_base_href_html(last_page)\n return '%s\\n%s' % (base_href_html, page_source)\n return ''"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef log_folder_setup(log_path, archive_logs=False):\n if log_path.endswith(\"/\"):\n log_path = log_path[:-1]\n if not os.path.exists(log_path):\n try:\n os.makedirs(log_path)\n except Exception:\n pass # Should only be reachable during multi-threaded runs\n else:\n archived_folder = \"%s/../archived_logs/\" % log_path\n if not os.path.exists(archived_folder):\n try:\n os.makedirs(archived_folder)\n except Exception:\n pass # Should only be reachable during multi-threaded runs\n if not \"\".join(sys.argv) == \"-c\":\n # Only move log files if the test run is not multi-threaded.\n # (Running tests with \"-n NUM\" will create threads that only\n # have \"-c\" in the sys.argv list. Easy to catch.)\n archived_logs = \"%slogs_%s\" % (\n archived_folder, int(time.time()))\n shutil.move(log_path, archived_logs)\n os.makedirs(log_path)\n if not settings.ARCHIVE_EXISTING_LOGS and not archive_logs:\n shutil.rmtree(archived_logs)", "response": "Setup the log folder."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef download_selenium_server():\n try:\n local_file = open(JAR_FILE, 'wb')\n remote_file = urlopen(SELENIUM_JAR)\n print('Downloading the Selenium Server JAR file...\\n')\n local_file.write(remote_file.read())\n local_file.close()\n remote_file.close()\n print('Download Complete!')\n except Exception:\n raise Exception(\"Error downloading the Selenium Server JAR file.\\n\"\n \"Details: %s\" % sys.exc_info()[1])", "response": "Downloads the Selenium Server JAR file from its\n online location and stores it locally."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef addSuccess(self, test, capt):\n self.__insert_test_result(constants.State.PASS, test)", "response": "Add a testcase as a success."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef addError(self, test, err, capt=None):\n self.__insert_test_result(constants.State.ERROR, test, err)", "response": "Add an error to the testcase."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nadding a testcase failure.", "response": "def addFailure(self, test, err, capt=None, tbinfo=None):\n \"\"\"\n After a test failure, we want to record testcase run information.\n \"\"\"\n self.__insert_test_result(constants.State.FAILURE, test, err)"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nadd a chrome proxy extension to the options.", "response": "def _add_chrome_proxy_extension(\n chrome_options, proxy_string, proxy_user, proxy_pass):\n \"\"\" Implementation of https://stackoverflow.com/a/35293284 for\n https://stackoverflow.com/questions/12848327/\n (Run Selenium on a proxy server that requires authentication.) \"\"\"\n if not \"\".join(sys.argv) == \"-c\":\n # Single-threaded\n proxy_helper.create_proxy_zip(proxy_string, proxy_user, proxy_pass)\n else:\n # Pytest multi-threaded test\n lock = threading.Lock()\n with lock:\n time.sleep(random.uniform(0.02, 0.15))\n if not os.path.exists(PROXY_ZIP_PATH):\n proxy_helper.create_proxy_zip(\n proxy_string, proxy_user, proxy_pass)\n time.sleep(random.uniform(0.1, 0.2))\n proxy_zip = PROXY_ZIP_PATH\n if not os.path.exists(PROXY_ZIP_PATH):\n # Handle \"Permission denied\" on the default proxy.zip path\n proxy_zip = PROXY_ZIP_PATH_2\n chrome_options.add_extension(proxy_zip)\n return chrome_options"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_local_driver(\n browser_name, headless, proxy_string, proxy_auth,\n proxy_user, proxy_pass, user_agent, disable_csp):\n '''\n Spins up a new web browser and returns the driver.\n Can also be used to spin up additional browsers for the same test.\n '''\n downloads_path = download_helper.get_downloads_folder()\n download_helper.reset_downloads_folder()\n\n if browser_name == constants.Browser.FIREFOX:\n try:\n try:\n # Use Geckodriver for Firefox if it's on the PATH\n profile = _create_firefox_profile(\n downloads_path, proxy_string, user_agent, disable_csp)\n firefox_capabilities = DesiredCapabilities.FIREFOX.copy()\n firefox_capabilities['marionette'] = True\n options = webdriver.FirefoxOptions()\n if headless:\n options.add_argument('-headless')\n if LOCAL_GECKODRIVER and os.path.exists(LOCAL_GECKODRIVER):\n make_driver_executable_if_not(LOCAL_GECKODRIVER)\n firefox_driver = webdriver.Firefox(\n firefox_profile=profile,\n capabilities=firefox_capabilities,\n options=options,\n executable_path=LOCAL_GECKODRIVER)\n else:\n firefox_driver = webdriver.Firefox(\n firefox_profile=profile,\n capabilities=firefox_capabilities,\n options=options)\n except WebDriverException:\n # Don't use Geckodriver: Only works for old versions of Firefox\n profile = _create_firefox_profile(\n downloads_path, proxy_string, user_agent, disable_csp)\n firefox_capabilities = DesiredCapabilities.FIREFOX.copy()\n firefox_capabilities['marionette'] = False\n firefox_driver = webdriver.Firefox(\n firefox_profile=profile, capabilities=firefox_capabilities)\n return firefox_driver\n except Exception as e:\n if headless:\n raise Exception(e)\n return webdriver.Firefox()\n elif browser_name == constants.Browser.INTERNET_EXPLORER:\n if not IS_WINDOWS:\n raise Exception(\n \"IE Browser is for Windows-based operating systems only!\")\n from selenium.webdriver.ie.options import Options\n ie_options = Options()\n ie_options.ignore_protected_mode_settings = False\n ie_options.ignore_zoom_level = True\n ie_options.require_window_focus = False\n ie_options.native_events = True\n ie_options.full_page_screenshot = True\n ie_options.persistent_hover = True\n ie_capabilities = ie_options.to_capabilities()\n if LOCAL_IEDRIVER and os.path.exists(LOCAL_IEDRIVER):\n make_driver_executable_if_not(LOCAL_IEDRIVER)\n return webdriver.Ie(\n capabilities=ie_capabilities,\n executable_path=LOCAL_IEDRIVER)\n else:\n return webdriver.Ie(capabilities=ie_capabilities)\n elif browser_name == constants.Browser.EDGE:\n if not IS_WINDOWS:\n raise Exception(\n \"Edge Browser is for Windows-based operating systems only!\")\n edge_capabilities = DesiredCapabilities.EDGE.copy()\n if LOCAL_EDGEDRIVER and os.path.exists(LOCAL_EDGEDRIVER):\n make_driver_executable_if_not(LOCAL_EDGEDRIVER)\n return webdriver.Edge(\n capabilities=edge_capabilities,\n executable_path=LOCAL_EDGEDRIVER)\n else:\n return webdriver.Edge(capabilities=edge_capabilities)\n elif browser_name == constants.Browser.SAFARI:\n return webdriver.Safari()\n elif browser_name == constants.Browser.OPERA:\n if LOCAL_OPERADRIVER and os.path.exists(LOCAL_OPERADRIVER):\n make_driver_executable_if_not(LOCAL_OPERADRIVER)\n return webdriver.Opera(executable_path=LOCAL_OPERADRIVER)\n else:\n return webdriver.Opera()\n elif browser_name == constants.Browser.PHANTOM_JS:\n with warnings.catch_warnings():\n # Ignore \"PhantomJS has been deprecated\" UserWarning\n warnings.simplefilter(\"ignore\", category=UserWarning)\n return webdriver.PhantomJS()\n elif browser_name == constants.Browser.GOOGLE_CHROME:\n try:\n chrome_options = _set_chrome_options(\n downloads_path, headless, proxy_string, proxy_auth,\n proxy_user, proxy_pass, user_agent, disable_csp)\n if headless:\n # Headless Chrome doesn't support extensions, which are\n # required when using a proxy server that has authentication.\n # Instead, base_case.py will use PyVirtualDisplay when not\n # using Chrome's built-in headless mode. See link for details:\n # https://bugs.chromium.org/p/chromium/issues/detail?id=706008\n if not proxy_auth:\n chrome_options.add_argument(\"--headless\")\n chrome_options.add_argument(\"--disable-gpu\")\n chrome_options.add_argument(\"--no-sandbox\")\n if LOCAL_CHROMEDRIVER and os.path.exists(LOCAL_CHROMEDRIVER):\n make_driver_executable_if_not(LOCAL_CHROMEDRIVER)\n return webdriver.Chrome(\n executable_path=LOCAL_CHROMEDRIVER, options=chrome_options)\n else:\n return webdriver.Chrome(options=chrome_options)\n except Exception as e:\n if headless:\n raise Exception(e)\n if LOCAL_CHROMEDRIVER and os.path.exists(LOCAL_CHROMEDRIVER):\n make_driver_executable_if_not(LOCAL_CHROMEDRIVER)\n return webdriver.Chrome(executable_path=LOCAL_CHROMEDRIVER)\n else:\n return webdriver.Chrome()\n else:\n raise Exception(\n \"%s is not a valid browser option for this system!\" % browser_name)", "response": "Returns a local driver for the given browser name."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef configure(self, options, conf):\n super(S3Logging, self).configure(options, conf)\n self.options = options", "response": "Configure the logging options."} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns the serializer instance that should be used for validating and deserializing input and serializing output.", "response": "def get_serializer(self, *args, **kwargs):\n \"\"\"\n Return the serializer instance that should be used for validating and\n deserializing input, and for serializing output.\n \"\"\"\n serializer_class = self.get_serializer_class()\n kwargs['context'] = self.get_serializer_context()\n return serializer_class(*args, **kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef authenticate_credentials(self, payload):\n User = get_user_model()\n username = jwt_get_username_from_payload(payload)\n\n if not username:\n msg = _('Invalid payload.')\n raise exceptions.AuthenticationFailed(msg)\n\n try:\n user = User.objects.get_by_natural_key(username)\n except User.DoesNotExist:\n msg = _('Invalid signature.')\n raise exceptions.AuthenticationFailed(msg)\n\n if not user.is_active:\n msg = _('User account is disabled.')\n raise exceptions.AuthenticationFailed(msg)\n\n return user", "response": "Returns an active user that matches the payload s user id and email."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _parse_row(rowvalues, rowtypes):\n n = len(rowvalues)\n assert n == len(rowtypes)\n if not n:\n return []\n range_start = None\n ranges = []\n for i in range(n):\n ctype = rowtypes[i]\n cval = rowvalues[i]\n # Check whether the cell is empty or not. If it is empty, and there is\n # an active range being tracked - terminate it. On the other hand, if\n # the cell is not empty and there isn't an active range, then start it.\n if ctype == 0 or ctype == 6 or (ctype == 1 and\n (cval == \"\" or cval.isspace())):\n if range_start is not None:\n ranges.append((range_start, i))\n range_start = None\n else:\n if range_start is None:\n range_start = i\n if range_start is not None:\n ranges.append((range_start, n))\n return ranges", "response": "Scan a single row from an Excel file and return the list of ranges corresponding to each consecutive span of non - empty cells in this row."} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ngives a list of ranges find those 2d - ranges that overlap with the ja - th range and merge them into the ja - th range.", "response": "def _collapse_ranges(ranges, ja):\n \"\"\"\n Within the `ranges` list find those 2d-ranges that overlap with `ranges[ja]`\n and merge them into `ranges[ja]`. Finally, return the new index of the\n ja-th range within the `ranges` list.\n \"\"\"\n arow0, _, acol0, acol1 = ranges[ja]\n jb = 0\n while jb < len(ranges):\n if jb == ja:\n jb += 1\n continue\n brow0, brow1, bcol0, bcol1 = ranges[jb]\n if bcol0 <= acol1 and brow1 >= arow0 and \\\n not(bcol0 == acol1 and brow1 == arow0):\n ranges[ja][0] = arow0 = min(arow0, brow0)\n ranges[ja][3] = acol1 = max(acol1, bcol1)\n del ranges[jb]\n if jb < ja:\n ja -= 1\n else:\n jb += 1\n return ja"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef find_linked_dynamic_libraries():\n with TaskContext(\"Find the required dynamic libraries\") as log:\n llvm = get_llvm()\n libs = required_link_libraries()\n resolved = []\n for libname in libs:\n if llvm:\n fullpath = os.path.join(llvm, \"lib\", libname)\n if os.path.isfile(fullpath):\n resolved.append(fullpath)\n log.info(\"Library `%s` found at %s\" % (libname, fullpath))\n continue\n else:\n log.info(\"%s does not exist\" % fullpath)\n # Rely on the shell `locate` command to find the dynamic libraries.\n proc = subprocess.Popen([\"locate\", libname], stdout=subprocess.PIPE,\n stderr=subprocess.PIPE)\n stdout, stderr = proc.communicate()\n if proc.returncode == 0:\n results = stdout.decode().strip().split(\"\\n\")\n results = [r for r in results if r]\n if results:\n results.sort(key=len)\n fullpath = results[0]\n assert os.path.isfile(fullpath), \"Invalid path: %r\" % (fullpath,)\n resolved.append(fullpath)\n log.info(\"Library `%s` found at %s\" % (libname, fullpath))\n continue\n else:\n log.fatal(\"Cannot locate dynamic library `%s`\" % libname)\n else:\n log.fatal(\"`locate` command returned the following error:\\n%s\"\n % stderr.decode())\n return resolved", "response": "This function attempts to locate the required link libraries and returns a list of absolute paths."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef wait_for_keypresses(self, refresh_rate=1):\n if not self._enable_keyboard:\n return\n with self._blessed_term.cbreak():\n while True:\n yield self._blessed_term.inkey(timeout=refresh_rate)", "response": "Yields keystrokes from the user s keyboard and returns them to caller."} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nnormalize a selector n - element object into a tuple containing start count and step.", "response": "def normalize_slice(e, n):\n \"\"\"\n Return the slice tuple normalized for an ``n``-element object.\n\n :param e: a slice object representing a selector\n :param n: number of elements in a sequence to which ``e`` is applied\n :returns: tuple ``(start, count, step)`` derived from ``e``.\n \"\"\"\n if n == 0:\n return (0, 0, 1)\n\n step = e.step\n if step is None:\n step = 1\n if step == 0:\n start = e.start\n count = e.stop\n if isinstance(start, int) and isinstance(count, int) and count >= 0:\n if start < 0:\n start += n\n if start < 0:\n return (0, 0, 0)\n return (start, count, 0)\n else:\n raise ValueError(\"Invalid slice %r\" % e)\n assert isinstance(step, int) and step != 0\n\n if e.start is None:\n start = 0 if step > 0 else n - 1\n else:\n start = e.start\n if start < 0:\n start += n\n if (start < 0 and step < 0) or (start >= n and step > 0):\n return (0, 0, 0)\n start = min(max(0, start), n - 1)\n assert isinstance(start, int) and 0 <= start < n, \\\n \"Invalid start: %r\" % start\n\n if e.stop is None:\n if step > 0:\n count = (n - 1 - start) // step + 1\n else:\n count = (start // -step) + 1\n else:\n stop = e.stop\n if stop < 0:\n stop += n\n if step > 0:\n if stop > start:\n count = (min(n, stop) - 1 - start) // step + 1\n else:\n count = 0\n else:\n if stop < start:\n count = (start - max(stop, -1) - 1) // -step + 1\n else:\n count = 0\n assert isinstance(count, int) and count >= 0\n assert count == 0 or 0 <= start + step * (count - 1) < n, \\\n \"Wrong tuple: (%d, %d, %d)\" % (start, count, step)\n\n return (start, count, step)"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nnormalize a range object to produce a tuple of size n.", "response": "def normalize_range(e, n):\n \"\"\"\n Return the range tuple normalized for an ``n``-element object.\n\n The semantics of a range is slightly different than that of a slice.\n In particular, a range is similar to a list in meaning (and on Py2 it was\n eagerly expanded into a list). Thus we do not allow the range to generate\n indices that would be invalid for an ``n``-array. Furthermore, we restrict\n the range to produce only positive or only negative indices. For example,\n ``range(2, -2, -1)`` expands into ``[2, 1, 0, -1]``, and it is confusing\n to treat the last \"-1\" as the last element in the list.\n\n :param e: a range object representing a selector\n :param n: number of elements in a sequence to which ``e`` is applied\n :returns: tuple ``(start, count, step)`` derived from ``e``, or None\n if the range is invalid.\n \"\"\"\n if e.step > 0:\n count = max(0, (e.stop - e.start - 1) // e.step + 1)\n else:\n count = max(0, (e.start - e.stop - 1) // -e.step + 1)\n\n if count == 0:\n return (0, 0, e.step)\n\n start = e.start\n finish = e.start + (count - 1) * e.step\n if start >= 0:\n if start >= n or finish < 0 or finish >= n:\n return None\n else:\n start += n\n finish += n\n if start < 0 or start >= n or finish < 0 or finish >= n:\n return None\n assert count >= 0\n return (start, count, e.step)"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nimporting and returns the requested module.", "response": "def load_module(module):\n \"\"\"\n Import and return the requested module.\n \"\"\"\n try:\n m = importlib.import_module(module)\n return m\n except ModuleNotFoundError: # pragma: no cover\n raise TImportError(\"Module `%s` is not installed. It is required for \"\n \"running this function.\" % module)"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef humanize_bytes(size):\n if size == 0: return \"0\"\n if size is None: return \"\"\n assert size >= 0, \"`size` cannot be negative, got %d\" % size\n suffixes = \"TGMK\"\n maxl = len(suffixes)\n for i in range(maxl + 1):\n shift = (maxl - i) * 10\n if size >> shift == 0: continue\n ndigits = 0\n for nd in [3, 2, 1]:\n if size >> (shift + 12 - nd * 3) == 0:\n ndigits = nd\n break\n if ndigits == 0 or size == (size >> shift) << shift:\n rounded_val = str(size >> shift)\n else:\n rounded_val = \"%.*f\" % (ndigits, size / (1 << shift))\n return \"%s%sB\" % (rounded_val, suffixes[i] if i < maxl else \"\")", "response": "Convert given number of bytes into a human - readable form."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nretrieve frame data within the current view window. This method will adjust the view window if it goes out-of-bounds.", "response": "def _fetch_data(self):\n \"\"\"\n Retrieve frame data within the current view window.\n\n This method will adjust the view window if it goes out-of-bounds.\n \"\"\"\n self._view_col0 = clamp(self._view_col0, 0, self._max_col0)\n self._view_row0 = clamp(self._view_row0, 0, self._max_row0)\n self._view_ncols = clamp(self._view_ncols, 0,\n self._conn.frame_ncols - self._view_col0)\n self._view_nrows = clamp(self._view_nrows, 0,\n self._conn.frame_nrows - self._view_row0)\n self._conn.fetch_data(\n self._view_row0,\n self._view_row0 + self._view_nrows,\n self._view_col0,\n self._view_col0 + self._view_ncols)"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get_files():\n sources = []\n headers = [\"datatable/include/datatable.h\"]\n assert os.path.isfile(headers[0])\n for dirpath, _, filenames in os.walk(\"c\"):\n for f in filenames:\n fullname = os.path.join(dirpath, f)\n if f.endswith(\".h\") or f.endswith(\".inc\"):\n headers.append(fullname)\n elif f.endswith(\".c\") or f.endswith(\".cc\"):\n sources.append(fullname)\n return (sources, headers)", "response": "Return the list of all source and header files in c folder."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef find_includes(filename):\n includes = []\n with open(filename, \"r\", encoding=\"utf-8\") as inp:\n for line in inp:\n line = line.strip()\n if not line or line.startswith(\"//\"):\n continue\n if line.startswith(\"#\"):\n mm = re.match(rx_include, line)\n if mm:\n includename = os.path.join(\"c\", mm.group(1))\n includes.append(includename)\n return includes", "response": "Find user includes requested from given source file."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef build_headermap(headers):\n # TODO: what happens if some headers are circularly dependent?\n headermap = {}\n for hfile in headers:\n headermap[hfile] = None\n for hfile in headers:\n assert (hfile.startswith(\"c/\") or\n hfile.startswith(\"datatable/include/\"))\n inc = find_includes(hfile)\n for f in inc:\n assert f != hfile, \"File %s includes itself?\" % f\n assert f.startswith(\"c/\")\n if f not in headers:\n raise ValueError(\"Unknown header \\\"%s\\\" included from %s\"\n % (f, hfile))\n headermap[hfile] = set(inc)\n return headermap", "response": "Builds a dictionary that contains all the files included in the header files."} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nbuild a dictionary of includes from the sources.", "response": "def build_sourcemap(sources):\n \"\"\"\n Similar to build_headermap(), but builds a dictionary of includes from\n the \"source\" files (i.e. \".c/.cc\" files).\n \"\"\"\n sourcemap = {}\n for sfile in sources:\n inc = find_includes(sfile)\n sourcemap[sfile] = set(inc)\n return sourcemap"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef get_c_sources(folder, include_headers=False):\n allowed_extensions = [\".c\", \".C\", \".cc\", \".cpp\", \".cxx\", \".c++\"]\n if include_headers:\n allowed_extensions += [\".h\", \".hpp\"]\n sources = []\n for root, _, files in os.walk(folder):\n for name in files:\n ext = os.path.splitext(name)[1]\n if name == \"types.cc\":\n # Make sure `types.cc` is compiled first, as it has multiple\n # useful static assertions.\n sources.insert(0, os.path.join(root, name))\n elif ext in allowed_extensions:\n sources.append(os.path.join(root, name))\n return sources", "response": "Find all C ++ source files in the folder."} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ninvoking from the C level, this function will return either the name of the folder where the datatable is to be saved; or None, indicating that the datatable should be read into RAM. This function may also raise an exception if it determines that it cannot find a good strategy to handle a dataset of the requested size.", "response": "def _get_destination(self, estimated_size):\n \"\"\"\n Invoked from the C level, this function will return either the name of\n the folder where the datatable is to be saved; or None, indicating that\n the datatable should be read into RAM. This function may also raise an\n exception if it determines that it cannot find a good strategy to\n handle a dataset of the requested size.\n \"\"\"\n global _psutil_load_attempted\n if not _psutil_load_attempted:\n _psutil_load_attempted = True\n try:\n import psutil\n except ImportError:\n psutil = None\n\n if self.verbose and estimated_size > 1:\n self.logger.debug(\"The Frame is estimated to require %s bytes\"\n % humanize_bytes(estimated_size))\n if estimated_size < 1024 or psutil is None:\n return None\n vm = psutil.virtual_memory()\n if self.verbose:\n self.logger.debug(\"Memory available = %s (out of %s)\"\n % (humanize_bytes(vm.available),\n humanize_bytes(vm.total)))\n if (estimated_size < vm.available and self._save_to is None or\n self._save_to == \"memory\"):\n if self.verbose:\n self.logger.debug(\"Frame will be loaded into memory\")\n return None\n else:\n if self._save_to:\n tmpdir = self._save_to\n os.makedirs(tmpdir)\n else:\n tmpdir = tempfile.mkdtemp()\n du = psutil.disk_usage(tmpdir)\n if self.verbose:\n self.logger.debug(\"Free disk space on drive %s = %s\"\n % (os.path.splitdrive(tmpdir)[0] or \"/\",\n humanize_bytes(du.free)))\n if du.free > estimated_size or self._save_to:\n if self.verbose:\n self.logger.debug(\"Frame will be stored in %s\"\n % tmpdir)\n return tmpdir\n raise RuntimeError(\"The Frame is estimated to require at lest %s \"\n \"of memory, and you don't have that much available \"\n \"either in RAM or on a hard drive.\"\n % humanize_bytes(estimated_size))"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef save_nff(self, dest, _strategy=\"auto\"):\n if _strategy not in (\"auto\", \"write\", \"mmap\"):\n raise TValueError(\"Invalid parameter _strategy: only 'write' / 'mmap' \"\n \"/ 'auto' are allowed\")\n\n dest = os.path.expanduser(dest)\n if not os.path.exists(dest):\n os.makedirs(dest)\n\n self.materialize()\n mins = self.min().to_list()\n maxs = self.max().to_list()\n\n metafile = os.path.join(dest, \"_meta.nff\")\n with _builtin_open(metafile, \"w\", encoding=\"utf-8\") as out:\n out.write(\"# NFF2\\n\")\n out.write(\"# nrows = %d\\n\" % self.nrows)\n out.write('filename,stype,meta,colname,min,max\\n')\n l = len(str(self.ncols))\n for i in range(self.ncols):\n filename = \"c%0*d\" % (l, i + 1)\n colname = self.names[i].replace('\"', '\"\"')\n stype = self.stypes[i]\n if stype == dt.stype.obj64:\n dtwarn(\"Column %r of type obj64 was not saved\" % self.names[i])\n continue\n smin = _stringify(mins[i][0])\n smax = _stringify(maxs[i][0])\n out.write('%s,%s,,\"%s\",%s,%s\\n'\n % (filename, stype.code, colname, smin, smax))\n filename = os.path.join(dest, filename)\n core._column_save_to_disk(self, i, filename, _strategy)", "response": "Save the current object to a binary NFF file."} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nsending an event to the log.", "response": "def event(\n title,\n text,\n alert_type=None,\n aggregation_key=None,\n source_type_name=None,\n date_happened=None,\n priority=None,\n tags=None,\n hostname=None,\n ):\n \"\"\"\n Send an event.\n \"\"\""} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef user_projects(request):\n projects_owned = (\n request.db.query(Project.id)\n .join(Role.project)\n .filter(Role.role_name == \"Owner\", Role.user == request.user)\n .subquery()\n )\n\n with_sole_owner = (\n request.db.query(Role.project_id)\n .join(projects_owned)\n .filter(Role.role_name == \"Owner\")\n .group_by(Role.project_id)\n .having(func.count(Role.project_id) == 1)\n .subquery()\n )\n\n return {\n \"projects_owned\": (\n request.db.query(Project)\n .join(projects_owned, Project.id == projects_owned.c.id)\n .order_by(Project.name)\n .all()\n ),\n \"projects_sole_owned\": (\n request.db.query(Project).join(with_sole_owner).order_by(Project.name).all()\n ),\n }", "response": "Return all the projects for which the user is a sole owner"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nmerging one or more revisions into a single resource.", "response": "def merge(config, revisions, **kwargs):\n \"\"\"\n Merge one or more revisions.\n\n Takes one or more revisions or \"heads\" for all heads and merges them into\n a single revision.\n \"\"\"\n with alembic_lock(\n config.registry[\"sqlalchemy.engine\"], config.alembic_config()\n ) as alembic_config:\n alembic.command.merge(alembic_config, revisions, **kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef column_windows(session, column, windowsize):\n\n def int_for_range(start_id, end_id):\n if end_id:\n return and_(column >= start_id, column < end_id)\n else:\n return column >= start_id\n\n q = session.query(\n column, func.row_number().over(order_by=column).label(\"rownum\")\n ).from_self(column)\n\n if windowsize > 1:\n q = q.filter(text(\"rownum %% %d=1\" % windowsize))\n\n intervals = [row[0] for row in q]\n\n while intervals:\n start = intervals.pop(0)\n if intervals:\n end = intervals[0]\n else:\n end = None\n\n yield int_for_range(start, end)", "response": "Return a series of WHERE clauses against a given column that break it into windows."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nbreak a Query into windows on a given column.", "response": "def windowed_query(q, column, windowsize):\n \"\"\"\"\n Break a Query into windows on a given column.\n \"\"\"\n\n for whereclause in column_windows(q.session, column, windowsize):\n for row in q.filter(whereclause).order_by(column):\n yield row"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef filter_query(s):\n matches = re.findall(r'(?:\"([^\"]*)\")|([^\"]*)', s)\n result_quoted = [t[0].strip() for t in matches if t[0]]\n result_unquoted = [t[1].strip() for t in matches if t[1]]\n return result_quoted, result_unquoted", "response": "Filters given query with the below regex\n and returns lists of quoted and unquoted strings\n "} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef form_query(query_type, query):\n fields = [\n field + \"^\" + str(SEARCH_BOOSTS[field]) if field in SEARCH_BOOSTS else field\n for field in SEARCH_FIELDS\n ]\n return Q(\"multi_match\", fields=fields, query=query, type=query_type)", "response": "Form a query for a multi - match."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nrun migrations in offline mode.", "response": "def run_migrations_offline():\n \"\"\"\n Run migrations in 'offline' mode.\n\n This configures the context with just a URL\n and not an Engine, though an Engine is acceptable\n here as well. By skipping the Engine creation\n we don't even need a DBAPI to be available.\n\n Calls to context.execute() here emit the given string to the\n script output.\n \"\"\"\n url = context.config.get_main_option(\"sqlalchemy.url\")\n context.configure(url=url, compare_server_default=True)\n\n with context.begin_transaction():\n context.run_migrations()"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef run_migrations_online():\n connectable = context.config.attributes.get(\"connection\", None)\n\n if connectable is None:\n options = context.config.get_section(context.config.config_ini_section)\n url = options.pop(\"url\")\n connectable = create_engine(url, poolclass=pool.NullPool)\n\n with connectable.connect() as connection:\n context.configure(\n connection=connection,\n target_metadata=db.metadata,\n compare_server_default=True,\n )\n with context.begin_transaction():\n context.run_migrations()", "response": "Run migrations in online mode."} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nshowing current branch points.", "response": "def branches(config, **kwargs):\n \"\"\"\n Show current branch points.\n \"\"\"\n with alembic_lock(\n config.registry[\"sqlalchemy.engine\"], config.alembic_config()\n ) as alembic_config:\n alembic.command.branches(alembic_config, **kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef is_safe_url(url, host=None):\n if url is not None:\n url = url.strip()\n if not url:\n return False\n # Chrome treats \\ completely as /\n url = url.replace(\"\\\\\", \"/\")\n # Chrome considers any URL with more than two slashes to be absolute, but\n # urlparse is not so flexible. Treat any url with three slashes as unsafe.\n if url.startswith(\"///\"):\n return False\n url_info = urlparse(url)\n # Forbid URLs like http:///example.com - with a scheme, but without a\n # hostname.\n # In that URL, example.com is not the hostname but, a path component.\n # However, Chrome will still consider example.com to be the hostname,\n # so we must not allow this syntax.\n if not url_info.netloc and url_info.scheme:\n return False\n # Forbid URLs that start with control characters. Some browsers (like\n # Chrome) ignore quite a few control characters at the start of a\n # URL and might consider the URL as scheme relative.\n if unicodedata.category(url[0])[0] == \"C\":\n return False\n return (not url_info.netloc or url_info.netloc == host) and (\n not url_info.scheme or url_info.scheme in {\"http\", \"https\"}\n )", "response": "Returns True if the given URL is a safe redirection."} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nsanitize any user - submitted data to ensure that it can be used in XML", "response": "def _clean_for_xml(data):\n \"\"\" Sanitize any user-submitted data to ensure that it can be used in XML \"\"\"\n\n # If data is None or an empty string, don't bother\n if data:\n # This turns a string like \"Hello\u2026\" into \"Hello…\"\n data = data.encode(\"ascii\", \"xmlcharrefreplace\").decode(\"ascii\")\n # However it's still possible that there are invalid characters in the string,\n # so simply remove any of those characters\n return _illegal_xml_chars_re.sub(\"\", data)\n return data"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nsupport multiple endpoints serving the same views by chaining calls to xmlrpc_method", "response": "def xmlrpc_method(**kwargs):\n \"\"\"\n Support multiple endpoints serving the same views by chaining calls to\n xmlrpc_method\n \"\"\"\n # Add some default arguments\n kwargs.update(\n require_csrf=False,\n require_methods=[\"POST\"],\n decorator=(submit_xmlrpc_metrics(method=kwargs[\"method\"]),),\n mapper=TypedMapplyViewMapper,\n )\n\n def decorator(f):\n rpc2 = _xmlrpc_method(endpoint=\"RPC2\", **kwargs)\n pypi = _xmlrpc_method(endpoint=\"pypi\", **kwargs)\n pypi_slash = _xmlrpc_method(endpoint=\"pypi_slash\", **kwargs)\n return rpc2(pypi_slash(pypi(f)))\n\n return decorator"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef current(config, **kwargs):\n with alembic_lock(\n config.registry[\"sqlalchemy.engine\"], config.alembic_config()\n ) as alembic_config:\n alembic.command.current(alembic_config, **kwargs)", "response": "Display the current revision for a database."} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nchecking if the given file is a valid distribution file.", "response": "def _is_valid_dist_file(filename, filetype):\n \"\"\"\n Perform some basic checks to see whether the indicated file could be\n a valid distribution file.\n \"\"\"\n\n # If our file is a zipfile, then ensure that it's members are only\n # compressed with supported compression methods.\n if zipfile.is_zipfile(filename):\n with zipfile.ZipFile(filename) as zfp:\n for zinfo in zfp.infolist():\n if zinfo.compress_type not in {\n zipfile.ZIP_STORED,\n zipfile.ZIP_DEFLATED,\n }:\n return False\n\n if filename.endswith(\".exe\"):\n # The only valid filetype for a .exe file is \"bdist_wininst\".\n if filetype != \"bdist_wininst\":\n return False\n\n # Ensure that the .exe is a valid zip file, and that all of the files\n # contained within it have safe filenames.\n try:\n with zipfile.ZipFile(filename, \"r\") as zfp:\n # We need the no branch below to work around a bug in\n # coverage.py where it's detecting a missed branch where there\n # isn't one.\n for zipname in zfp.namelist(): # pragma: no branch\n if not _safe_zipnames.match(zipname):\n return False\n except zipfile.BadZipFile:\n return False\n elif filename.endswith(\".msi\"):\n # The only valid filetype for a .msi is \"bdist_msi\"\n if filetype != \"bdist_msi\":\n return False\n\n # Check the first 8 bytes of the MSI file. This was taken from the\n # legacy implementation of PyPI which itself took it from the\n # implementation of `file` I believe.\n with open(filename, \"rb\") as fp:\n if fp.read(8) != b\"\\xD0\\xCF\\x11\\xE0\\xA1\\xB1\\x1A\\xE1\":\n return False\n elif filename.endswith(\".zip\") or filename.endswith(\".egg\"):\n # Ensure that the .zip/.egg is a valid zip file, and that it has a\n # PKG-INFO file.\n try:\n with zipfile.ZipFile(filename, \"r\") as zfp:\n for zipname in zfp.namelist():\n parts = os.path.split(zipname)\n if len(parts) == 2 and parts[1] == \"PKG-INFO\":\n # We need the no branch below to work around a bug in\n # coverage.py where it's detecting a missed branch\n # where there isn't one.\n break # pragma: no branch\n else:\n return False\n except zipfile.BadZipFile:\n return False\n elif filename.endswith(\".whl\"):\n # Ensure that the .whl is a valid zip file, and that it has a WHEEL\n # file.\n try:\n with zipfile.ZipFile(filename, \"r\") as zfp:\n for zipname in zfp.namelist():\n parts = os.path.split(zipname)\n if len(parts) == 2 and parts[1] == \"WHEEL\":\n # We need the no branch below to work around a bug in\n # coverage.py where it's detecting a missed branch\n # where there isn't one.\n break # pragma: no branch\n else:\n return False\n except zipfile.BadZipFile:\n return False\n\n # If we haven't yet decided it's not valid, then we'll assume it is and\n # allow it.\n return True"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncheck to see if a file already exists and if it s content matches the given hashes.", "response": "def _is_duplicate_file(db_session, filename, hashes):\n \"\"\"\n Check to see if file already exists, and if it's content matches.\n A file is considered to exist if its filename *or* blake2 digest are\n present in a file row in the database.\n\n Returns:\n - True: This file is a duplicate and all further processing should halt.\n - False: This file exists, but it is not a duplicate.\n - None: This file does not exist.\n \"\"\"\n\n file_ = (\n db_session.query(File)\n .filter(\n (File.filename == filename)\n | (File.blake2_256_digest == hashes[\"blake2_256\"])\n )\n .first()\n )\n\n if file_ is not None:\n return (\n file_.filename == filename\n and file_.sha256_digest == hashes[\"sha256\"]\n and file_.md5_digest == hashes[\"md5\"]\n and file_.blake2_256_digest == hashes[\"blake2_256\"]\n )\n\n return None"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef shell(config, type_):\n\n # Imported here because we don't want to trigger an import from anything\n # but warehouse.cli at the module scope.\n from warehouse.db import Session\n\n if type_ is None:\n type_ = autodetect()\n\n runner = {\"bpython\": bpython, \"ipython\": ipython, \"plain\": plain}[type_]\n\n session = Session(bind=config.registry[\"sqlalchemy.engine\"])\n\n try:\n runner(config=config, db=session)\n except ImportError:\n raise click.ClickException(\n \"The {!r} shell is not available.\".format(type_)\n ) from None", "response": "Open up a Python shell with Warehouse preconfigured in it."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nlisting changeset scripts in chronological order.", "response": "def history(config, revision_range, **kwargs):\n \"\"\"\n List changeset scripts in chronological order.\n \"\"\"\n with alembic_lock(\n config.registry[\"sqlalchemy.engine\"], config.alembic_config()\n ) as alembic_config:\n alembic.command.history(alembic_config, revision_range, **kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef reindex(config):\n\n request = config.task(_reindex).get_request()\n config.task(_reindex).run(request)", "response": "Recreate the Search Index."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef heads(config, **kwargs):\n with alembic_lock(\n config.registry[\"sqlalchemy.engine\"], config.alembic_config()\n ) as alembic_config:\n alembic.command.heads(alembic_config, **kwargs)", "response": "Show current available heads."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef reindex(self, request):\n r = redis.StrictRedis.from_url(request.registry.settings[\"celery.scheduler_url\"])\n try:\n with SearchLock(r, timeout=30 * 60, blocking_timeout=30):\n p = urllib.parse.urlparse(request.registry.settings[\"elasticsearch.url\"])\n client = elasticsearch.Elasticsearch(\n [urllib.parse.urlunparse(p[:2] + (\"\",) * 4)],\n verify_certs=True,\n ca_certs=certifi.where(),\n timeout=30,\n retry_on_timeout=True,\n serializer=serializer.serializer,\n )\n number_of_replicas = request.registry.get(\"elasticsearch.replicas\", 0)\n refresh_interval = request.registry.get(\"elasticsearch.interval\", \"1s\")\n\n # We use a randomly named index so that we can do a zero downtime reindex.\n # Essentially we'll use a randomly named index which we will use until all\n # of the data has been reindexed, at which point we'll point an alias at\n # our randomly named index, and then delete the old randomly named index.\n\n # Create the new index and associate all of our doc types with it.\n index_base = request.registry[\"elasticsearch.index\"]\n random_token = binascii.hexlify(os.urandom(5)).decode(\"ascii\")\n new_index_name = \"{}-{}\".format(index_base, random_token)\n doc_types = request.registry.get(\"search.doc_types\", set())\n shards = request.registry.get(\"elasticsearch.shards\", 1)\n\n # Create the new index with zero replicas and index refreshes disabled\n # while we are bulk indexing.\n new_index = get_index(\n new_index_name,\n doc_types,\n using=client,\n shards=shards,\n replicas=0,\n interval=\"-1\",\n )\n new_index.create(wait_for_active_shards=shards)\n\n # From this point on, if any error occurs, we want to be able to delete our\n # in progress index.\n try:\n request.db.execute(\"SET statement_timeout = '600s'\")\n\n for _ in parallel_bulk(\n client, _project_docs(request.db), index=new_index_name\n ):\n pass\n except: # noqa\n new_index.delete()\n raise\n finally:\n request.db.rollback()\n request.db.close()\n\n # Now that we've finished indexing all of our data we can update the\n # replicas and refresh intervals.\n client.indices.put_settings(\n index=new_index_name,\n body={\n \"index\": {\n \"number_of_replicas\": number_of_replicas,\n \"refresh_interval\": refresh_interval,\n }\n },\n )\n\n # Point the alias at our new randomly named index and delete the old index.\n if client.indices.exists_alias(name=index_base):\n to_delete = set()\n actions = []\n for name in client.indices.get_alias(name=index_base):\n to_delete.add(name)\n actions.append({\"remove\": {\"index\": name, \"alias\": index_base}})\n actions.append({\"add\": {\"index\": new_index_name, \"alias\": index_base}})\n client.indices.update_aliases({\"actions\": actions})\n client.indices.delete(\",\".join(to_delete))\n else:\n client.indices.put_alias(name=index_base, index=new_index_name)\n except redis.exceptions.LockError as exc:\n raise self.retry(countdown=60, exc=exc)", "response": "Recreate the Search Index."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get(self):\n parser = reqparse.RequestParser()\n parser.add_argument('public_key', type=parameters.valid_ed25519,\n required=True)\n parser.add_argument('spent', type=parameters.valid_bool)\n args = parser.parse_args(strict=True)\n\n pool = current_app.config['bigchain_pool']\n with pool() as bigchain:\n outputs = bigchain.get_outputs_filtered(args['public_key'],\n args['spent'])\n return [{'transaction_id': output.txid, 'output_index': output.output}\n for output in outputs]", "response": "API endpoint to retrieve a list of links to transaction\n outputs.\n "} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef run_show_config(args):\n # TODO Proposal: remove the \"hidden\" configuration. Only show config. If\n # the system needs to be configured, then display information on how to\n # configure the system.\n config = copy.deepcopy(bigchaindb.config)\n del config['CONFIGURED']\n print(json.dumps(config, indent=4, sort_keys=True))", "response": "Show the current configuration"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef run_configure(args):\n config_path = args.config or bigchaindb.config_utils.CONFIG_DEFAULT_PATH\n\n config_file_exists = False\n # if the config path is `-` then it's stdout\n if config_path != '-':\n config_file_exists = os.path.exists(config_path)\n\n if config_file_exists and not args.yes:\n want = input_on_stderr('Config file `{}` exists, do you want to '\n 'override it? (cannot be undone) [y/N]: '.format(config_path))\n if want != 'y':\n return\n\n conf = copy.deepcopy(bigchaindb.config)\n\n # select the correct config defaults based on the backend\n print('Generating default configuration for backend {}'\n .format(args.backend), file=sys.stderr)\n database_keys = bigchaindb._database_keys_map[args.backend]\n conf['database'] = bigchaindb._database_map[args.backend]\n\n if not args.yes:\n for key in ('bind', ):\n val = conf['server'][key]\n conf['server'][key] = input_on_stderr('API Server {}? (default `{}`): '.format(key, val), val)\n\n for key in ('scheme', 'host', 'port'):\n val = conf['wsserver'][key]\n conf['wsserver'][key] = input_on_stderr('WebSocket Server {}? (default `{}`): '.format(key, val), val)\n\n for key in database_keys:\n val = conf['database'][key]\n conf['database'][key] = input_on_stderr('Database {}? (default `{}`): '.format(key, val), val)\n\n for key in ('host', 'port'):\n val = conf['tendermint'][key]\n conf['tendermint'][key] = input_on_stderr('Tendermint {}? (default `{}`)'.format(key, val), val)\n\n if config_path != '-':\n bigchaindb.config_utils.write_config(conf, config_path)\n else:\n print(json.dumps(conf, indent=4, sort_keys=True))\n print('Configuration written to {}'.format(config_path), file=sys.stderr)\n print('Ready to go!', file=sys.stderr)", "response": "Run a script to configure the current node."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef run_election_new_upsert_validator(args, bigchain):\n\n new_validator = {\n 'public_key': {'value': public_key_from_base64(args.public_key),\n 'type': 'ed25519-base16'},\n 'power': args.power,\n 'node_id': args.node_id\n }\n\n return create_new_election(args.sk, bigchain, ValidatorElection, new_validator)", "response": "Initiates an election to add or update a validator to an existing network"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef run_election_approve(args, bigchain):\n\n key = load_node_key(args.sk)\n tx = bigchain.get_transaction(args.election_id)\n voting_powers = [v.amount for v in tx.outputs if key.public_key in v.public_keys]\n if len(voting_powers) > 0:\n voting_power = voting_powers[0]\n else:\n logger.error('The key you provided does not match any of the eligible voters in this election.')\n return False\n\n inputs = [i for i in tx.to_inputs() if key.public_key in i.owners_before]\n election_pub_key = ValidatorElection.to_public_key(tx.id)\n approval = Vote.generate(inputs,\n [([election_pub_key], voting_power)],\n tx.id).sign([key.private_key])\n approval.validate(bigchain)\n\n resp = bigchain.write_transaction(approval, 'broadcast_tx_commit')\n\n if resp == (202, ''):\n logger.info('[SUCCESS] Your vote has been submitted')\n return approval.id\n else:\n logger.error('Failed to commit vote')\n return False", "response": "Approve an election node"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nretrieves information about an election", "response": "def run_election_show(args, bigchain):\n \"\"\"Retrieves information about an election\n\n :param args: dict\n args = {\n 'election_id': the transaction_id for an election (str)\n }\n :param bigchain: an instance of BigchainDB\n \"\"\"\n\n election = bigchain.get_transaction(args.election_id)\n if not election:\n logger.error(f'No election found with election_id {args.election_id}')\n return\n\n response = election.show_election(bigchain)\n\n logger.info(response)\n\n return response"} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nstarts the processes to run the node", "response": "def run_start(args):\n \"\"\"Start the processes to run the node\"\"\"\n\n # Configure Logging\n setup_logging()\n\n logger.info('BigchainDB Version %s', bigchaindb.__version__)\n run_recover(bigchaindb.lib.BigchainDB())\n\n if not args.skip_initialize_database:\n logger.info('Initializing database')\n _run_init()\n\n logger.info('Starting BigchainDB main process.')\n from bigchaindb.start import start\n start(args)"} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nshows the supported Tendermint version", "response": "def run_tendermint_version(args):\n \"\"\"Show the supported Tendermint version(s)\"\"\"\n supported_tm_ver = {\n 'description': 'BigchainDB supports the following Tendermint version(s)',\n 'tendermint': __tm_supported_versions__,\n }\n print(json.dumps(supported_tm_ver, indent=4, sort_keys=True))"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nsearch for assets with the given text search string.", "response": "def text_search(conn, search, *, language='english', case_sensitive=False,\n diacritic_sensitive=False, text_score=False, limit=0, table=None):\n \"\"\"Return all the assets that match the text search.\n\n The results are sorted by text score.\n For more information about the behavior of text search on MongoDB see\n https://docs.mongodb.com/manual/reference/operator/query/text/#behavior\n\n Args:\n search (str): Text search string to query the text index\n language (str, optional): The language for the search and the rules for\n stemmer and tokenizer. If the language is ``None`` text search uses\n simple tokenization and no stemming.\n case_sensitive (bool, optional): Enable or disable case sensitive\n search.\n diacritic_sensitive (bool, optional): Enable or disable case sensitive\n diacritic search.\n text_score (bool, optional): If ``True`` returns the text score with\n each document.\n limit (int, optional): Limit the number of returned documents.\n\n Returns:\n :obj:`list` of :obj:`dict`: a list of assets\n\n Raises:\n OperationError: If the backend does not support text search\n \"\"\"\n\n raise OperationError('This query is only supported when running '\n 'BigchainDB with MongoDB as the backend.')"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef main():\n\n ctx = {}\n\n def pretty_json(data):\n return json.dumps(data, indent=2, sort_keys=True)\n\n client = server.create_app().test_client()\n\n host = 'example.com:9984'\n\n # HTTP Index\n res = client.get('/', environ_overrides={'HTTP_HOST': host})\n res_data = json.loads(res.data.decode())\n ctx['index'] = pretty_json(res_data)\n\n # API index\n res = client.get('/api/v1/', environ_overrides={'HTTP_HOST': host})\n ctx['api_index'] = pretty_json(json.loads(res.data.decode()))\n\n # tx create\n privkey = 'CfdqtD7sS7FgkMoGPXw55MVGGFwQLAoHYTcBhZDtF99Z'\n pubkey = '4K9sWUMFwTgaDGPfdynrbxWqWS6sWmKbZoTjxLtVUibD'\n asset = {'msg': 'Hello BigchainDB!'}\n tx = Transaction.create([pubkey], [([pubkey], 1)], asset=asset, metadata={'sequence': 0})\n tx = tx.sign([privkey])\n ctx['tx'] = pretty_json(tx.to_dict())\n ctx['public_keys'] = tx.outputs[0].public_keys[0]\n ctx['txid'] = tx.id\n\n # tx transfer\n privkey_transfer = '3AeWpPdhEZzWLYfkfYHBfMFC2r1f8HEaGS9NtbbKssya'\n pubkey_transfer = '3yfQPHeWAa1MxTX9Zf9176QqcpcnWcanVZZbaHb8B3h9'\n\n cid = 0\n input_ = Input(fulfillment=tx.outputs[cid].fulfillment,\n fulfills=TransactionLink(txid=tx.id, output=cid),\n owners_before=tx.outputs[cid].public_keys)\n tx_transfer = Transaction.transfer([input_], [([pubkey_transfer], 1)], asset_id=tx.id, metadata={'sequence': 1})\n tx_transfer = tx_transfer.sign([privkey])\n ctx['tx_transfer'] = pretty_json(tx_transfer.to_dict())\n ctx['public_keys_transfer'] = tx_transfer.outputs[0].public_keys[0]\n ctx['tx_transfer_id'] = tx_transfer.id\n\n # privkey_transfer_last = 'sG3jWDtdTXUidBJK53ucSTrosktG616U3tQHBk81eQe'\n pubkey_transfer_last = '3Af3fhhjU6d9WecEM9Uw5hfom9kNEwE7YuDWdqAUssqm'\n\n cid = 0\n input_ = Input(fulfillment=tx_transfer.outputs[cid].fulfillment,\n fulfills=TransactionLink(txid=tx_transfer.id, output=cid),\n owners_before=tx_transfer.outputs[cid].public_keys)\n tx_transfer_last = Transaction.transfer([input_], [([pubkey_transfer_last], 1)],\n asset_id=tx.id, metadata={'sequence': 2})\n tx_transfer_last = tx_transfer_last.sign([privkey_transfer])\n ctx['tx_transfer_last'] = pretty_json(tx_transfer_last.to_dict())\n ctx['tx_transfer_last_id'] = tx_transfer_last.id\n ctx['public_keys_transfer_last'] = tx_transfer_last.outputs[0].public_keys[0]\n\n # block\n node_private = \"5G2kE1zJAgTajkVSbPAQWo4c2izvtwqaNHYsaNpbbvxX\"\n node_public = \"DngBurxfeNVKZWCEcDnLj1eMPAS7focUZTE5FndFGuHT\"\n signature = \"53wxrEQDYk1dXzmvNSytbCfmNVnPqPkDQaTnAe8Jf43s6ssejPxezkCvUnGTnduNUmaLjhaan1iRLi3peu6s5DzA\"\n\n app_hash = 'f6e0c49c6d94d6924351f25bb334cf2a99af4206339bf784e741d1a5ab599056'\n block = lib.Block(height=1, transactions=[tx.to_dict()], app_hash=app_hash)\n block_dict = block._asdict()\n block_dict.pop('app_hash')\n ctx['block'] = pretty_json(block_dict)\n ctx['blockid'] = block.height\n\n # block status\n block_list = [\n block.height\n ]\n ctx['block_list'] = pretty_json(block_list)\n\n\n base_path = os.path.join(os.path.dirname(__file__),\n 'source/http-samples')\n if not os.path.exists(base_path):\n os.makedirs(base_path)\n\n for name, tpl in TPLS.items():\n path = os.path.join(base_path, name + '.http')\n code = tpl % ctx\n with open(path, 'w') as handle:\n handle.write(code)", "response": "Main function for the\n test"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nserialize a dict into a JSON formatted string.", "response": "def serialize(data):\n \"\"\"Serialize a dict into a JSON formatted string.\n\n This function enforces rules like the separator and order of keys.\n This ensures that all dicts are serialized in the same way.\n\n This is specially important for hashing data. We need to make sure that\n everyone serializes their data in the same way so that we do not have\n hash mismatches for the same structure due to serialization\n differences.\n\n Args:\n data (dict): dict to serialize\n\n Returns:\n str: JSON formatted string\n\n \"\"\"\n return rapidjson.dumps(data, skipkeys=False, ensure_ascii=False,\n sort_keys=True)"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef validate_txn_obj(obj_name, obj, key, validation_fun):\n backend = bigchaindb.config['database']['backend']\n\n if backend == 'localmongodb':\n data = obj.get(key, {})\n if isinstance(data, dict):\n validate_all_keys_in_obj(obj_name, data, validation_fun)\n elif isinstance(data, list):\n validate_all_items_in_list(obj_name, data, validation_fun)", "response": "Validate value of key in obj using validation_fun."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nvalidate all keys in obj by using validation_fun.", "response": "def validate_all_keys_in_obj(obj_name, obj, validation_fun):\n \"\"\"Validate all (nested) keys in `obj` by using `validation_fun`.\n\n Args:\n obj_name (str): name for `obj` being validated.\n obj (dict): dictionary object.\n validation_fun (function): function used to validate the value\n of `key`.\n\n Returns:\n None: indicates validation successful\n\n Raises:\n ValidationError: `validation_fun` will raise this error on failure\n \"\"\"\n for key, value in obj.items():\n validation_fun(obj_name, key)\n if isinstance(value, dict):\n validate_all_keys_in_obj(obj_name, value, validation_fun)\n elif isinstance(value, list):\n validate_all_items_in_list(obj_name, value, validation_fun)"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef validate_all_values_for_key_in_obj(obj, key, validation_fun):\n for vkey, value in obj.items():\n if vkey == key:\n validation_fun(value)\n elif isinstance(value, dict):\n validate_all_values_for_key_in_obj(value, key, validation_fun)\n elif isinstance(value, list):\n validate_all_values_for_key_in_list(value, key, validation_fun)", "response": "Validate value for all occurrence of key in obj using validation_fun."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef validate_key(obj_name, key):\n if re.search(r'^[$]|\\.|\\x00', key):\n error_str = ('Invalid key name \"{}\" in {} object. The '\n 'key name cannot contain characters '\n '\".\", \"$\" or null characters').format(key, obj_name)\n raise ValidationError(error_str)", "response": "Validates that the key name is valid."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef create_app(*, debug=False, threads=1, bigchaindb_factory=None):\n\n if not bigchaindb_factory:\n bigchaindb_factory = BigchainDB\n\n app = Flask(__name__)\n app.wsgi_app = StripContentTypeMiddleware(app.wsgi_app)\n\n CORS(app)\n\n app.debug = debug\n\n app.config['bigchain_pool'] = utils.pool(bigchaindb_factory, size=threads)\n\n add_routes(app)\n\n return app", "response": "Create a Flask application."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nwrap and return an application ready to be run.", "response": "def create_server(settings, log_config=None, bigchaindb_factory=None):\n \"\"\"Wrap and return an application ready to be run.\n\n Args:\n settings (dict): a dictionary containing the settings, more info\n here http://docs.gunicorn.org/en/latest/settings.html\n\n Return:\n an initialized instance of the application.\n \"\"\"\n\n settings = copy.deepcopy(settings)\n\n if not settings.get('workers'):\n settings['workers'] = (multiprocessing.cpu_count() * 2) + 1\n\n if not settings.get('threads'):\n # Note: Threading is not recommended currently, as the frontend workload\n # is largely CPU bound and parallisation across Python threads makes it\n # slower.\n settings['threads'] = 1\n\n settings['custom_log_config'] = log_config\n app = create_app(debug=settings.get('debug', False),\n threads=settings['threads'],\n bigchaindb_factory=bigchaindb_factory)\n standalone = StandaloneApplication(app, options=settings)\n return standalone"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef setup_logging():\n\n logging_configs = DEFAULT_LOGGING_CONFIG\n new_logging_configs = bigchaindb.config['log']\n\n if 'file' in new_logging_configs:\n filename = new_logging_configs['file']\n logging_configs['handlers']['file']['filename'] = filename\n\n if 'error_file' in new_logging_configs:\n error_filename = new_logging_configs['error_file']\n logging_configs['handlers']['errors']['filename'] = error_filename\n\n if 'level_console' in new_logging_configs:\n level = _normalize_log_level(new_logging_configs['level_console'])\n logging_configs['handlers']['console']['level'] = level\n\n if 'level_logfile' in new_logging_configs:\n level = _normalize_log_level(new_logging_configs['level_logfile'])\n logging_configs['handlers']['file']['level'] = level\n\n if 'fmt_console' in new_logging_configs:\n fmt = new_logging_configs['fmt_console']\n logging_configs['formatters']['console']['format'] = fmt\n\n if 'fmt_logfile' in new_logging_configs:\n fmt = new_logging_configs['fmt_logfile']\n logging_configs['formatters']['file']['format'] = fmt\n\n if 'datefmt_console' in new_logging_configs:\n fmt = new_logging_configs['datefmt_console']\n logging_configs['formatters']['console']['datefmt'] = fmt\n\n if 'datefmt_logfile' in new_logging_configs:\n fmt = new_logging_configs['datefmt_logfile']\n logging_configs['formatters']['file']['datefmt'] = fmt\n\n log_levels = new_logging_configs.get('granular_levels', {})\n\n for logger_name, level in log_levels.items():\n level = _normalize_log_level(level)\n try:\n logging_configs['loggers'][logger_name]['level'] = level\n except KeyError:\n logging_configs['loggers'][logger_name] = {'level': level}\n\n set_logging_config(logging_configs)", "response": "Function to configure log hadlers."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_api_v1_info(api_prefix):\n websocket_root = base_ws_uri() + EVENTS_ENDPOINT\n docs_url = [\n 'https://docs.bigchaindb.com/projects/server/en/v',\n version.__version__,\n '/http-client-server-api.html',\n ]\n\n return {\n 'docs': ''.join(docs_url),\n 'transactions': '{}transactions/'.format(api_prefix),\n 'blocks': '{}blocks/'.format(api_prefix),\n 'assets': '{}assets/'.format(api_prefix),\n 'outputs': '{}outputs/'.format(api_prefix),\n 'streams': websocket_root,\n 'metadata': '{}metadata/'.format(api_prefix),\n 'validators': '{}validators'.format(api_prefix),\n }", "response": "Return a dict with all the information specific for the v1 of the\n api."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef input_on_stderr(prompt='', default=None, convert=None):\n\n print(prompt, end='', file=sys.stderr)\n value = builtins.input()\n return _convert(value, default, convert)", "response": "Output a string to stderr and wait for input."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get(self):\n parser = reqparse.RequestParser()\n parser.add_argument('search', type=str, required=True)\n parser.add_argument('limit', type=int)\n args = parser.parse_args()\n\n if not args['search']:\n return make_error(400, 'text_search cannot be empty')\n if not args['limit']:\n # if the limit is not specified do not pass None to `text_search`\n del args['limit']\n\n pool = current_app.config['bigchain_pool']\n\n with pool() as bigchain:\n assets = bigchain.text_search(**args)\n\n try:\n # This only works with MongoDB as the backend\n return list(assets)\n except OperationError as e:\n return make_error(\n 400,\n '({}): {}'.format(type(e).__name__, e)\n )", "response": "This endpoint is used to perform a text search on the assets. It will return a list of assets that match the query."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ninitialize chain upon genesis or a migration", "response": "def init_chain(self, genesis):\n \"\"\"Initialize chain upon genesis or a migration\"\"\"\n\n app_hash = ''\n height = 0\n\n known_chain = self.bigchaindb.get_latest_abci_chain()\n if known_chain is not None:\n chain_id = known_chain['chain_id']\n\n if known_chain['is_synced']:\n msg = f'Got invalid InitChain ABCI request ({genesis}) - ' + \\\n 'the chain {chain_id} is already synced.'\n logger.error(msg)\n sys.exit(1)\n\n if chain_id != genesis.chain_id:\n validators = self.bigchaindb.get_validators()\n self.log_abci_migration_error(chain_id, validators)\n sys.exit(1)\n\n # set migration values for app hash and height\n block = self.bigchaindb.get_latest_block()\n app_hash = '' if block is None else block['app_hash']\n height = 0 if block is None else block['height'] + 1\n\n known_validators = self.bigchaindb.get_validators()\n validator_set = [vutils.decode_validator(v)\n for v in genesis.validators]\n\n if known_validators and known_validators != validator_set:\n self.log_abci_migration_error(known_chain['chain_id'],\n known_validators)\n sys.exit(1)\n\n block = Block(app_hash=app_hash, height=height, transactions=[])\n self.bigchaindb.store_block(block._asdict())\n self.bigchaindb.store_validator_set(height + 1, validator_set)\n abci_chain_height = 0 if known_chain is None else known_chain['height']\n self.bigchaindb.store_abci_chain(abci_chain_height,\n genesis.chain_id, True)\n self.chain = {'height': abci_chain_height, 'is_synced': True,\n 'chain_id': genesis.chain_id}\n return ResponseInitChain()"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef info(self, request):\n\n self.abort_if_abci_chain_is_not_synced()\n\n # Check if BigchainDB supports the Tendermint version\n if not (hasattr(request, 'version') and tendermint_version_is_compatible(request.version)):\n logger.error(f'Unsupported Tendermint version: {getattr(request, \"version\", \"no version\")}.'\n f' Currently, BigchainDB only supports {__tm_supported_versions__}. Exiting!')\n sys.exit(1)\n\n logger.info(f\"Tendermint version: {request.version}\")\n\n r = ResponseInfo()\n block = self.bigchaindb.get_latest_block()\n if block:\n chain_shift = 0 if self.chain is None else self.chain['height']\n r.last_block_height = block['height'] - chain_shift\n r.last_block_app_hash = block['app_hash'].encode('utf-8')\n else:\n r.last_block_height = 0\n r.last_block_app_hash = b''\n return r", "response": "Return the height of the latest committed block."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef check_tx(self, raw_transaction):\n\n self.abort_if_abci_chain_is_not_synced()\n\n logger.debug('check_tx: %s', raw_transaction)\n transaction = decode_transaction(raw_transaction)\n if self.bigchaindb.is_valid_transaction(transaction):\n logger.debug('check_tx: VALID')\n return ResponseCheckTx(code=CodeTypeOk)\n else:\n logger.debug('check_tx: INVALID')\n return ResponseCheckTx(code=CodeTypeError)", "response": "Validate the transaction before entry into the mempool."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef begin_block(self, req_begin_block):\n self.abort_if_abci_chain_is_not_synced()\n\n chain_shift = 0 if self.chain is None else self.chain['height']\n logger.debug('BEGIN BLOCK, height:%s, num_txs:%s',\n req_begin_block.header.height + chain_shift,\n req_begin_block.header.num_txs)\n\n self.block_txn_ids = []\n self.block_transactions = []\n return ResponseBeginBlock()", "response": "Initialize list of transaction ids and block hash."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef deliver_tx(self, raw_transaction):\n\n self.abort_if_abci_chain_is_not_synced()\n\n logger.debug('deliver_tx: %s', raw_transaction)\n transaction = self.bigchaindb.is_valid_transaction(\n decode_transaction(raw_transaction), self.block_transactions)\n\n if not transaction:\n logger.debug('deliver_tx: INVALID')\n return ResponseDeliverTx(code=CodeTypeError)\n else:\n logger.debug('storing tx')\n self.block_txn_ids.append(transaction.id)\n self.block_transactions.append(transaction)\n return ResponseDeliverTx(code=CodeTypeOk)", "response": "Validate the transaction before mutating the state."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef end_block(self, request_end_block):\n\n self.abort_if_abci_chain_is_not_synced()\n\n chain_shift = 0 if self.chain is None else self.chain['height']\n\n height = request_end_block.height + chain_shift\n self.new_height = height\n\n # store pre-commit state to recover in case there is a crash during\n # `end_block` or `commit`\n logger.debug(f'Updating pre-commit state: {self.new_height}')\n pre_commit_state = dict(height=self.new_height,\n transactions=self.block_txn_ids)\n self.bigchaindb.store_pre_commit_state(pre_commit_state)\n\n block_txn_hash = calculate_hash(self.block_txn_ids)\n block = self.bigchaindb.get_latest_block()\n\n if self.block_txn_ids:\n self.block_txn_hash = calculate_hash([block['app_hash'], block_txn_hash])\n else:\n self.block_txn_hash = block['app_hash']\n\n validator_update = Election.process_block(self.bigchaindb,\n self.new_height,\n self.block_transactions)\n\n return ResponseEndBlock(validator_updates=validator_update)", "response": "This function is called when the block is finished."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef commit(self):\n\n self.abort_if_abci_chain_is_not_synced()\n\n data = self.block_txn_hash.encode('utf-8')\n\n # register a new block only when new transactions are received\n if self.block_txn_ids:\n self.bigchaindb.store_bulk_transactions(self.block_transactions)\n\n block = Block(app_hash=self.block_txn_hash,\n height=self.new_height,\n transactions=self.block_txn_ids)\n # NOTE: storing the block should be the last operation during commit\n # this effects crash recovery. Refer BEP#8 for details\n self.bigchaindb.store_block(block._asdict())\n\n logger.debug('Commit-ing new block with hash: apphash=%s ,'\n 'height=%s, txn ids=%s', data, self.new_height,\n self.block_txn_ids)\n\n if self.events_queue:\n event = Event(EventTypes.BLOCK_VALID, {\n 'height': self.new_height,\n 'transactions': self.block_transactions\n })\n self.events_queue.put(event)\n\n return ResponseCommit(data=data)", "response": "Store the new height and along with block hash."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef validate(self, bigchain, current_transactions=[]):\n\n current_validators = self.get_validators(bigchain)\n\n super(ValidatorElection, self).validate(bigchain, current_transactions=current_transactions)\n\n # NOTE: change more than 1/3 of the current power is not allowed\n if self.asset['data']['power'] >= (1/3)*sum(current_validators.values()):\n raise InvalidPowerChange('`power` change must be less than 1/3 of total power')\n\n return self", "response": "Validate that the current validator is allowed."} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncreate a new queue for a specific combination of event types and return it.", "response": "def get_subscriber_queue(self, event_types=None):\n \"\"\"Create a new queue for a specific combination of event types\n and return it.\n\n Returns:\n a :class:`multiprocessing.Queue`.\n Raises:\n RuntimeError if called after `run`\n \"\"\"\n\n try:\n self.started_queue.get(timeout=1)\n raise RuntimeError('Cannot create a new subscriber queue while Exchange is running.')\n except Empty:\n pass\n\n if event_types is None:\n event_types = EventTypes.ALL\n\n queue = Queue()\n self.queues[event_types].append(queue)\n return queue"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ngive an event send it to all the subscribers.", "response": "def dispatch(self, event):\n \"\"\"Given an event, send it to all the subscribers.\n\n Args\n event (:class:`~bigchaindb.events.EventTypes`): the event to\n dispatch to all the subscribers.\n \"\"\"\n\n for event_types, queues in self.queues.items():\n if event.type & event_types:\n for queue in queues:\n queue.put(event)"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef init_database(connection=None, dbname=None):\n\n connection = connection or connect()\n dbname = dbname or bigchaindb.config['database']['name']\n\n create_database(connection, dbname)\n create_tables(connection, dbname)", "response": "Initialize the configured backend for use with BigchainDB."} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nvalidates all nested language key in obj.", "response": "def validate_language_key(obj, key):\n \"\"\"Validate all nested \"language\" key in `obj`.\n\n Args:\n obj (dict): dictionary whose \"language\" key is to be validated.\n\n Returns:\n None: validation successful\n\n Raises:\n ValidationError: will raise exception in case language is not valid.\n \"\"\"\n backend = bigchaindb.config['database']['backend']\n\n if backend == 'localmongodb':\n data = obj.get(key, {})\n if isinstance(data, dict):\n validate_all_values_for_key_in_obj(data, 'language', validate_language)\n elif isinstance(data, list):\n validate_all_values_for_key_in_list(data, 'language', validate_language)"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get(self, block_id):\n\n pool = current_app.config['bigchain_pool']\n\n with pool() as bigchain:\n block = bigchain.get_block(block_id=block_id)\n\n if not block:\n return make_error(404)\n\n return block", "response": "Get details about a block."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ngenerating base58 encode public - private key pair from a hex encoded private key", "response": "def key_pair_from_ed25519_key(hex_private_key):\n \"\"\"Generate base58 encode public-private key pair from a hex encoded private key\"\"\"\n priv_key = crypto.Ed25519SigningKey(bytes.fromhex(hex_private_key)[:32], encoding='bytes')\n public_key = priv_key.get_verifying_key()\n return CryptoKeypair(private_key=priv_key.encode(encoding='base58').decode('utf-8'),\n public_key=public_key.encode(encoding='base58').decode('utf-8'))"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef public_key_from_ed25519_key(hex_public_key):\n public_key = crypto.Ed25519VerifyingKey(bytes.fromhex(hex_public_key), encoding='bytes')\n return public_key.encode(encoding='base58').decode('utf-8')", "response": "Generate base58 public key from hex encoded public key"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nencode a fulfillment as a details dictionary.", "response": "def _fulfillment_to_details(fulfillment):\n \"\"\"Encode a fulfillment as a details dictionary\n\n Args:\n fulfillment: Crypto-conditions Fulfillment object\n \"\"\"\n\n if fulfillment.type_name == 'ed25519-sha-256':\n return {\n 'type': 'ed25519-sha-256',\n 'public_key': base58.b58encode(fulfillment.public_key).decode(),\n }\n\n if fulfillment.type_name == 'threshold-sha-256':\n subconditions = [\n _fulfillment_to_details(cond['body'])\n for cond in fulfillment.subconditions\n ]\n return {\n 'type': 'threshold-sha-256',\n 'threshold': fulfillment.threshold,\n 'subconditions': subconditions,\n }\n\n raise UnsupportedTypeError(fulfillment.type_name)"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _fulfillment_from_details(data, _depth=0):\n if _depth == 100:\n raise ThresholdTooDeep()\n\n if data['type'] == 'ed25519-sha-256':\n public_key = base58.b58decode(data['public_key'])\n return Ed25519Sha256(public_key=public_key)\n\n if data['type'] == 'threshold-sha-256':\n threshold = ThresholdSha256(data['threshold'])\n for cond in data['subconditions']:\n cond = _fulfillment_from_details(cond, _depth+1)\n threshold.add_subfulfillment(cond)\n return threshold\n\n raise UnsupportedTypeError(data.get('type'))", "response": "Load a fulfillment for a signing spec dictionary."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef to_dict(self):\n try:\n fulfillment = self.fulfillment.serialize_uri()\n except (TypeError, AttributeError, ASN1EncodeError, ASN1DecodeError):\n fulfillment = _fulfillment_to_details(self.fulfillment)\n\n try:\n # NOTE: `self.fulfills` can be `None` and that's fine\n fulfills = self.fulfills.to_dict()\n except AttributeError:\n fulfills = None\n\n input_ = {\n 'owners_before': self.owners_before,\n 'fulfills': fulfills,\n 'fulfillment': fulfillment,\n }\n return input_", "response": "Transforms the object to a Python dictionary."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef from_dict(cls, data):\n fulfillment = data['fulfillment']\n if not isinstance(fulfillment, (Fulfillment, type(None))):\n try:\n fulfillment = Fulfillment.from_uri(data['fulfillment'])\n except ASN1DecodeError:\n # TODO Remove as it is legacy code, and simply fall back on\n # ASN1DecodeError\n raise InvalidSignature(\"Fulfillment URI couldn't been parsed\")\n except TypeError:\n # NOTE: See comment about this special case in\n # `Input.to_dict`\n fulfillment = _fulfillment_from_details(data['fulfillment'])\n fulfills = TransactionLink.from_dict(data['fulfills'])\n return cls(fulfillment, data['owners_before'], fulfills)", "response": "Transforms a Python dictionary to an Input object."} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ntransforms the object to a Python dictionary.", "response": "def to_dict(self):\n \"\"\"Transforms the object to a Python dictionary.\n\n Returns:\n (dict|None): The link as an alternative serialization format.\n \"\"\"\n if self.txid is None and self.output is None:\n return None\n else:\n return {\n 'transaction_id': self.txid,\n 'output_index': self.output,\n }"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef to_dict(self):\n # TODO FOR CC: It must be able to recognize a hashlock condition\n # and fulfillment!\n condition = {}\n try:\n condition['details'] = _fulfillment_to_details(self.fulfillment)\n except AttributeError:\n pass\n\n try:\n condition['uri'] = self.fulfillment.condition_uri\n except AttributeError:\n condition['uri'] = self.fulfillment\n\n output = {\n 'public_keys': self.public_keys,\n 'condition': condition,\n 'amount': str(self.amount),\n }\n return output", "response": "Transforms the object to a Python dictionary."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ngenerate a new Output from a specifically formed tuple or list.", "response": "def generate(cls, public_keys, amount):\n \"\"\"Generates a Output from a specifically formed tuple or list.\n\n Note:\n If a ThresholdCondition has to be generated where the threshold\n is always the number of subconditions it is split between, a\n list of the following structure is sufficient:\n\n [(address|condition)*, [(address|condition)*, ...], ...]\n\n Args:\n public_keys (:obj:`list` of :obj:`str`): The public key of\n the users that should be able to fulfill the Condition\n that is being created.\n amount (:obj:`int`): The amount locked by the Output.\n\n Returns:\n An Output that can be used in a Transaction.\n\n Raises:\n TypeError: If `public_keys` is not an instance of `list`.\n ValueError: If `public_keys` is an empty list.\n \"\"\"\n threshold = len(public_keys)\n if not isinstance(amount, int):\n raise TypeError('`amount` must be a int')\n if amount < 1:\n raise AmountError('`amount` needs to be greater than zero')\n if not isinstance(public_keys, list):\n raise TypeError('`public_keys` must be an instance of list')\n if len(public_keys) == 0:\n raise ValueError('`public_keys` needs to contain at least one'\n 'owner')\n elif len(public_keys) == 1 and not isinstance(public_keys[0], list):\n if isinstance(public_keys[0], Fulfillment):\n ffill = public_keys[0]\n else:\n ffill = Ed25519Sha256(\n public_key=base58.b58decode(public_keys[0]))\n return cls(ffill, public_keys, amount=amount)\n else:\n initial_cond = ThresholdSha256(threshold=threshold)\n threshold_cond = reduce(cls._gen_condition, public_keys,\n initial_cond)\n return cls(threshold_cond, public_keys, amount=amount)"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ngenerates ThresholdSha256 conditions from a list of new owners or a single owner.", "response": "def _gen_condition(cls, initial, new_public_keys):\n \"\"\"Generates ThresholdSha256 conditions from a list of new owners.\n\n Note:\n This method is intended only to be used with a reduce function.\n For a description on how to use this method, see\n :meth:`~.Output.generate`.\n\n Args:\n initial (:class:`cryptoconditions.ThresholdSha256`):\n A Condition representing the overall root.\n new_public_keys (:obj:`list` of :obj:`str`|str): A list of new\n owners or a single new owner.\n\n Returns:\n :class:`cryptoconditions.ThresholdSha256`:\n \"\"\"\n try:\n threshold = len(new_public_keys)\n except TypeError:\n threshold = None\n\n if isinstance(new_public_keys, list) and len(new_public_keys) > 1:\n ffill = ThresholdSha256(threshold=threshold)\n reduce(cls._gen_condition, new_public_keys, ffill)\n elif isinstance(new_public_keys, list) and len(new_public_keys) <= 1:\n raise ValueError('Sublist cannot contain single owner')\n else:\n try:\n new_public_keys = new_public_keys.pop()\n except AttributeError:\n pass\n # NOTE: Instead of submitting base58 encoded addresses, a user\n # of this class can also submit fully instantiated\n # Cryptoconditions. In the case of casting\n # `new_public_keys` to a Ed25519Fulfillment with the\n # result of a `TypeError`, we're assuming that\n # `new_public_keys` is a Cryptocondition then.\n if isinstance(new_public_keys, Fulfillment):\n ffill = new_public_keys\n else:\n ffill = Ed25519Sha256(\n public_key=base58.b58decode(new_public_keys))\n initial.add_subfulfillment(ffill)\n return initial"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef from_dict(cls, data):\n try:\n fulfillment = _fulfillment_from_details(data['condition']['details'])\n except KeyError:\n # NOTE: Hashlock condition case\n fulfillment = data['condition']['uri']\n try:\n amount = int(data['amount'])\n except ValueError:\n raise AmountError('Invalid amount: %s' % data['amount'])\n return cls(fulfillment, data['public_keys'], amount)", "response": "Transforms a Python dictionary to an Output object."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef transfer(cls, inputs, recipients, asset_id, metadata=None):\n (inputs, outputs) = cls.validate_transfer(inputs, recipients, asset_id, metadata)\n return cls(cls.TRANSFER, {'id': asset_id}, inputs, outputs, metadata)", "response": "A simple way to generate a Transaction object."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nconvert a Transaction s outputs to spendable inputs.", "response": "def to_inputs(self, indices=None):\n \"\"\"Converts a Transaction's outputs to spendable inputs.\n\n Note:\n Takes the Transaction's outputs and derives inputs\n from that can then be passed into `Transaction.transfer` as\n `inputs`.\n A list of integers can be passed to `indices` that\n defines which outputs should be returned as inputs.\n If no `indices` are passed (empty list or None) all\n outputs of the Transaction are returned.\n\n Args:\n indices (:obj:`list` of int): Defines which\n outputs should be returned as inputs.\n\n Returns:\n :obj:`list` of :class:`~bigchaindb.common.transaction.\n Input`\n \"\"\"\n # NOTE: If no indices are passed, we just assume to take all outputs\n # as inputs.\n indices = indices or range(len(self.outputs))\n return [\n Input(self.outputs[idx].fulfillment,\n self.outputs[idx].public_keys,\n TransactionLink(self.id, idx))\n for idx in indices\n ]"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef add_input(self, input_):\n if not isinstance(input_, Input):\n raise TypeError('`input_` must be a Input instance')\n self.inputs.append(input_)", "response": "Adds an input to a Transaction s list of inputs."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef add_output(self, output):\n if not isinstance(output, Output):\n raise TypeError('`output` must be an Output instance or None')\n self.outputs.append(output)", "response": "Adds an output to a Transaction s list of outputs."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef sign(self, private_keys):\n # TODO: Singing should be possible with at least one of all private\n # keys supplied to this method.\n if private_keys is None or not isinstance(private_keys, list):\n raise TypeError('`private_keys` must be a list instance')\n\n # NOTE: Generate public keys from private keys and match them in a\n # dictionary:\n # key: public_key\n # value: private_key\n def gen_public_key(private_key):\n # TODO FOR CC: Adjust interface so that this function becomes\n # unnecessary\n\n # cc now provides a single method `encode` to return the key\n # in several different encodings.\n public_key = private_key.get_verifying_key().encode()\n # Returned values from cc are always bytestrings so here we need\n # to decode to convert the bytestring into a python str\n return public_key.decode()\n\n key_pairs = {gen_public_key(PrivateKey(private_key)):\n PrivateKey(private_key) for private_key in private_keys}\n\n tx_dict = self.to_dict()\n tx_dict = Transaction._remove_signatures(tx_dict)\n tx_serialized = Transaction._to_str(tx_dict)\n for i, input_ in enumerate(self.inputs):\n self.inputs[i] = self._sign_input(input_, tx_serialized, key_pairs)\n\n self._hash()\n\n return self", "response": "Fulfills a previous Transaction s Output by signing Inputs."} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nsigns a single Input.", "response": "def _sign_input(cls, input_, message, key_pairs):\n \"\"\"Signs a single Input.\n\n Note:\n This method works only for the following Cryptoconditions\n currently:\n - Ed25519Fulfillment\n - ThresholdSha256.\n\n Args:\n input_ (:class:`~bigchaindb.common.transaction.\n Input`) The Input to be signed.\n message (str): The message to be signed\n key_pairs (dict): The keys to sign the Transaction with.\n \"\"\"\n if isinstance(input_.fulfillment, Ed25519Sha256):\n return cls._sign_simple_signature_fulfillment(input_, message,\n key_pairs)\n elif isinstance(input_.fulfillment, ThresholdSha256):\n return cls._sign_threshold_signature_fulfillment(input_, message,\n key_pairs)\n else:\n raise ValueError(\"Fulfillment couldn't be matched to \"\n 'Cryptocondition fulfillment type.')"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nsigns a simple signaturefulfillment.", "response": "def _sign_simple_signature_fulfillment(cls, input_, message, key_pairs):\n \"\"\"Signs a Ed25519Fulfillment.\n\n Args:\n input_ (:class:`~bigchaindb.common.transaction.\n Input`) The input to be signed.\n message (str): The message to be signed\n key_pairs (dict): The keys to sign the Transaction with.\n \"\"\"\n # NOTE: To eliminate the dangers of accidentally signing a condition by\n # reference, we remove the reference of input_ here\n # intentionally. If the user of this class knows how to use it,\n # this should never happen, but then again, never say never.\n input_ = deepcopy(input_)\n public_key = input_.owners_before[0]\n message = sha3_256(message.encode())\n if input_.fulfills:\n message.update('{}{}'.format(\n input_.fulfills.txid, input_.fulfills.output).encode())\n\n try:\n # cryptoconditions makes no assumptions of the encoding of the\n # message to sign or verify. It only accepts bytestrings\n input_.fulfillment.sign(\n message.digest(), base58.b58decode(key_pairs[public_key].encode()))\n except KeyError:\n raise KeypairMismatchException('Public key {} is not a pair to '\n 'any of the private keys'\n .format(public_key))\n return input_"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _sign_threshold_signature_fulfillment(cls, input_, message, key_pairs):\n input_ = deepcopy(input_)\n message = sha3_256(message.encode())\n if input_.fulfills:\n message.update('{}{}'.format(\n input_.fulfills.txid, input_.fulfills.output).encode())\n\n for owner_before in set(input_.owners_before):\n # TODO: CC should throw a KeypairMismatchException, instead of\n # our manual mapping here\n\n # TODO FOR CC: Naming wise this is not so smart,\n # `get_subcondition` in fact doesn't return a\n # condition but a fulfillment\n\n # TODO FOR CC: `get_subcondition` is singular. One would not\n # expect to get a list back.\n ccffill = input_.fulfillment\n subffills = ccffill.get_subcondition_from_vk(\n base58.b58decode(owner_before))\n if not subffills:\n raise KeypairMismatchException('Public key {} cannot be found '\n 'in the fulfillment'\n .format(owner_before))\n try:\n private_key = key_pairs[owner_before]\n except KeyError:\n raise KeypairMismatchException('Public key {} is not a pair '\n 'to any of the private keys'\n .format(owner_before))\n\n # cryptoconditions makes no assumptions of the encoding of the\n # message to sign or verify. It only accepts bytestrings\n for subffill in subffills:\n subffill.sign(\n message.digest(), base58.b58decode(private_key.encode()))\n return input_", "response": "Signs a ThresholdSha256.\n\n Args:\n input_ (:class:`~bigchaindb.common.transaction.\n Input`) The Input to be signed.\n message (str): The message to be signed\n key_pairs (dict): The keys to sign the Transaction with."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _inputs_valid(self, output_condition_uris):\n\n if len(self.inputs) != len(output_condition_uris):\n raise ValueError('Inputs and '\n 'output_condition_uris must have the same count')\n\n tx_dict = self.tx_dict if self.tx_dict else self.to_dict()\n tx_dict = Transaction._remove_signatures(tx_dict)\n tx_dict['id'] = None\n tx_serialized = Transaction._to_str(tx_dict)\n\n def validate(i, output_condition_uri=None):\n \"\"\"Validate input against output condition URI\"\"\"\n return self._input_valid(self.inputs[i], self.operation,\n tx_serialized, output_condition_uri)\n\n return all(validate(i, cond)\n for i, cond in enumerate(output_condition_uris))", "response": "Validates an Input against a given set of Outputs."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef to_dict(self):\n return {\n 'inputs': [input_.to_dict() for input_ in self.inputs],\n 'outputs': [output.to_dict() for output in self.outputs],\n 'operation': str(self.operation),\n 'metadata': self.metadata,\n 'asset': self.asset,\n 'version': self.version,\n 'id': self._id,\n }", "response": "Transforms the object to a Python dictionary."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ngetting the asset id from a list of Transactions.", "response": "def get_asset_id(cls, transactions):\n \"\"\"Get the asset id from a list of :class:`~.Transactions`.\n\n This is useful when we want to check if the multiple inputs of a\n transaction are related to the same asset id.\n\n Args:\n transactions (:obj:`list` of :class:`~bigchaindb.common.\n transaction.Transaction`): A list of Transactions.\n Usually input Transactions that should have a matching\n asset ID.\n\n Returns:\n str: ID of the asset.\n\n Raises:\n :exc:`AssetIdMismatch`: If the inputs are related to different\n assets.\n \"\"\"\n\n if not isinstance(transactions, list):\n transactions = [transactions]\n\n # create a set of the transactions' asset ids\n asset_ids = {tx.id if tx.operation == tx.CREATE\n else tx.asset['id']\n for tx in transactions}\n\n # check that all the transasctions have the same asset id\n if len(asset_ids) > 1:\n raise AssetIdMismatch(('All inputs of all transactions passed'\n ' need to have the same asset id'))\n return asset_ids.pop()"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nvalidate the ID of a transaction.", "response": "def validate_id(tx_body):\n \"\"\"Validate the transaction ID of a transaction\n\n Args:\n tx_body (dict): The Transaction to be transformed.\n \"\"\"\n # NOTE: Remove reference to avoid side effects\n # tx_body = deepcopy(tx_body)\n tx_body = rapidjson.loads(rapidjson.dumps(tx_body))\n\n try:\n proposed_tx_id = tx_body['id']\n except KeyError:\n raise InvalidHash('No transaction id found!')\n\n tx_body['id'] = None\n\n tx_body_serialized = Transaction._to_str(tx_body)\n valid_tx_id = Transaction._to_hash(tx_body_serialized)\n\n if proposed_tx_id != valid_tx_id:\n err_msg = (\"The transaction's id '{}' isn't equal to \"\n \"the hash of its body, i.e. it's not valid.\")\n raise InvalidHash(err_msg.format(proposed_tx_id))"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ntransforming a Python dictionary to a Transaction object.", "response": "def from_dict(cls, tx, skip_schema_validation=True):\n \"\"\"Transforms a Python dictionary to a Transaction object.\n\n Args:\n tx_body (dict): The Transaction to be transformed.\n\n Returns:\n :class:`~bigchaindb.common.transaction.Transaction`\n \"\"\"\n operation = tx.get('operation', Transaction.CREATE) if isinstance(tx, dict) else Transaction.CREATE\n cls = Transaction.resolve_class(operation)\n\n if not skip_schema_validation:\n cls.validate_id(tx)\n cls.validate_schema(tx)\n\n inputs = [Input.from_dict(input_) for input_ in tx['inputs']]\n outputs = [Output.from_dict(output) for output in tx['outputs']]\n return cls(tx['operation'], tx['asset'], inputs, outputs,\n tx['metadata'], tx['version'], hash_id=tx['id'], tx_dict=tx)"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef from_db(cls, bigchain, tx_dict_list):\n return_list = True\n if isinstance(tx_dict_list, dict):\n tx_dict_list = [tx_dict_list]\n return_list = False\n\n tx_map = {}\n tx_ids = []\n for tx in tx_dict_list:\n tx.update({'metadata': None})\n tx_map[tx['id']] = tx\n tx_ids.append(tx['id'])\n\n assets = list(bigchain.get_assets(tx_ids))\n for asset in assets:\n if asset is not None:\n tx = tx_map[asset['id']]\n del asset['id']\n tx['asset'] = asset\n\n tx_ids = list(tx_map.keys())\n metadata_list = list(bigchain.get_metadata(tx_ids))\n for metadata in metadata_list:\n tx = tx_map[metadata['id']]\n tx.update({'metadata': metadata.get('metadata')})\n\n if return_list:\n tx_list = []\n for tx_id, tx in tx_map.items():\n tx_list.append(cls.from_dict(tx))\n return tx_list\n else:\n tx = list(tx_map.values())[0]\n return cls.from_dict(tx)", "response": "This method reconstructs a transaction dict that was returned by the database and returns a list of Transaction objects."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef resolve_class(operation):\n\n create_txn_class = Transaction.type_registry.get(Transaction.CREATE)\n return Transaction.type_registry.get(operation, create_txn_class)", "response": "For the given tx return its implementation class"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef connect(backend=None, host=None, port=None, name=None, max_tries=None,\n connection_timeout=None, replicaset=None, ssl=None, login=None, password=None,\n ca_cert=None, certfile=None, keyfile=None, keyfile_passphrase=None,\n crlfile=None):\n \"\"\"Create a new connection to the database backend.\n\n All arguments default to the current configuration's values if not\n given.\n\n Args:\n backend (str): the name of the backend to use.\n host (str): the host to connect to.\n port (int): the port to connect to.\n name (str): the name of the database to use.\n replicaset (str): the name of the replica set (only relevant for\n MongoDB connections).\n\n Returns:\n An instance of :class:`~bigchaindb.backend.connection.Connection`\n based on the given (or defaulted) :attr:`backend`.\n\n Raises:\n :exc:`~ConnectionError`: If the connection to the database fails.\n :exc:`~ConfigurationError`: If the given (or defaulted) :attr:`backend`\n is not supported or could not be loaded.\n :exc:`~AuthenticationError`: If there is a OperationFailure due to\n Authentication failure after connecting to the database.\n \"\"\"\n\n backend = backend or bigchaindb.config['database']['backend']\n host = host or bigchaindb.config['database']['host']\n port = port or bigchaindb.config['database']['port']\n dbname = name or bigchaindb.config['database']['name']\n # Not sure how to handle this here. This setting is only relevant for\n # mongodb.\n # I added **kwargs for both RethinkDBConnection and MongoDBConnection\n # to handle these these additional args. In case of RethinkDBConnection\n # it just does not do anything with it.\n #\n # UPD: RethinkDBConnection is not here anymore cause we no longer support RethinkDB.\n # The problem described above might be reconsidered next time we introduce a backend,\n # if it ever happens.\n replicaset = replicaset or bigchaindb.config['database'].get('replicaset')\n ssl = ssl if ssl is not None else bigchaindb.config['database'].get('ssl', False)\n login = login or bigchaindb.config['database'].get('login')\n password = password or bigchaindb.config['database'].get('password')\n ca_cert = ca_cert or bigchaindb.config['database'].get('ca_cert', None)\n certfile = certfile or bigchaindb.config['database'].get('certfile', None)\n keyfile = keyfile or bigchaindb.config['database'].get('keyfile', None)\n keyfile_passphrase = keyfile_passphrase or bigchaindb.config['database'].get('keyfile_passphrase', None)\n crlfile = crlfile or bigchaindb.config['database'].get('crlfile', None)\n\n try:\n module_name, _, class_name = BACKENDS[backend].rpartition('.')\n Class = getattr(import_module(module_name), class_name)\n except KeyError:\n raise ConfigurationError('Backend `{}` is not supported. '\n 'BigchainDB currently supports {}'.format(backend, BACKENDS.keys()))\n except (ImportError, AttributeError) as exc:\n raise ConfigurationError('Error loading backend `{}`'.format(backend)) from exc\n\n logger.debug('Connection: {}'.format(Class))\n return Class(host=host, port=port, dbname=dbname,\n max_tries=max_tries, connection_timeout=connection_timeout,\n replicaset=replicaset, ssl=ssl, login=login, password=password,\n ca_cert=ca_cert, certfile=certfile, keyfile=keyfile,\n keyfile_passphrase=keyfile_passphrase, crlfile=crlfile)", "response": "Create a new connection to the database."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef connect(self):\n\n attempt = 0\n for i in self.max_tries_counter:\n attempt += 1\n try:\n self._conn = self._connect()\n except ConnectionError as exc:\n logger.warning('Attempt %s/%s. Connection to %s:%s failed after %sms.',\n attempt, self.max_tries if self.max_tries != 0 else '\u221e',\n self.host, self.port, self.connection_timeout)\n if attempt == self.max_tries:\n logger.critical('Cannot connect to the Database. Giving up.')\n raise ConnectionError() from exc\n else:\n break", "response": "Try to connect to the database."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get(self):\n\n pool = current_app.config['bigchain_pool']\n\n with pool() as bigchain:\n validators = bigchain.get_validators()\n\n return validators", "response": "API endpoint to get validators set."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns the validator set from the most recent approved block", "response": "def get_validator_change(cls, bigchain):\n \"\"\"Return the validator set from the most recent approved block\n\n :return: {\n 'height': ,\n 'validators': \n }\n \"\"\"\n latest_block = bigchain.get_latest_block()\n if latest_block is None:\n return None\n return bigchain.get_validator_change(latest_block['height'])"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_validators(cls, bigchain, height=None):\n validators = {}\n for validator in bigchain.get_validators(height):\n # NOTE: we assume that Tendermint encodes public key in base64\n public_key = public_key_from_ed25519_key(key_from_base64(validator['public_key']['value']))\n validators[public_key] = validator['voting_power']\n\n return validators", "response": "Return a dictionary of validators with key as public_key and the voting_power value as the voting_power"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef recipients(cls, bigchain):\n\n recipients = []\n for public_key, voting_power in cls.get_validators(bigchain).items():\n recipients.append(([public_key], voting_power))\n\n return recipients", "response": "Convert validator dictionary to a list of recipients for Transaction"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nvalidating an election transaction.", "response": "def validate(self, bigchain, current_transactions=[]):\n \"\"\"Validate election transaction\n\n NOTE:\n * A valid election is initiated by an existing validator.\n\n * A valid election is one where voters are validators and votes are\n allocated according to the voting power of each validator node.\n\n Args:\n :param bigchain: (BigchainDB) an instantiated bigchaindb.lib.BigchainDB object.\n :param current_transactions: (list) A list of transactions to be validated along with the election\n\n Returns:\n Election: a Election object or an object of the derived Election subclass.\n\n Raises:\n ValidationError: If the election is invalid\n \"\"\"\n input_conditions = []\n\n duplicates = any(txn for txn in current_transactions if txn.id == self.id)\n if bigchain.is_committed(self.id) or duplicates:\n raise DuplicateTransaction('transaction `{}` already exists'\n .format(self.id))\n\n if not self.inputs_valid(input_conditions):\n raise InvalidSignature('Transaction signature is invalid.')\n\n current_validators = self.get_validators(bigchain)\n\n # NOTE: Proposer should be a single node\n if len(self.inputs) != 1 or len(self.inputs[0].owners_before) != 1:\n raise MultipleInputsError('`tx_signers` must be a list instance of length one')\n\n # NOTE: Check if the proposer is a validator.\n [election_initiator_node_pub_key] = self.inputs[0].owners_before\n if election_initiator_node_pub_key not in current_validators.keys():\n raise InvalidProposer('Public key is not a part of the validator set')\n\n # NOTE: Check if all validators have been assigned votes equal to their voting power\n if not self.is_same_topology(current_validators, self.outputs):\n raise UnequalValidatorSet('Validator set much be exactly same to the outputs of election')\n\n return self"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nvalidating the election transaction.", "response": "def validate_schema(cls, tx):\n \"\"\"Validate the election transaction. Since `ELECTION` extends `CREATE` transaction, all the validations for\n `CREATE` transaction should be inherited\n \"\"\"\n _validate_schema(TX_SCHEMA_COMMON, tx)\n _validate_schema(TX_SCHEMA_CREATE, tx)\n if cls.TX_SCHEMA_CUSTOM:\n _validate_schema(cls.TX_SCHEMA_CUSTOM, tx)"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nchecking if the election can be concluded or not.", "response": "def has_concluded(self, bigchain, current_votes=[]):\n \"\"\"Check if the election can be concluded or not.\n\n * Elections can only be concluded if the validator set has not changed\n since the election was initiated.\n * Elections can be concluded only if the current votes form a supermajority.\n\n Custom elections may override this function and introduce additional checks.\n \"\"\"\n if self.has_validator_set_changed(bigchain):\n return False\n\n election_pk = self.to_public_key(self.id)\n votes_committed = self.get_commited_votes(bigchain, election_pk)\n votes_current = self.count_votes(election_pk, current_votes)\n\n total_votes = sum(output.amount for output in self.outputs)\n if (votes_committed < (2/3) * total_votes) and \\\n (votes_committed + votes_current >= (2/3)*total_votes):\n return True\n\n return False"} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nprocesses the block and returns a list of all the elections and vote transactions that are in the block.", "response": "def process_block(cls, bigchain, new_height, txns):\n \"\"\"Looks for election and vote transactions inside the block, records\n and processes elections.\n\n Every election is recorded in the database.\n\n Every vote has a chance to conclude the corresponding election. When\n an election is concluded, the corresponding database record is\n marked as such.\n\n Elections and votes are processed in the order in which they\n appear in the block. Elections are concluded in the order of\n appearance of their first votes in the block.\n\n For every election concluded in the block, calls its `on_approval`\n method. The returned value of the last `on_approval`, if any,\n is a validator set update to be applied in one of the following blocks.\n\n `on_approval` methods are implemented by elections of particular type.\n The method may contain side effects but should be idempotent. To account\n for other concluded elections, if it requires so, the method should\n rely on the database state.\n \"\"\"\n # elections initiated in this block\n initiated_elections = cls._get_initiated_elections(new_height, txns)\n\n if initiated_elections:\n bigchain.store_elections(initiated_elections)\n\n # elections voted for in this block and their votes\n elections = cls._get_votes(txns)\n\n validator_update = None\n for election_id, votes in elections.items():\n election = bigchain.get_transaction(election_id)\n if election is None:\n continue\n\n if not election.has_concluded(bigchain, votes):\n continue\n\n validator_update = election.on_approval(bigchain, new_height)\n election.store(bigchain, new_height, is_concluded=True)\n\n return [validator_update] if validator_update else []"} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nroll back the transaction with the given list of transaction IDs.", "response": "def rollback(cls, bigchain, new_height, txn_ids):\n \"\"\"Looks for election and vote transactions inside the block and\n cleans up the database artifacts possibly created in `process_blocks`.\n\n Part of the `end_block`/`commit` crash recovery.\n \"\"\"\n\n # delete election records for elections initiated at this height and\n # elections concluded at this height\n bigchain.delete_elections(new_height)\n\n txns = [bigchain.get_transaction(tx_id) for tx_id in txn_ids]\n\n elections = cls._get_votes(txns)\n for election_id in elections:\n election = bigchain.get_transaction(election_id)\n election.on_rollback(bigchain, new_height)"} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nvalidates the validator election vote transaction.", "response": "def validate_schema(cls, tx):\n \"\"\"Validate the validator election vote transaction. Since `VOTE` extends `TRANSFER`\n transaction, all the validations for `CREATE` transaction should be inherited\n \"\"\"\n _validate_schema(TX_SCHEMA_COMMON, tx)\n _validate_schema(TX_SCHEMA_TRANSFER, tx)\n _validate_schema(cls.TX_SCHEMA_CUSTOM, tx)"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nmapping a function to the leafs of a mapping.", "response": "def map_leafs(func, mapping):\n \"\"\"Map a function to the leafs of a mapping.\"\"\"\n\n def _inner(mapping, path=None):\n if path is None:\n path = []\n\n for key, val in mapping.items():\n if isinstance(val, collections.Mapping):\n _inner(val, path + [key])\n else:\n mapping[key] = func(val, path=path+[key])\n\n return mapping\n\n return _inner(copy.deepcopy(mapping))"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef file_config(filename=None):\n logger.debug('On entry into file_config(), filename = {}'.format(filename))\n\n if filename is None:\n filename = CONFIG_DEFAULT_PATH\n\n logger.debug('file_config() will try to open `{}`'.format(filename))\n with open(filename) as f:\n try:\n config = json.load(f)\n except ValueError as err:\n raise exceptions.ConfigurationError(\n 'Failed to parse the JSON configuration from `{}`, {}'.format(filename, err)\n )\n\n logger.info('Configuration loaded from `{}`'.format(filename))\n\n return config", "response": "Returns the config values found in a JSON file."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef env_config(config):\n\n def load_from_env(value, path):\n var_name = CONFIG_SEP.join([CONFIG_PREFIX] + list(map(lambda s: s.upper(), path)))\n\n return os.environ.get(var_name, value)\n\n return map_leafs(load_from_env, config)", "response": "Returns a new configuration with the values found in the environment."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef update_types(config, reference, list_sep=':'):\n\n def _coerce(current, value):\n # Coerce a value to the `current` type.\n try:\n # First we try to apply current to the value, since it\n # might be a function\n return current(value)\n except TypeError:\n # Then we check if current is a list AND if the value\n # is a string.\n if isinstance(current, list) and isinstance(value, str):\n # If so, we use the colon as the separator\n return value.split(list_sep)\n\n try:\n # If we are here, we should try to apply the type\n # of `current` to the value\n return type(current)(value)\n except TypeError:\n # Worst case scenario we return the value itself.\n return value\n\n def _update_type(value, path):\n current = reference\n\n for elem in path:\n try:\n current = current[elem]\n except KeyError:\n return value\n\n return _coerce(current, value)\n\n return map_leafs(_update_type, config)", "response": "Return a new configuration where all the values in types\nFormula are aligned with the ones in the default configuration\nFormula."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef set_config(config):\n # Deep copy the default config into bigchaindb.config\n bigchaindb.config = copy.deepcopy(bigchaindb._config)\n # Update the default config with whatever is in the passed config\n update(bigchaindb.config, update_types(config, bigchaindb.config))\n bigchaindb.config['CONFIGURED'] = True", "response": "Set the default config dict for the current language of the current language and update that with whatever is in the provided config dict."} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nupdates the config dict with what is in the provided config dict and then set the CONFIGURED flag to True", "response": "def update_config(config):\n \"\"\"Update bigchaindb.config with whatever is in the provided config dict,\n and then set bigchaindb.config['CONFIGURED'] = True\n\n Args:\n config (dict): the config dict to read for changes\n to the default config\n \"\"\"\n\n # Update the default config with whatever is in the passed config\n update(bigchaindb.config, update_types(config, bigchaindb.config))\n bigchaindb.config['CONFIGURED'] = True"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nwrite the provided configuration to a specific location.", "response": "def write_config(config, filename=None):\n \"\"\"Write the provided configuration to a specific location.\n\n Args:\n config (dict): a dictionary with the configuration to load.\n filename (str): the name of the file that will store the new configuration. Defaults to ``None``.\n If ``None``, the HOME of the current user and the string ``.bigchaindb`` will be used.\n \"\"\"\n if not filename:\n filename = CONFIG_DEFAULT_PATH\n\n with open(filename, 'w') as f:\n json.dump(config, f, indent=4)"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nrun ``file_config`` and ``env_config`` if the module has not been initialized.", "response": "def autoconfigure(filename=None, config=None, force=False):\n \"\"\"Run ``file_config`` and ``env_config`` if the module has not\n been initialized.\n \"\"\"\n if not force and is_configured():\n logger.debug('System already configured, skipping autoconfiguration')\n return\n\n # start with the current configuration\n newconfig = bigchaindb.config\n\n # update configuration from file\n try:\n newconfig = update(newconfig, file_config(filename=filename))\n except FileNotFoundError as e:\n if filename:\n raise\n else:\n logger.info('Cannot find config file `%s`.' % e.filename)\n\n # override configuration with env variables\n newconfig = env_config(newconfig)\n if config:\n newconfig = update(newconfig, config)\n set_config(newconfig)"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nfind and load the chosen validation plugin.", "response": "def load_validation_plugin(name=None):\n \"\"\"Find and load the chosen validation plugin.\n\n Args:\n name (string): the name of the entry_point, as advertised in the\n setup.py of the providing package.\n\n Returns:\n an uninstantiated subclass of ``bigchaindb.validation.AbstractValidationRules``\n \"\"\"\n if not name:\n return BaseValidationRules\n\n # TODO: This will return the first plugin with group `bigchaindb.validation`\n # and name `name` in the active WorkingSet.\n # We should probably support Requirements specs in the config, e.g.\n # validation_plugin: 'my-plugin-package==0.0.1;default'\n plugin = None\n for entry_point in iter_entry_points('bigchaindb.validation', name):\n plugin = entry_point.load()\n\n # No matching entry_point found\n if not plugin:\n raise ResolutionError(\n 'No plugin found in group `bigchaindb.validation` with name `{}`'.\n format(name))\n\n # Is this strictness desireable?\n # It will probably reduce developer headaches in the wild.\n if not issubclass(plugin, (BaseValidationRules,)):\n raise TypeError('object of type \"{}\" does not implement `bigchaindb.'\n 'validation.BaseValidationRules`'.format(type(plugin)))\n\n return plugin"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ncreates a pool that imposes a limit on the number of stored items.", "response": "def pool(builder, size, timeout=None):\n \"\"\"Create a pool that imposes a limit on the number of stored\n instances.\n\n Args:\n builder: a function to build an instance.\n size: the size of the pool.\n timeout(Optional[float]): the seconds to wait before raising\n a ``queue.Empty`` exception if no instances are available\n within that time.\n Raises:\n If ``timeout`` is defined but the request is taking longer\n than the specified time, the context manager will raise\n a ``queue.Empty`` exception.\n\n Returns:\n A context manager that can be used with the ``with``\n statement.\n\n \"\"\"\n\n lock = threading.Lock()\n local_pool = queue.Queue()\n current_size = 0\n\n @contextlib.contextmanager\n def pooled():\n nonlocal current_size\n instance = None\n\n # If we still have free slots, then we have room to create new\n # instances.\n if current_size < size:\n with lock:\n # We need to check again if we have slots available, since\n # the situation might be different after acquiring the lock\n if current_size < size:\n current_size += 1\n instance = builder()\n\n # Watchout: current_size can be equal to size if the previous part of\n # the function has been executed, that's why we need to check if the\n # instance is None.\n if instance is None:\n instance = local_pool.get(timeout=timeout)\n\n yield instance\n\n local_pool.put(instance)\n\n return pooled"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef condition_details_has_owner(condition_details, owner):\n if 'subconditions' in condition_details:\n result = condition_details_has_owner(condition_details['subconditions'], owner)\n if result:\n return True\n\n elif isinstance(condition_details, list):\n for subcondition in condition_details:\n result = condition_details_has_owner(subcondition, owner)\n if result:\n return True\n else:\n if 'public_key' in condition_details \\\n and owner == condition_details['public_key']:\n return True\n return False", "response": "Check if the public_key of owner is in the condition details\n ."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nchecks if the Tendermint version is compatible with BigchainDB", "response": "def tendermint_version_is_compatible(running_tm_ver):\n \"\"\"\n Check Tendermint compatability with BigchainDB server\n\n :param running_tm_ver: Version number of the connected Tendermint instance\n :type running_tm_ver: str\n :return: True/False depending on the compatability with BigchainDB server\n :rtype: bool\n \"\"\"\n\n # Splitting because version can look like this e.g. 0.22.8-40d6dc2e\n tm_ver = running_tm_ver.split('-')\n if not tm_ver:\n return False\n for ver in __tm_supported_versions__:\n if version.parse(ver) == version.parse(tm_ver[0]):\n return True\n return False"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nruns the recorded chain of methods on instance.", "response": "def run(self, instance):\n \"\"\"Run the recorded chain of methods on `instance`.\n\n Args:\n instance: an object.\n \"\"\"\n\n last = instance\n\n for item in self.stack:\n if isinstance(item, str):\n last = getattr(last, item)\n else:\n last = last(*item[0], **item[1])\n\n self.stack = []\n return last"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef base_ws_uri():\n\n scheme = config['wsserver']['advertised_scheme']\n host = config['wsserver']['advertised_host']\n port = config['wsserver']['advertised_port']\n return '{}://{}:{}'.format(scheme, host, port)", "response": "Returns the base websocket URL that is advertised to external clients."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get(self, tx_id):\n pool = current_app.config['bigchain_pool']\n\n with pool() as bigchain:\n tx = bigchain.get_transaction(tx_id)\n\n if not tx:\n return make_error(404)\n\n return tx.to_dict()", "response": "This endpoint returns the details about a transaction."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef post(self):\n parser = reqparse.RequestParser()\n parser.add_argument('mode', type=parameters.valid_mode,\n default='broadcast_tx_async')\n args = parser.parse_args()\n mode = str(args['mode'])\n\n pool = current_app.config['bigchain_pool']\n\n # `force` will try to format the body of the POST request even if the\n # `content-type` header is not set to `application/json`\n tx = request.get_json(force=True)\n\n try:\n tx_obj = Transaction.from_dict(tx)\n except SchemaValidationError as e:\n return make_error(\n 400,\n message='Invalid transaction schema: {}'.format(\n e.__cause__.message)\n )\n except ValidationError as e:\n return make_error(\n 400,\n 'Invalid transaction ({}): {}'.format(type(e).__name__, e)\n )\n\n with pool() as bigchain:\n try:\n bigchain.validate_transaction(tx_obj)\n except ValidationError as e:\n return make_error(\n 400,\n 'Invalid transaction ({}): {}'.format(type(e).__name__, e)\n )\n else:\n status_code, message = bigchain.write_transaction(tx_obj, mode)\n\n if status_code == 202:\n response = jsonify(tx)\n response.status_code = 202\n return response\n else:\n return make_error(status_code, message)", "response": "This endpoint is used to push transactions to the Federation. It is used to push transactions to the Federation."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ngetting outputs for a public key", "response": "def get_outputs_by_public_key(self, public_key):\n \"\"\"Get outputs for a public key\"\"\"\n txs = list(query.get_owned_ids(self.connection, public_key))\n return [TransactionLink(tx['id'], index)\n for tx in txs\n for index, output in enumerate(tx['outputs'])\n if condition_details_has_owner(output['condition']['details'],\n public_key)]"} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nremoves outputs that have been spent", "response": "def filter_spent_outputs(self, outputs):\n \"\"\"Remove outputs that have been spent\n\n Args:\n outputs: list of TransactionLink\n \"\"\"\n links = [o.to_dict() for o in outputs]\n txs = list(query.get_spending_transactions(self.connection, links))\n spends = {TransactionLink.from_dict(input_['fulfills'])\n for tx in txs\n for input_ in tx['inputs']}\n return [ff for ff in outputs if ff not in spends]"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nloading a schema from disk", "response": "def _load_schema(name, path=__file__):\n \"\"\"Load a schema from disk\"\"\"\n path = os.path.join(os.path.dirname(path), name + '.yaml')\n with open(path) as handle:\n schema = yaml.safe_load(handle)\n fast_schema = rapidjson.Validator(rapidjson.dumps(schema))\n return path, (schema, fast_schema)"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nvalidates data against a schema", "response": "def _validate_schema(schema, body):\n \"\"\"Validate data against a schema\"\"\"\n\n # Note\n #\n # Schema validation is currently the major CPU bottleneck of\n # BigchainDB. the `jsonschema` library validates python data structures\n # directly and produces nice error messages, but validation takes 4+ ms\n # per transaction which is pretty slow. The rapidjson library validates\n # much faster at 1.5ms, however it produces _very_ poor error messages.\n # For this reason we use both, rapidjson as an optimistic pathway and\n # jsonschema as a fallback in case there is a failure, so we can produce\n # a helpful error message.\n\n try:\n schema[1](rapidjson.dumps(body))\n except ValueError as exc:\n try:\n jsonschema.validate(body, schema[0])\n except jsonschema.ValidationError as exc2:\n raise SchemaValidationError(str(exc2)) from exc2\n logger.warning('code problem: jsonschema did not raise an exception, wheras rapidjson raised %s', exc)\n raise SchemaValidationError(str(exc)) from exc"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nvalidate a transaction dict.", "response": "def validate_transaction_schema(tx):\n \"\"\"Validate a transaction dict.\n\n TX_SCHEMA_COMMON contains properties that are common to all types of\n transaction. TX_SCHEMA_[TRANSFER|CREATE] add additional constraints on top.\n \"\"\"\n _validate_schema(TX_SCHEMA_COMMON, tx)\n if tx['operation'] == 'TRANSFER':\n _validate_schema(TX_SCHEMA_TRANSFER, tx)\n else:\n _validate_schema(TX_SCHEMA_CREATE, tx)"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef post_transaction(self, transaction, mode):\n if not mode or mode not in self.mode_list:\n raise ValidationError('Mode must be one of the following {}.'\n .format(', '.join(self.mode_list)))\n\n tx_dict = transaction.tx_dict if transaction.tx_dict else transaction.to_dict()\n payload = {\n 'method': mode,\n 'jsonrpc': '2.0',\n 'params': [encode_transaction(tx_dict)],\n 'id': str(uuid4())\n }\n # TODO: handle connection errors!\n return requests.post(self.endpoint, json=payload)", "response": "Submit a valid transaction to the mempool."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nsubmitting a valid transaction to the mempool.", "response": "def write_transaction(self, transaction, mode):\n # This method offers backward compatibility with the Web API.\n \"\"\"Submit a valid transaction to the mempool.\"\"\"\n response = self.post_transaction(transaction, mode)\n return self._process_post_response(response.json(), mode)"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef update_utxoset(self, transaction):\n spent_outputs = [\n spent_output for spent_output in transaction.spent_outputs\n ]\n if spent_outputs:\n self.delete_unspent_outputs(*spent_outputs)\n self.store_unspent_outputs(\n *[utxo._asdict() for utxo in transaction.unspent_outputs]\n )", "response": "Update the UTXO set given a transaction."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nstoring the given unspent outputs.", "response": "def store_unspent_outputs(self, *unspent_outputs):\n \"\"\"Store the given ``unspent_outputs`` (utxos).\n\n Args:\n *unspent_outputs (:obj:`tuple` of :obj:`dict`): Variable\n length tuple or list of unspent outputs.\n \"\"\"\n if unspent_outputs:\n return backend.query.store_unspent_outputs(\n self.connection, *unspent_outputs)"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning the merkle root of the utxoset.", "response": "def get_utxoset_merkle_root(self):\n \"\"\"Returns the merkle root of the utxoset. This implies that\n the utxoset is first put into a merkle tree.\n\n For now, the merkle tree and its root will be computed each\n time. This obviously is not efficient and a better approach\n that limits the repetition of the same computation when\n unnecesary should be sought. For instance, future optimizations\n could simply re-compute the branches of the tree that were\n affected by a change.\n\n The transaction hash (id) and output index should be sufficient\n to uniquely identify a utxo, and consequently only that\n information from a utxo record is needed to compute the merkle\n root. Hence, each node of the merkle tree should contain the\n tuple (txid, output_index).\n\n .. important:: The leaves of the tree will need to be sorted in\n some kind of lexicographical order.\n\n Returns:\n str: Merkle root in hexadecimal form.\n \"\"\"\n utxoset = backend.query.get_unspent_outputs(self.connection)\n # TODO Once ready, use the already pre-computed utxo_hash field.\n # See common/transactions.py for details.\n hashes = [\n sha3_256(\n '{}{}'.format(utxo['transaction_id'], utxo['output_index']).encode()\n ).digest() for utxo in utxoset\n ]\n # TODO Notice the sorted call!\n return merkleroot(sorted(hashes))"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef get_unspent_outputs(self):\n cursor = backend.query.get_unspent_outputs(self.connection)\n return (record for record in cursor)", "response": "Get the utxoset.\n generator of unspent outputs."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef delete_unspent_outputs(self, *unspent_outputs):\n if unspent_outputs:\n return backend.query.delete_unspent_outputs(\n self.connection, *unspent_outputs)", "response": "Deletes the given unspent outputs."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_transactions_filtered(self, asset_id, operation=None):\n txids = backend.query.get_txids_filtered(self.connection, asset_id,\n operation)\n for txid in txids:\n yield self.get_transaction(txid)", "response": "Get a list of transactions filtered on some criteria"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_outputs_filtered(self, owner, spent=None):\n outputs = self.fastquery.get_outputs_by_public_key(owner)\n if spent is None:\n return outputs\n elif spent is True:\n return self.fastquery.filter_unspent_outputs(outputs)\n elif spent is False:\n return self.fastquery.filter_spent_outputs(outputs)", "response": "Get a list of outputs filtered on some criteria"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_block(self, block_id):\n\n block = backend.query.get_block(self.connection, block_id)\n latest_block = self.get_latest_block()\n latest_block_height = latest_block['height'] if latest_block else 0\n\n if not block and block_id > latest_block_height:\n return\n\n result = {'height': block_id,\n 'transactions': []}\n\n if block:\n transactions = backend.query.get_transactions(self.connection, block['transactions'])\n result['transactions'] = [t.to_dict() for t in Transaction.from_db(self, transactions)]\n\n return result", "response": "Get the block with the specified block_id."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_block_containing_tx(self, txid):\n blocks = list(backend.query.get_block_with_transaction(self.connection, txid))\n if len(blocks) > 1:\n logger.critical('Transaction id %s exists in multiple blocks', txid)\n\n return [block['height'] for block in blocks]", "response": "Retrieve the list of blocks containing a transaction with transaction id txid"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nvalidate a transaction against the current status of the database.", "response": "def validate_transaction(self, tx, current_transactions=[]):\n \"\"\"Validate a transaction against the current status of the database.\"\"\"\n\n transaction = tx\n\n # CLEANUP: The conditional below checks for transaction in dict format.\n # It would be better to only have a single format for the transaction\n # throught the code base.\n if isinstance(transaction, dict):\n try:\n transaction = Transaction.from_dict(tx)\n except SchemaValidationError as e:\n logger.warning('Invalid transaction schema: %s', e.__cause__.message)\n return False\n except ValidationError as e:\n logger.warning('Invalid transaction (%s): %s', type(e).__name__, e)\n return False\n return transaction.validate(self, current_transactions)"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nsearching the assets for the given text search string.", "response": "def text_search(self, search, *, limit=0, table='assets'):\n \"\"\"Return an iterator of assets that match the text search\n\n Args:\n search (str): Text search string to query the text index\n limit (int, optional): Limit the number of returned documents.\n\n Returns:\n iter: An iterator of assets that match the text search.\n \"\"\"\n return backend.query.text_search(self.connection, search, limit=limit,\n table=table)"} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nstores a set of validators at a given height.", "response": "def store_validator_set(self, height, validators):\n \"\"\"Store validator set at a given `height`.\n NOTE: If the validator set already exists at that `height` then an\n exception will be raised.\n \"\"\"\n return backend.query.store_validator_set(self.connection, {'height': height,\n 'validators': validators})"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef migrate_abci_chain(self):\n latest_chain = self.get_latest_abci_chain()\n if latest_chain is None:\n return\n\n block = self.get_latest_block()\n\n suffix = '-migrated-at-height-'\n chain_id = latest_chain['chain_id']\n block_height_str = str(block['height'])\n new_chain_id = chain_id.split(suffix)[0] + suffix + block_height_str\n\n self.store_abci_chain(block['height'] + 1, new_chain_id, False)", "response": "Generate and record a new ABCI chain ID."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _multiprocessing_to_asyncio(in_queue, out_queue, loop):\n\n while True:\n value = in_queue.get()\n loop.call_soon_threadsafe(out_queue.put_nowait, value)", "response": "Bridge between a synchronous multiprocessing queue and an asyncio queue."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef websocket_handler(request):\n\n logger.debug('New websocket connection.')\n websocket = web.WebSocketResponse()\n yield from websocket.prepare(request)\n uuid = uuid4()\n request.app['dispatcher'].subscribe(uuid, websocket)\n\n while True:\n # Consume input buffer\n try:\n msg = yield from websocket.receive()\n except RuntimeError as e:\n logger.debug('Websocket exception: %s', str(e))\n break\n except CancelledError:\n logger.debug('Websocket closed')\n break\n if msg.type == aiohttp.WSMsgType.CLOSED:\n logger.debug('Websocket closed')\n break\n elif msg.type == aiohttp.WSMsgType.ERROR:\n logger.debug('Websocket exception: %s', websocket.exception())\n break\n\n request.app['dispatcher'].unsubscribe(uuid)\n return websocket", "response": "Handle a new socket connection."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef init_app(event_source, *, loop=None):\n\n dispatcher = Dispatcher(event_source)\n\n # Schedule the dispatcher\n loop.create_task(dispatcher.publish())\n\n app = web.Application(loop=loop)\n app['dispatcher'] = dispatcher\n app.router.add_get(EVENTS_ENDPOINT, websocket_handler)\n return app", "response": "Init the application server."} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncreates and start the WebSocket server.", "response": "def start(sync_event_source, loop=None):\n \"\"\"Create and start the WebSocket server.\"\"\"\n\n if not loop:\n loop = asyncio.get_event_loop()\n\n event_source = asyncio.Queue(loop=loop)\n\n bridge = threading.Thread(target=_multiprocessing_to_asyncio,\n args=(sync_event_source, event_source, loop),\n daemon=True)\n bridge.start()\n\n app = init_app(event_source, loop=loop)\n aiohttp.web.run_app(app,\n host=config['wsserver']['host'],\n port=config['wsserver']['port'])"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef publish(self):\n\n while True:\n event = yield from self.event_source.get()\n str_buffer = []\n\n if event == POISON_PILL:\n return\n\n if isinstance(event, str):\n str_buffer.append(event)\n\n elif event.type == EventTypes.BLOCK_VALID:\n str_buffer = map(json.dumps, eventify_block(event.data))\n\n for str_item in str_buffer:\n for _, websocket in self.subscribers.items():\n yield from websocket.send_str(str_item)", "response": "Publish new events to the subscribers."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncomputing the merkle root for a given list of bytes.", "response": "def merkleroot(hashes):\n \"\"\"Computes the merkle root for a given list.\n\n Args:\n hashes (:obj:`list` of :obj:`bytes`): The leaves of the tree.\n\n Returns:\n str: Merkle root in hexadecimal form.\n\n \"\"\"\n # XXX TEMPORARY -- MUST REVIEW and possibly CHANGE\n # The idea here is that the UTXO SET would be empty and this function\n # would be invoked to compute the merkle root, and since there is nothing,\n # i.e. an empty list, then the hash of the empty string is returned.\n # This seems too easy but maybe that is good enough? TO REVIEW!\n if not hashes:\n return sha3_256(b'').hexdigest()\n # XXX END TEMPORARY -- MUST REVIEW ...\n if len(hashes) == 1:\n return hexlify(hashes[0]).decode()\n if len(hashes) % 2 == 1:\n hashes.append(hashes[-1])\n parent_hashes = [\n sha3_256(hashes[i] + hashes[i+1]).digest()\n for i in range(0, len(hashes)-1, 2)\n ]\n return merkleroot(parent_hashes)"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef public_key64_to_address(base64_public_key):\n ed25519_public_key = public_key_from_base64(base64_public_key)\n encoded_public_key = amino_encoded_public_key(ed25519_public_key)\n return hashlib.new('ripemd160', encoded_public_key).hexdigest().upper()", "response": "Convert base64 public key to a public key64 address."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nvalidating the transaction spend", "response": "def validate(self, bigchain, current_transactions=[]):\n \"\"\"Validate transaction spend\n Args:\n bigchain (BigchainDB): an instantiated bigchaindb.BigchainDB object.\n Returns:\n The transaction (Transaction) if the transaction is valid else it\n raises an exception describing the reason why the transaction is\n invalid.\n Raises:\n ValidationError: If the transaction is invalid\n \"\"\"\n input_conditions = []\n\n if self.operation == Transaction.CREATE:\n duplicates = any(txn for txn in current_transactions if txn.id == self.id)\n if bigchain.is_committed(self.id) or duplicates:\n raise DuplicateTransaction('transaction `{}` already exists'\n .format(self.id))\n\n if not self.inputs_valid(input_conditions):\n raise InvalidSignature('Transaction signature is invalid.')\n\n elif self.operation == Transaction.TRANSFER:\n self.validate_transfer_inputs(bigchain, current_transactions)\n\n return self"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _connect(self):\n\n try:\n # FYI: the connection process might raise a\n # `ServerSelectionTimeoutError`, that is a subclass of\n # `ConnectionFailure`.\n # The presence of ca_cert, certfile, keyfile, crlfile implies the\n # use of certificates for TLS connectivity.\n if self.ca_cert is None or self.certfile is None or \\\n self.keyfile is None or self.crlfile is None:\n client = pymongo.MongoClient(self.host,\n self.port,\n replicaset=self.replicaset,\n serverselectiontimeoutms=self.connection_timeout,\n ssl=self.ssl,\n **MONGO_OPTS)\n if self.login is not None and self.password is not None:\n client[self.dbname].authenticate(self.login, self.password)\n else:\n logger.info('Connecting to MongoDB over TLS/SSL...')\n client = pymongo.MongoClient(self.host,\n self.port,\n replicaset=self.replicaset,\n serverselectiontimeoutms=self.connection_timeout,\n ssl=self.ssl,\n ssl_ca_certs=self.ca_cert,\n ssl_certfile=self.certfile,\n ssl_keyfile=self.keyfile,\n ssl_pem_passphrase=self.keyfile_passphrase,\n ssl_crlfile=self.crlfile,\n ssl_cert_reqs=CERT_REQUIRED,\n **MONGO_OPTS)\n if self.login is not None:\n client[self.dbname].authenticate(self.login,\n mechanism='MONGODB-X509')\n\n return client\n\n except (pymongo.errors.ConnectionFailure,\n pymongo.errors.OperationFailure) as exc:\n logger.info('Exception in _connect(): {}'.format(exc))\n raise ConnectionError(str(exc)) from exc\n except pymongo.errors.ConfigurationError as exc:\n raise ConfigurationError from exc", "response": "Connect to the database."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nadd the routes to an app", "response": "def add_routes(app):\n \"\"\"Add the routes to an app\"\"\"\n for (prefix, routes) in API_SECTIONS:\n api = Api(app, prefix=prefix)\n for ((pattern, resource, *args), kwargs) in routes:\n kwargs.setdefault('strict_slashes', False)\n api.add_resource(resource, pattern, *args, **kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef self(self) -> 'EFBChat':\n self.chat_name = \"You\"\n self.chat_alias = None\n self.chat_uid = EFBChat.SELF_ID\n self.chat_type = ChatType.User\n return self", "response": "Sets the chat as yourself."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef system(self) -> 'EFBChat':\n self.chat_name = \"System\"\n self.chat_alias = None\n self.chat_uid = EFBChat.SYSTEM_ID\n self.chat_type = ChatType.System\n return self", "response": "Set the chat as a system chat."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef long_name(self) -> str:\n if self.chat_alias:\n return \"{0} ({1})\".format(self.chat_alias, self.chat_name)\n else:\n return self.chat_name", "response": "Short property to provide the long name of the user."} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nverify the completeness of the data.", "response": "def verify(self):\n \"\"\"\n Verify the completeness of the data.\n\n Raises:\n ValueError: When this chat is invalid.\n \"\"\"\n if any(not i for i in (self.chat_uid, self.module_id)):\n raise ValueError(\"Chat data is incomplete.\")\n if not isinstance(self.chat_type, ChatType):\n raise ValueError(\"Invalid chat type.\")\n if self.chat_type == ChatType.Group:\n if any(not isinstance(i, EFBChat) or not i.chat_type == ChatType.User for i in self.members):\n raise ValueError(\"The group has an invalid member.\")\n if self.group is not None and (not isinstance(self.group, EFBChat) or\n not self.group.chat_type == ChatType.Group):\n raise ValueError(\"The member is in an invalid group.\")"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_extra_functions(self) -> Dict[str, Callable]:\n methods = {}\n for mName in dir(self):\n m = getattr(self, mName)\n if callable(m) and getattr(m, \"extra_fn\", False):\n methods[mName] = m\n return methods", "response": "Get a list of additional features."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef prerequisite_check():\n\n # Check Python version\n if sys.version_info < (3, 6):\n version_str = \"%s.%s.%s\" % sys.version_info[:3]\n # TRANSLATORS: This word is used as a part of search query suggested to users,\n # it may appears in context like \"Ubuntu 16.04 install Python 3.7\"\n search_url = build_search_query(_(\"install\") + \" Python 3.7\")\n return _(\"EH Forwarder Bot requires a minimum of Python 3.6 to run. You \"\n \"are currently using Python {version}. \\n\"\n \"\\n\"\n \"You may want to try:\\n\"\n \"{url}\").format(version=version_str, url=search_url)\n\n # Check installations of modules\n modules_err = _(\"You may want to visit the modules repository to find a list of \"\n \"available modules to install.\\n\"\n \"https://github.com/blueset/ehForwarderBot/wiki/Channels-Repository\")\n # 1. At least 1 master channel must be installed\n try:\n next(pkg_resources.iter_entry_points(\"ehforwarderbot.master\"))\n except StopIteration:\n return _(\"No master channel detected. EH Forwarder Bot requires at least one \"\n \"master channel installed to run.\") + \"\\n\\n\" + modules_err\n\n # 2. At least 1 slave channel must be installed\n try:\n next(pkg_resources.iter_entry_points(\"ehforwarderbot.slave\"))\n except StopIteration:\n return _(\"No slave channel detected. EH Forwarder Bot requires at least one \"\n \"slave channel installed to run.\") + \"\\n\\n\" + modules_err", "response": "Check prerequisites of the framework including Python version installation of modules and etc."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ngets a list of additional features.", "response": "def get_extra_functions(self) -> Dict[str, Callable]:\n \"\"\"Get a list of additional features\n\n Returns:\n Dict[str, Callable]: A dict of methods marked as additional features.\n Method can be called with ``get_extra_functions()[\"methodName\"]()``.\n \"\"\"\n if self.channel_type == ChannelType.Master:\n raise NameError(\"get_extra_function is not available on master channels.\")\n methods = {}\n for mName in dir(self):\n m = getattr(self, mName)\n if callable(m) and getattr(m, \"extra_fn\", False):\n methods[mName] = m\n return methods"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef verify(self):\n if self.author is None or not isinstance(self.author, EFBChat):\n raise ValueError(\"Author is not valid.\")\n else:\n self.author.verify()\n if self.chat is None or not isinstance(self.chat, EFBChat):\n raise ValueError(\"Chat is not valid.\")\n elif self.chat is not self.author: # Prevent repetitive verification\n self.chat.verify()\n if self.type is None or not isinstance(self.type, MsgType):\n raise ValueError(\"Type is not valid.\")\n if self.deliver_to is None or not isinstance(self.deliver_to, EFBChannel):\n raise ValueError(\"Deliver_to is not valid.\")\n if self.type in (MsgType.Audio, MsgType.File, MsgType.Image, MsgType.Sticker, MsgType.Video) and \\\n ((not self.edit) or (self.edit and self.edit_media)):\n if self.file is None or not hasattr(self.file, \"read\") or not hasattr(self.file, \"close\"):\n raise ValueError(\"File is not valid.\")\n if self.mime is None or not self.mime or not isinstance(self.mime, str):\n raise ValueError(\"MIME is not valid.\")\n if self.path is None or not self.path or not isinstance(self.path, str):\n raise ValueError(\"Path is not valid.\")\n if self.type == MsgType.Location and (self.attributes is None\n or not isinstance(self.attributes, EFBMsgLocationAttribute)):\n raise ValueError(\"Attribute of location message is invalid.\")\n if self.type == MsgType.Link and (self.attributes is None\n or not isinstance(self.attributes, EFBMsgLinkAttribute)):\n raise ValueError(\"Attribute of link message is invalid.\")\n if self.type == MsgType.Status and (self.attributes is None\n or not isinstance(self.attributes, EFBMsgStatusAttribute)):\n raise ValueError(\"Attribute of status message is invalid.\")\n\n if self.attributes:\n self.attributes.verify()\n\n if self.commands:\n self.commands.verify()\n\n if self.substitutions:\n self.substitutions.verify()", "response": "Verify the validity of the message."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef bump_version(v: version.Version, level: str) -> str:\n release: List[int] = list(v.release)\n stage: Optional[str]\n pre: Optional[int]\n stage, pre = v.pre if v.pre else (None, None)\n dev: Optional[int] = v.dev\n post: Optional[int] = v.post\n\n if level in (\"major\", \"minor\", \"patch\"):\n segments = 0\n if level == \"major\":\n # if the version code is in format of x.0.0, and pre/dev is not empty\n # do not increase the version number\n segments = 1\n elif level == \"minor\":\n # if the version code is in format of x.x.0, and pre/dev is not empty\n # do not increase the version number\n segments = 2\n elif level == \"patch\":\n # if the version code is in format of x.x.x, and pre/dev is not empty\n # do not increase the version number\n segments = 3\n if not any(release[segments:]) and (stage is not None or dev is not None):\n pass\n else:\n release[segments - 1] += 1\n release[segments:] = [0] * max(len(release) - segments, 0)\n stage = pre = post = dev = None\n elif level == \"alpha\":\n if stage is None:\n if dev is None:\n release[-1] += 1\n stage, pre = \"a\", 1\n elif stage > \"a\":\n release[-1] += 1\n stage, pre = \"a\", 1\n elif stage == \"a\":\n pre += 1\n post = dev = None\n elif level == \"beta\":\n if stage is None:\n if dev is None:\n release[-1] += 1\n stage, pre = \"b\", 1\n elif stage > \"b\":\n release[-1] += 1\n stage, pre = \"b\", 1\n elif stage == \"b\":\n pre += 1\n elif stage < \"b\":\n pre = 1\n stage = \"b\"\n post = dev = None\n elif level == \"post\":\n if post is not None:\n post += 1\n else:\n post = 1\n dev = None\n elif level == \"dev\":\n if dev is not None:\n dev += 1\n else:\n if stage:\n pre += 1\n else:\n release[-1] += 1\n dev = 1\n\n ver = \".\".join(str(i) for i in release)\n if stage is not None:\n ver += f\"{stage}{pre}\"\n if post is not None:\n ver += f\".post{post}\"\n if dev is not None:\n ver += f\".dev{dev}\"\n\n return ver", "response": "Bump the version number of the current version."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef extra(name: str, desc: str) -> Callable:\n\n def attr_dec(f):\n f.__setattr__(\"extra_fn\", True)\n f.__setattr__(\"name\", name)\n f.__setattr__(\"desc\", desc)\n return f\n\n return attr_dec", "response": "Decorator for slave channel s additional features interface."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn the base data path for the current EFB.", "response": "def get_base_path() -> Path:\n \"\"\"\n Get the base data path for EFB. This can be defined by the\n environment variable ``EFB_DATA_PATH``.\n \n If ``EFB_DATA_PATH`` is not defined, this gives\n ``~/.ehforwarderbot``.\n \n This method creates the queried path if not existing.\n \n Returns:\n The base path.\n \"\"\"\n env_data_path = os.environ.get(\"EFB_DATA_PATH\", None)\n if env_data_path:\n base_path = Path(env_data_path).resolve()\n else:\n base_path = Path.home() / \".ehforwarderbot\"\n if not base_path.exists():\n base_path.mkdir(parents=True)\n return base_path"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning the path for persistent storage of a module.", "response": "def get_data_path(module_id: str) -> Path:\n \"\"\"\n Get the path for persistent storage of a module.\n \n This method creates the queried path if not existing.\n \n Args:\n module_id (str): Module ID\n\n Returns:\n The data path of indicated module.\n \"\"\"\n profile = coordinator.profile\n data_path = get_base_path() / 'profiles' / profile / module_id\n if not data_path.exists():\n data_path.mkdir(parents=True)\n return data_path"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_config_path(module_id: str = None, ext: str = 'yaml') -> Path:\n if module_id:\n config_path = get_data_path(module_id)\n else:\n profile = coordinator.profile\n config_path = get_base_path() / 'profiles' / profile\n if not config_path.exists():\n config_path.mkdir(parents=True)\n return config_path / \"config.{}\".format(ext)", "response": "Get path to the config file. Defaulted to ~/.ehforwarderbot / profiles / profile_name / channel_id / config. yaml."} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ngets the path to custom modules", "response": "def get_custom_modules_path() -> Path:\n \"\"\"\n Get the path to custom channels\n\n Returns:\n The path for custom channels.\n \"\"\"\n channel_path = get_base_path() / \"modules\"\n if not channel_path.exists():\n channel_path.mkdir(parents=True)\n return channel_path"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nlocate a module by module ID.", "response": "def locate_module(module_id: str, module_type: str = None):\n \"\"\"\n Locate module by module ID\n\n Args:\n module_id: Module ID\n module_type: Type of module, one of ``'master'``, ``'slave'`` and ``'middleware'``\n \"\"\"\n\n entry_point = None\n\n if module_type:\n entry_point = 'ehforwarderbot.%s' % module_type\n\n module_id = module_id.split('#', 1)[0]\n\n if entry_point:\n for i in pkg_resources.iter_entry_points(entry_point):\n if i.name == module_id:\n return i.load()\n\n return pydoc.locate(module_id)"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nregistering the channel with the coordinator.", "response": "def add_channel(channel: EFBChannel):\n \"\"\"\n Register the channel with the coordinator.\n\n Args:\n channel (EFBChannel): Channel to register\n \"\"\"\n global master, slaves\n if isinstance(channel, EFBChannel):\n if channel.channel_type == ChannelType.Slave:\n slaves[channel.channel_id] = channel\n else:\n master = channel\n else:\n raise TypeError(\"Channel instance is expected\")"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef add_middleware(middleware: EFBMiddleware):\n global middlewares\n if isinstance(middleware, EFBMiddleware):\n middlewares.append(middleware)\n else:\n raise TypeError(\"Middleware instance is expected\")", "response": "Register a middleware to the coordinator."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef send_message(msg: 'EFBMsg') -> Optional['EFBMsg']:\n global middlewares, master, slaves\n\n if msg is None:\n return\n\n # Go through middlewares\n for i in middlewares:\n m = i.process_message(msg)\n if m is None:\n return None\n # for mypy type check\n assert m is not None\n msg = m\n\n msg.verify()\n\n if msg.deliver_to.channel_id == master.channel_id:\n return master.send_message(msg)\n elif msg.deliver_to.channel_id in slaves:\n return slaves[msg.deliver_to.channel_id].send_message(msg)\n else:\n raise EFBChannelNotFound(msg)", "response": "Sends a message to the destination channel."} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nsends a status message to the destination channel.", "response": "def send_status(status: 'EFBStatus'):\n \"\"\"\n Deliver a message to the destination channel.\n\n Args:\n status (EFBStatus): The status\n \"\"\"\n global middlewares, master\n if status is None:\n return\n\n s: 'Optional[EFBStatus]' = status\n\n # Go through middlewares\n for i in middlewares:\n s = i.process_status(cast('EFBStatus', s))\n if s is None:\n return\n\n status = cast('EFBStatus', s)\n\n status.verify()\n\n status.destination_channel.send_status(status)"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef get_module_by_id(module_id: str) -> Union[EFBChannel, EFBMiddleware]:\n try:\n if master.channel_id == module_id:\n return master\n except NameError:\n pass\n if module_id in slaves:\n return slaves[module_id]\n for i in middlewares:\n if i.middleware_id == module_id:\n return i\n raise NameError(\"Module ID {} is not found\".format(module_id))", "response": "Returns the module instance of a provided module ID."} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ninitialize all channels and modules.", "response": "def init(conf):\n \"\"\"\n Initialize all channels.\n \"\"\"\n\n logger = logging.getLogger(__name__)\n\n # Initialize mimetypes library\n mimetypes.init([pkg_resources.resource_filename('ehforwarderbot', 'mimetypes')])\n\n # Initialize all channels\n # (Load libraries and modules and init them)\n\n for i in conf['slave_channels']:\n logger.log(99, \"\\x1b[0;37;46m %s \\x1b[0m\", _(\"Initializing slave {}...\").format(i))\n\n cls = utils.locate_module(i, 'slave')\n telemetry_set_metadata({i: cls.__version__})\n instance_id = i.split('#', 1)[1:]\n instance_id = (instance_id and instance_id[0]) or None\n coordinator.add_channel(cls(instance_id=instance_id))\n\n logger.log(99, \"\\x1b[0;37;42m %s \\x1b[0m\",\n _(\"Slave channel {name} ({id}) # {instance_id} is initialized.\")\n .format(name=cls.channel_name, id=cls.channel_id,\n instance_id=instance_id or _(\"Default profile\")))\n\n logger.log(99, \"\\x1b[0;37;46m %s \\x1b[0m\",\n _(\"Initializing master {}...\").format(conf['master_channel']))\n instance_id = conf['master_channel'].split('#', 1)[1:]\n instance_id = (instance_id and instance_id[0]) or None\n module = utils.locate_module(conf['master_channel'], 'master')\n coordinator.add_channel(module(instance_id=instance_id))\n telemetry_set_metadata({conf['master_channel']: module.__version__})\n logger.log(99, \"\\x1b[0;37;42m %s \\x1b[0m\",\n _(\"Master channel {name} ({id}) # {instance_id} is initialized.\")\n .format(name=coordinator.master.channel_name,\n id=coordinator.master.channel_id,\n instance_id=instance_id or _(\"Default profile\")))\n\n logger.log(99, \"\\x1b[1;37;42m %s \\x1b[0m\", _(\"All channels initialized.\"))\n for i in conf['middlewares']:\n logger.log(99, \"\\x1b[0;37;46m %s \\x1b[0m\", _(\"Initializing middleware {}...\").format(i))\n cls = utils.locate_module(i, 'middleware')\n telemetry_set_metadata({i: cls.__version__})\n\n instance_id = i.split('#', 1)[1:]\n instance_id = (instance_id and instance_id[0]) or None\n coordinator.add_middleware(cls(instance_id=instance_id))\n logger.log(99, \"\\x1b[0;37;42m %s \\x1b[0m\",\n _(\"Middleware {name} ({id}) # {instance_id} is initialized.\")\n .format(name=cls.middleware_name, id=cls.middleware_id,\n instance_id=instance_id or _(\"Default profile\")))\n\n logger.log(99, \"\\x1b[1;37;42m %s \\x1b[0m\", _(\"All middlewares are initialized.\"))\n\n coordinator.master_thread = threading.Thread(target=coordinator.master.poll)\n coordinator.slave_threads = {key: threading.Thread(target=coordinator.slaves[key].poll)\n for key in coordinator.slaves}"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nstarts threads for polling", "response": "def poll():\n \"\"\"\n Start threads for polling\n \"\"\"\n coordinator.master_thread.start()\n for i in coordinator.slave_threads:\n coordinator.slave_threads[i].start()"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef setup_telemetry(key: str):\n\n if not isinstance(key, str):\n return\n\n if key not in (CAPTURE_EXCEPTIONS, CAPTURE_LOG, CAPTURE_LOG_ANALYSIS):\n return\n\n telemetry_config = {}\n if key in (CAPTURE_LOG, CAPTURE_LOG_ANALYSIS):\n telemetry_config.update({\"sentry\": {\"enable\": True, \"capture_logs\": True}})\n if key == CAPTURE_LOG_ANALYSIS:\n telemetry_config.update({\n \"logz\": {\"enable\": True},\n \"loggly\": {\"enable\": True},\n \"logdna\": {\"enable\": True}\n })\n\n global telemetry\n\n import telemetry_1a23\n telemetry_1a23.init('ehforwarderbot', telemetry_config)\n telemetry_1a23.set_metadata({\"ehforwarderbot\": __version__})\n\n telemetry = telemetry_1a23", "response": "Setup telemetry for the given key."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_connection(**kwargs):\n try:\n logging.debug(\"Connecting to mapd db...\")\n con = pymapd.connect(\n user=kwargs[\"db_user\"],\n password=kwargs[\"db_passwd\"],\n host=kwargs[\"db_server\"],\n port=kwargs[\"db_port\"],\n dbname=kwargs[\"db_name\"],\n )\n logging.info(\"Succesfully connected to mapd db\")\n return con\n except (pymapd.exceptions.OperationalError, pymapd.exceptions.Error):\n logging.exception(\"Error connecting to database.\")\n return False", "response": "Returns a connection to the mapd database."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef execute_query(**kwargs):\n start_time = timeit.default_timer()\n try:\n # Run the query\n query_result = con.execute(kwargs[\"query_mapdql\"])\n logging.debug(\n \"Completed iteration \"\n + str(kwargs[\"iteration\"])\n + \" of query \"\n + kwargs[\"query_name\"]\n )\n except (pymapd.exceptions.ProgrammingError, pymapd.exceptions.Error):\n logging.exception(\n \"Error running query \"\n + kwargs[\"query_name\"]\n + \" during iteration \"\n + str(kwargs[\"iteration\"])\n )\n return False\n\n # Calculate times\n query_elapsed_time = (timeit.default_timer() - start_time) * 1000\n execution_time = query_result._result.execution_time_ms\n connect_time = round((query_elapsed_time - execution_time), 1)\n\n # Iterate through each result from the query\n logging.debug(\n \"Counting results from query\"\n + kwargs[\"query_name\"]\n + \" iteration \"\n + str(kwargs[\"iteration\"])\n )\n result_count = 0\n start_time = timeit.default_timer()\n while query_result.fetchone():\n result_count += 1\n results_iter_time = round(\n ((timeit.default_timer() - start_time) * 1000), 1\n )\n\n query_execution = {\n \"result_count\": result_count,\n \"execution_time\": execution_time,\n \"connect_time\": connect_time,\n \"results_iter_time\": results_iter_time,\n \"total_time\": execution_time + connect_time + results_iter_time,\n }\n logging.debug(\n \"Execution results for query\"\n + kwargs[\"query_name\"]\n + \" iteration \"\n + str(kwargs[\"iteration\"])\n + \": \"\n + str(query_execution)\n )\n return query_execution", "response": "Executes a query against the connected db and returns a dictionary containing the result of the query."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef calculate_query_times(**kwargs):\n return {\n \"total_time_avg\": round(numpy.mean(kwargs[\"total_times\"]), 1),\n \"total_time_min\": round(numpy.min(kwargs[\"total_times\"]), 1),\n \"total_time_max\": round(numpy.max(kwargs[\"total_times\"]), 1),\n \"total_time_85\": round(numpy.percentile(kwargs[\"total_times\"], 85), 1),\n \"execution_time_avg\": round(numpy.mean(kwargs[\"execution_times\"]), 1),\n \"execution_time_min\": round(numpy.min(kwargs[\"execution_times\"]), 1),\n \"execution_time_max\": round(numpy.max(kwargs[\"execution_times\"]), 1),\n \"execution_time_85\": round(\n numpy.percentile(kwargs[\"execution_times\"], 85), 1\n ),\n \"execution_time_25\": round(\n numpy.percentile(kwargs[\"execution_times\"], 25), 1\n ),\n \"execution_time_std\": round(numpy.std(kwargs[\"execution_times\"]), 1),\n \"connect_time_avg\": round(numpy.mean(kwargs[\"connect_times\"]), 1),\n \"connect_time_min\": round(numpy.min(kwargs[\"connect_times\"]), 1),\n \"connect_time_max\": round(numpy.max(kwargs[\"connect_times\"]), 1),\n \"connect_time_85\": round(\n numpy.percentile(kwargs[\"connect_times\"], 85), 1\n ),\n \"results_iter_time_avg\": round(\n numpy.mean(kwargs[\"results_iter_times\"]), 1\n ),\n \"results_iter_time_min\": round(\n numpy.min(kwargs[\"results_iter_times\"]), 1\n ),\n \"results_iter_time_max\": round(\n numpy.max(kwargs[\"results_iter_times\"]), 1\n ),\n \"results_iter_time_85\": round(\n numpy.percentile(kwargs[\"results_iter_times\"], 85), 1\n ),\n }", "response": "Calculates aggregate query times from all iteration times"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a list of memory usage metrics for a given memory level", "response": "def get_mem_usage(**kwargs):\n \"\"\"\n Calculates memory statistics from mapd_server _client.get_memory call\n\n Kwargs:\n con(class 'pymapd.connection.Connection'): Mapd connection\n mem_type(str): [gpu, cpu] Type of memory to gather metrics for\n\n Returns:\n ramusage(dict):::\n usedram(float): Amount of memory (in MB) used\n freeram(float): Amount of memory (in MB) free\n totalallocated(float): Total amount of memory (in MB) allocated\n errormessage(str): Error if returned by get_memory call\n rawdata(list): Raw data returned from get_memory call\n \"\"\"\n try:\n con_mem_data_list = con._client.get_memory(\n session=kwargs[\"con\"]._session, memory_level=kwargs[\"mem_type\"]\n )\n usedram = 0\n freeram = 0\n for con_mem_data in con_mem_data_list:\n page_size = con_mem_data.page_size\n node_memory_data_list = con_mem_data.node_memory_data\n for node_memory_data in node_memory_data_list:\n ram = node_memory_data.num_pages * page_size\n is_free = node_memory_data.is_free\n if is_free:\n freeram += ram\n else:\n usedram += ram\n totalallocated = usedram + freeram\n if totalallocated > 0:\n totalallocated = round(totalallocated / 1024 / 1024, 1)\n usedram = round(usedram / 1024 / 1024, 1)\n freeram = round(freeram / 1024 / 1024, 1)\n ramusage = {}\n ramusage[\"usedram\"] = usedram\n ramusage[\"freeram\"] = freeram\n ramusage[\"totalallocated\"] = totalallocated\n ramusage[\"errormessage\"] = \"\"\n except Exception as e:\n errormessage = \"Get memory failed with error: \" + str(e)\n logging.error(errormessage)\n ramusage[\"errormessage\"] = errormessage\n return ramusage"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nadjusting the directory until a compilation database is found.", "response": "def find_compilation_database(path):\n \"\"\"Adjusts the directory until a compilation database is found.\"\"\"\n result = './'\n while not os.path.isfile(os.path.join(result, path)):\n if os.path.realpath(result) == '/':\n print('Error: could not find compilation database.')\n sys.exit(1)\n result += '../'\n return os.path.realpath(result)"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning a command line for clang - tidy.", "response": "def get_tidy_invocation(f, clang_tidy_binary, checks, tmpdir, build_path,\n header_filter, extra_arg, extra_arg_before, quiet,\n config):\n \"\"\"Gets a command line for clang-tidy.\"\"\"\n start = [clang_tidy_binary]\n if header_filter is not None:\n start.append('-header-filter=' + header_filter)\n else:\n # Show warnings in all in-project headers by default.\n start.append('-header-filter=^' + build_path + '/.*')\n if checks:\n start.append('-checks=' + checks)\n if tmpdir is not None:\n start.append('-export-fixes')\n # Get a temporary file. We immediately close the handle so clang-tidy can\n # overwrite it.\n (handle, name) = tempfile.mkstemp(suffix='.yaml', dir=tmpdir)\n os.close(handle)\n start.append(name)\n for arg in extra_arg:\n start.append('-extra-arg=%s' % arg)\n for arg in extra_arg_before:\n start.append('-extra-arg-before=%s' % arg)\n start.append('-p=' + build_path)\n if quiet:\n start.append('-quiet')\n if config:\n start.append('-config=' + config)\n start.append(f)\n return start"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef merge_replacement_files(tmpdir, mergefile):\n # The fixes suggested by clang-tidy >= 4.0.0 are given under\n # the top level key 'Diagnostics' in the output yaml files\n mergekey=\"Diagnostics\"\n merged=[]\n for replacefile in glob.iglob(os.path.join(tmpdir, '*.yaml')):\n content = yaml.safe_load(open(replacefile, 'r'))\n if not content:\n continue # Skip empty files.\n merged.extend(content.get(mergekey, []))\n\n if merged:\n # MainSourceFile: The key is required by the definition inside\n # include/clang/Tooling/ReplacementsYaml.h, but the value\n # is actually never used inside clang-apply-replacements,\n # so we set it to '' here.\n output = { 'MainSourceFile': '', mergekey: merged }\n with open(mergefile, 'w') as out:\n yaml.safe_dump(output, out)\n else:\n # Empty the file:\n open(mergefile, 'w').close()", "response": "Merge all replacement files in a directory into a single file"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef check_clang_apply_replacements_binary(args):\n try:\n subprocess.check_call([args.clang_apply_replacements_binary, '--version'])\n except:\n print('Unable to run clang-apply-replacements. Is clang-apply-replacements '\n 'binary correctly specified?', file=sys.stderr)\n traceback.print_exc()\n sys.exit(1)", "response": "Checks if invoking supplied clang - apply - replacements binary works."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ncall clang - apply - fixes on a given directory.", "response": "def apply_fixes(args, tmpdir):\n \"\"\"Calls clang-apply-fixes on a given directory.\"\"\"\n invocation = [args.clang_apply_replacements_binary]\n if args.format:\n invocation.append('-format')\n if args.style:\n invocation.append('-style=' + args.style)\n invocation.append(tmpdir)\n subprocess.call(invocation)"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef run_tidy(args, tmpdir, build_path, queue, lock, failed_files):\n while True:\n name = queue.get()\n invocation = get_tidy_invocation(name, args.clang_tidy_binary, args.checks,\n tmpdir, build_path, args.header_filter,\n args.extra_arg, args.extra_arg_before,\n args.quiet, args.config)\n\n proc = subprocess.Popen(invocation, stdout=subprocess.PIPE, stderr=subprocess.PIPE)\n output, err = proc.communicate()\n if proc.returncode != 0:\n failed_files.append(name)\n with lock:\n if len(output) > 0:\n sys.stdout.write(output.decode('utf-8') + '\\n')\n if len(err) > 0:\n sys.stderr.write(err.decode('utf-8') + '\\n')\n queue.task_done()", "response": "Takes filenames out of queue and runs clang - tidy on them."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef main():\n in_file = open(sys.argv[1], 'r')\n out_file = open(sys.argv[2], 'w')\n\n red_ast = RedBaron(in_file.read())\n\n thrift_specs = [ts.parent for ts in red_ast.find_all(\n 'name', 'thrift_spec') if ts.parent.type == 'assignment' and ts.parent.parent.name in ['TDatumVal', 'TColumnData']]\n\n nodes = []\n for ts in thrift_specs:\n node = ts.copy()\n node.target = ts.parent.name + '.' + str(node.target)\n nodes.append(node)\n ts.value = 'None'\n\n red_ast.extend(nodes)\n out_file.write(red_ast.dumps())", "response": "This function rewrites Thrift - generated Python clients to handle recursive structs."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nstoring and links the callback to the event.", "response": "def on(cls, *, event: str, callback: Callable):\n \"\"\"Stores and links the callback(s) to the event.\n\n Args:\n event (str): A string that specifies a Slack or websocket event.\n e.g. 'channel_joined' or 'open'\n callback (Callable): Any object or a list of objects that can be called.\n e.g. or\n [,]\n\n Raises:\n SlackClientError: The specified callback is not callable.\n SlackClientError: The callback must accept keyword arguments (**kwargs).\n \"\"\"\n if isinstance(callback, list):\n for cb in callback:\n cls._validate_callback(cb)\n previous_callbacks = cls._callbacks[event]\n cls._callbacks[event] = list(set(previous_callbacks + callback))\n else:\n cls._validate_callback(callback)\n cls._callbacks[event].append(callback)"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nstart an RTM Session with Slack.", "response": "def start(self) -> asyncio.Future:\n \"\"\"Starts an RTM Session with Slack.\n\n Makes an authenticated call to Slack's RTM API to retrieve\n a websocket URL and then connects to the message server.\n As events stream-in we run any associated callbacks stored\n on the client.\n\n If 'auto_reconnect' is specified we\n retrieve a new url and reconnect any time the connection\n is lost unintentionally or an exception is thrown.\n\n Raises:\n SlackApiError: Unable to retreive RTM URL from Slack.\n \"\"\"\n # TODO: Add Windows support for graceful shutdowns.\n if os.name != \"nt\":\n signals = (signal.SIGHUP, signal.SIGTERM, signal.SIGINT)\n for s in signals:\n self._event_loop.add_signal_handler(s, self.stop)\n\n future = asyncio.ensure_future(self._connect_and_read(), loop=self._event_loop)\n\n if self.run_async or self._event_loop.is_running():\n return future\n\n return self._event_loop.run_until_complete(future)"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef stop(self):\n self._logger.debug(\"The Slack RTMClient is shutting down.\")\n self._stopped = True\n self._close_websocket()", "response": "Closes the websocket connection."} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nsends a message over the WebSocket.", "response": "def send_over_websocket(self, *, payload: dict):\n \"\"\"Sends a message to Slack over the WebSocket connection.\n\n Note:\n The RTM API only supports posting simple messages formatted using\n our default message formatting mode. It does not support\n attachments or other message formatting modes. For this reason\n we recommend users send messages via the Web API methods.\n e.g. web_client.chat_postMessage()\n\n If the message \"id\" is not specified in the payload, it'll be added.\n\n Args:\n payload (dict): The message to send over the wesocket.\n e.g.\n {\n \"id\": 1,\n \"type\": \"typing\",\n \"channel\": \"C024BE91L\"\n }\n\n Raises:\n SlackClientNotConnectedError: Websocket connection is closed.\n \"\"\"\n if self._websocket is None or self._event_loop is None:\n raise client_err.SlackClientNotConnectedError(\n \"Websocket connection is closed.\"\n )\n if \"id\" not in payload:\n payload[\"id\"] = self._next_msg_id()\n asyncio.ensure_future(\n self._websocket.send_str(json.dumps(payload)), loop=self._event_loop\n )"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef typing(self, *, channel: str):\n payload = {\"id\": self._next_msg_id(), \"type\": \"typing\", \"channel\": channel}\n self.send_over_websocket(payload=payload)", "response": "Sends a typing indicator to the specified channel."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nchecking if the specified callback is callable and accepts a kwargs param.", "response": "def _validate_callback(callback):\n \"\"\"Checks if the specified callback is callable and accepts a kwargs param.\n\n Args:\n callback (obj): Any object or a list of objects that can be called.\n e.g. \n\n Raises:\n SlackClientError: The specified callback is not callable.\n SlackClientError: The callback must accept keyword arguments (**kwargs).\n \"\"\"\n\n cb_name = callback.__name__ if hasattr(callback, \"__name__\") else callback\n if not callable(callback):\n msg = \"The specified callback '{}' is not callable.\".format(cb_name)\n raise client_err.SlackClientError(msg)\n callback_params = inspect.signature(callback).parameters.values()\n if not any(\n param for param in callback_params if param.kind == param.VAR_KEYWORD\n ):\n msg = \"The callback '{}' must accept keyword arguments (**kwargs).\".format(\n cb_name\n )\n raise client_err.SlackClientError(msg)"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nestablishing a connection to Slack s RTM API and reads events as they come in.", "response": "async def _connect_and_read(self):\n \"\"\"Retreives and connects to Slack's RTM API.\n\n Makes an authenticated call to Slack's RTM API to retrieve\n a websocket URL. Then connects to the message server and\n reads event messages as they come in.\n\n If 'auto_reconnect' is specified we\n retrieve a new url and reconnect any time the connection\n is lost unintentionally or an exception is thrown.\n\n Raises:\n SlackApiError: Unable to retreive RTM URL from Slack.\n websockets.exceptions: Errors thrown by the 'websockets' library.\n \"\"\"\n while not self._stopped:\n try:\n self._connection_attempts += 1\n async with aiohttp.ClientSession(\n loop=self._event_loop,\n timeout=aiohttp.ClientTimeout(total=self.timeout),\n ) as session:\n self._session = session\n url, data = await self._retreive_websocket_info()\n async with session.ws_connect(\n url,\n heartbeat=self.ping_interval,\n ssl=self.ssl,\n proxy=self.proxy,\n ) as websocket:\n self._logger.debug(\"The Websocket connection has been opened.\")\n self._websocket = websocket\n self._dispatch_event(event=\"open\", data=data)\n await self._read_messages()\n except (\n client_err.SlackClientNotConnectedError,\n client_err.SlackApiError,\n # TODO: Catch websocket exceptions thrown by aiohttp.\n ) as exception:\n self._logger.debug(str(exception))\n self._dispatch_event(event=\"error\", data=exception)\n if self.auto_reconnect and not self._stopped:\n await self._wait_exponentially(exception)\n continue\n self._logger.exception(\n \"The Websocket encountered an error. Closing the connection...\"\n )\n self._close_websocket()\n raise"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\nasync def _read_messages(self):\n while not self._stopped and self._websocket is not None:\n async for message in self._websocket:\n if message.type == aiohttp.WSMsgType.TEXT:\n payload = message.json()\n event = payload.pop(\"type\", \"Unknown\")\n self._dispatch_event(event, data=payload)\n elif message.type == aiohttp.WSMsgType.ERROR:\n break", "response": "Process messages received on the WebSocket connection."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _dispatch_event(self, event, data=None):\n for callback in self._callbacks[event]:\n self._logger.debug(\n \"Running %s callbacks for event: '%s'\",\n len(self._callbacks[event]),\n event,\n )\n try:\n if self._stopped and event not in [\"close\", \"error\"]:\n # Don't run callbacks if client was stopped unless they're close/error callbacks.\n break\n\n if self.run_async:\n self._execute_callback_async(callback, data)\n else:\n self._execute_callback(callback, data)\n except Exception as err:\n name = callback.__name__\n module = callback.__module__\n msg = f\"When calling '#{name}()' in the '{module}' module the following error was raised: {err}\"\n self._logger.error(msg)\n raise", "response": "Dispatches the event and executes any associated callbacks."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nexecute the callback asynchronously.", "response": "def _execute_callback_async(self, callback, data):\n \"\"\"Execute the callback asynchronously.\n\n If the callback is not a coroutine, convert it.\n\n Note: The WebClient passed into the callback is running in \"async\" mode.\n This means all responses will be futures.\n \"\"\"\n if asyncio.iscoroutine(callback):\n asyncio.ensure_future(\n callback(rtm_client=self, web_client=self._web_client, data=data)\n )\n else:\n asyncio.ensure_future(\n asyncio.coroutine(callback)(\n rtm_client=self, web_client=self._web_client, data=data\n )\n )"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _execute_callback(self, callback, data):\n web_client = WebClient(\n token=self.token, base_url=self.base_url, ssl=self.ssl, proxy=self.proxy\n )\n with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:\n # Execute the callback on a separate thread,\n future = executor.submit(\n callback, rtm_client=self, web_client=web_client, data=data\n )\n\n while future.running():\n pass\n\n future.result()", "response": "Execute the callback in another thread. Wait for and return the results."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nwait exponentially longer for each connection attempt.", "response": "async def _wait_exponentially(self, exception, max_wait_time=300):\n \"\"\"Wait exponentially longer for each connection attempt.\n\n Calculate the number of seconds to wait and then add\n a random number of milliseconds to avoid coincendental\n synchronized client retries. Wait up to the maximium amount\n of wait time specified via 'max_wait_time'. However,\n if Slack returned how long to wait use that.\n \"\"\"\n wait_time = min(\n (2 ** self._connection_attempts) + random.random(), max_wait_time\n )\n try:\n wait_time = exception.response[\"headers\"][\"Retry-After\"]\n except (KeyError, AttributeError):\n pass\n self._logger.debug(\"Waiting %s seconds before reconnecting.\", wait_time)\n await asyncio.sleep(float(wait_time))"} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncloses the websocket connection.", "response": "def _close_websocket(self):\n \"\"\"Closes the websocket connection.\"\"\"\n close_method = getattr(self._websocket, \"close\", None)\n if callable(close_method):\n asyncio.ensure_future(close_method(), loop=self._event_loop)\n self._websocket = None\n self._dispatch_event(event=\"close\")"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nupdating the onboarding welcome message after recieving a reaction added event from Slack.", "response": "async def update_emoji(**payload):\n \"\"\"Update the onboarding welcome message after recieving a \"reaction_added\"\n event from Slack. Update timestamp for welcome message as well.\n \"\"\"\n data = payload[\"data\"]\n web_client = payload[\"web_client\"]\n channel_id = data[\"item\"][\"channel\"]\n user_id = data[\"user\"]\n\n # Get the original tutorial sent.\n onboarding_tutorial = onboarding_tutorials_sent[channel_id][user_id]\n\n # Mark the reaction task as completed.\n onboarding_tutorial.reaction_task_completed = True\n\n # Get the new message payload\n message = onboarding_tutorial.get_message_payload()\n\n # Post the updated message in Slack\n updated_message = await web_client.chat_update(**message)\n\n # Update the timestamp saved on the onboarding tutorial object\n onboarding_tutorial.timestamp = updated_message[\"ts\"]"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\nasync def update_pin(**payload):\n data = payload[\"data\"]\n web_client = payload[\"web_client\"]\n channel_id = data[\"channel_id\"]\n user_id = data[\"user\"]\n\n # Get the original tutorial sent.\n onboarding_tutorial = onboarding_tutorials_sent[channel_id][user_id]\n\n # Mark the pin task as completed.\n onboarding_tutorial.pin_task_completed = True\n\n # Get the new message payload\n message = onboarding_tutorial.get_message_payload()\n\n # Post the updated message in Slack\n updated_message = await web_client.chat_update(**message)\n\n # Update the timestamp saved on the onboarding tutorial object\n onboarding_tutorial.timestamp = updated_message[\"ts\"]", "response": "Update the onboarding welcome message after recieving a pin_added event from Slack. Update timestamp for welcome message as well."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ndisplay the onboarding welcome message after receiving a message that contains start.", "response": "async def message(**payload):\n \"\"\"Display the onboarding welcome message after receiving a message\n that contains \"start\".\n \"\"\"\n data = payload[\"data\"]\n web_client = payload[\"web_client\"]\n channel_id = data.get(\"channel\")\n user_id = data.get(\"user\")\n text = data.get(\"text\")\n\n if text and text.lower() == \"start\":\n return await start_onboarding(web_client, user_id, channel_id)"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef channels_rename(self, *, channel: str, name: str, **kwargs) -> SlackResponse:\n self._validate_xoxp_token()\n kwargs.update({\"channel\": channel, \"name\": name})\n return self.api_call(\"channels.rename\", json=kwargs)", "response": "Renames a channel.\n\n Args:\n channel (str): The channel id. e.g. 'C1234567890'\n name (str): The new channel name. e.g. 'newchannel'"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef channels_replies(self, *, channel: str, thread_ts: str, **kwargs) -> SlackResponse:\n kwargs.update({\"channel\": channel, \"thread_ts\": thread_ts})\n return self.api_call(\"channels.replies\", http_verb=\"GET\", params=kwargs)", "response": "Retrieve a thread of messages posted to a channel."} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ndeletes a message. Args: channel (str): Channel containing the message to be deleted. e.g. 'C1234567890' ts (str): Timestamp of the message to be deleted. e.g. '1234567890.123456'", "response": "def chat_delete(self, *, channel: str, ts: str, **kwargs) -> SlackResponse:\n \"\"\"Deletes a message.\n\n Args:\n channel (str): Channel containing the message to be deleted. e.g. 'C1234567890'\n ts (str): Timestamp of the message to be deleted. e.g. '1234567890.123456'\n \"\"\"\n kwargs.update({\"channel\": channel, \"ts\": ts})\n return self.api_call(\"chat.delete\", json=kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef chat_getPermalink(\n self, *, channel: str, message_ts: str, **kwargs\n ) -> SlackResponse:\n \"\"\"Retrieve a permalink URL for a specific extant message\n\n Args:\n channel (str): The channel id. e.g. 'C1234567890'\n message_ts (str): The timestamp. e.g. '1234567890.123456'\n \"\"\"\n kwargs.update({\"channel\": channel, \"message_ts\": message_ts})\n return self.api_call(\"chat.getPermalink\", http_verb=\"GET\", params=kwargs)", "response": "Returns a permalink URL for a specific extant message\n ."} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nshare a me message into a channel.", "response": "def chat_meMessage(self, *, channel: str, text: str, **kwargs) -> SlackResponse:\n \"\"\"Share a me message into a channel.\n\n Args:\n channel (str): The channel id. e.g. 'C1234567890'\n text (str): The message you'd like to share. e.g. 'Hello world'\n \"\"\"\n kwargs.update({\"channel\": channel, \"text\": text})\n return self.api_call(\"chat.meMessage\", json=kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nsending an ephemeral message to a user in a channel.", "response": "def chat_postEphemeral(self, *, channel: str, user: str, **kwargs) -> SlackResponse:\n \"\"\"Sends an ephemeral message to a user in a channel.\n\n Args:\n channel (str): The channel id. e.g. 'C1234567890'\n user (str): The id of user who should see the message. e.g. 'U0BPQUNTA'\n text (str): The message you'd like to share. e.g. 'Hello world'\n text is not required when presenting blocks.\n blocks (list): A dictionary list of blocks.\n Blocks are required when not presenting text.\n e.g. [{\"type\": \"section\", \"text\": {\"type\": \"plain_text\", \"text\": \"Hello world\"}}]\n \"\"\"\n kwargs.update({\"channel\": channel, \"user\": user})\n return self.api_call(\"chat.postEphemeral\", json=kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef chat_unfurl(\n self, *, channel: str, ts: str, unfurls: dict, **kwargs\n ) -> SlackResponse:\n \"\"\"Provide custom unfurl behavior for user-posted URLs.\n\n Args:\n channel (str): The Channel ID of the message. e.g. 'C1234567890'\n ts (str): Timestamp of the message to add unfurl behavior to. e.g. '1234567890.123456'\n unfurls (dict): a dict of the specific URLs you're offering an unfurl for.\n e.g. {\"https://example.com/\": {\"text\": \"Every day is the test.\"}}\n \"\"\"\n self._validate_xoxp_token()\n kwargs.update({\"channel\": channel, \"ts\": ts, \"unfurls\": unfurls})\n return self.api_call(\"chat.unfurl\", json=kwargs)", "response": "Use this method to add custom unfurl behavior for user - posted URLs."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef conversations_invite(\n self, *, channel: str, users: List[str], **kwargs\n ) -> SlackResponse:\n \"\"\"Invites users to a channel.\n\n Args:\n channel (str): The channel id. e.g. 'C1234567890'\n users (list): An list of user id's to invite. e.g. ['U2345678901', 'U3456789012']\n \"\"\"\n self._validate_xoxp_token()\n kwargs.update({\"channel\": channel, \"users\": users})\n return self.api_call(\"conversations.invite\", json=kwargs)", "response": "Invites users to a channel."} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nretrieving a thread of messages posted to a conversation.", "response": "def conversations_replies(self, *, channel: str, ts: str, **kwargs) -> SlackResponse:\n \"\"\"Retrieve a thread of messages posted to a conversation\n\n Args:\n channel (str): Conversation ID to fetch thread from. e.g. 'C1234567890'\n ts (str): Unique identifier of a thread's parent message. e.g. '1234567890.123456'\n \"\"\"\n kwargs.update({\"channel\": channel, \"ts\": ts})\n return self.api_call(\"conversations.replies\", http_verb=\"GET\", params=kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nset the topic for a conversation.", "response": "def conversations_setTopic(\n self, *, channel: str, topic: str, **kwargs\n ) -> SlackResponse:\n \"\"\"Sets the topic for a conversation.\n\n Args:\n channel (str): The channel id. e.g. 'C1234567890'\n topic (str): The new topic for the channel. e.g. 'My Topic'\n \"\"\"\n kwargs.update({\"channel\": channel, \"topic\": topic})\n return self.api_call(\"conversations.setTopic\", json=kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef dialog_open(self, *, dialog: dict, trigger_id: str, **kwargs) -> SlackResponse:\n kwargs.update({\"dialog\": dialog, \"trigger_id\": trigger_id})\n return self.api_call(\"dialog.open\", json=kwargs)", "response": "Open a dialog with a user."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef dnd_endDnd(self, **kwargs) -> SlackResponse:\n self._validate_xoxp_token()\n return self.api_call(\"dnd.endDnd\", json=kwargs)", "response": "Ends the current user s Do Not Disturb session immediately."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nending the current user s snooze mode immediately.", "response": "def dnd_endSnooze(self, **kwargs) -> SlackResponse:\n \"\"\"Ends the current user's snooze mode immediately.\"\"\"\n self._validate_xoxp_token()\n return self.api_call(\"dnd.endSnooze\", json=kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nturn on Do Not Disturb mode for the current user.", "response": "def dnd_setSnooze(self, *, num_minutes: int, **kwargs) -> SlackResponse:\n \"\"\"Turns on Do Not Disturb mode for the current user, or changes its duration.\n\n Args:\n num_minutes (int): The snooze duration. e.g. 60\n \"\"\"\n self._validate_xoxp_token()\n kwargs.update({\"num_minutes\": num_minutes})\n return self.api_call(\"dnd.setSnooze\", http_verb=\"GET\", params=kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef files_comments_add(self, *, comment: str, file: str, **kwargs) -> SlackResponse:\n kwargs.update({\"comment\": comment, \"file\": file})\n return self.api_call(\"files.comments.add\", json=kwargs)", "response": "Adds a comment to an existing file."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef files_comments_delete(self, *, file: str, id: str, **kwargs) -> SlackResponse:\n kwargs.update({\"file\": file, \"id\": id})\n return self.api_call(\"files.comments.delete\", json=kwargs)", "response": "Deletes an existing comment on a file."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nediting an existing file comment.", "response": "def files_comments_edit(\n self, *, comment: str, file: str, id: str, **kwargs\n ) -> SlackResponse:\n \"\"\"Edit an existing file comment.\n\n Args:\n comment (str): The body of the comment.\n e.g. 'Everyone should take a moment to read this file.'\n file (str): The file id. e.g. 'F1234467890'\n id (str): The file comment id. e.g. 'Fc1234567890'\n \"\"\"\n kwargs.update({\"comment\": comment, \"file\": file, \"id\": id})\n return self.api_call(\"files.comments.edit\", json=kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ndeleting a file. .", "response": "def files_delete(self, *, id: str, **kwargs) -> SlackResponse:\n \"\"\"Deletes a file.\n\n Args:\n id (str): The file id. e.g. 'F1234467890'\n \"\"\"\n kwargs.update({\"id\": id})\n return self.api_call(\"files.delete\", json=kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef files_info(self, *, id: str, **kwargs) -> SlackResponse:\n kwargs.update({\"id\": id})\n return self.api_call(\"files.info\", http_verb=\"GET\", params=kwargs)", "response": "Gets information about a team file."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef files_list(self, **kwargs) -> SlackResponse:\n self._validate_xoxp_token()\n return self.api_call(\"files.list\", http_verb=\"GET\", params=kwargs)", "response": "Lists & filters team files."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nenables a file for public sharing.", "response": "def files_sharedPublicURL(self, *, id: str, **kwargs) -> SlackResponse:\n \"\"\"Enables a file for public/external sharing.\n\n Args:\n id (str): The file id. e.g. 'F1234467890'\n \"\"\"\n self._validate_xoxp_token()\n kwargs.update({\"id\": id})\n return self.api_call(\"files.sharedPublicURL\", json=kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef files_upload(\n self, *, file: Union[str, IOBase] = None, content: str = None, **kwargs\n ) -> SlackResponse:\n \"\"\"Uploads or creates a file.\n\n Args:\n file (str): Supply a file path.\n when you'd like to upload a specific file. e.g. 'dramacat.gif'\n content (str): Supply content when you'd like to create an\n editable text file containing the specified text. e.g. 'launch plan'\n Raises:\n SlackRequestError: If niether or both the `file` and `content` args are specified.\n \"\"\"\n if file is None and content is None:\n raise e.SlackRequestError(\"The file or content argument must be specified.\")\n if file is not None and content is not None:\n raise e.SlackRequestError(\n \"You cannot specify both the file and the content argument.\"\n )\n\n if file:\n return self.api_call(\"files.upload\", files={\"file\": file}, data=kwargs)\n elif content:\n data = kwargs.copy()\n data.update({\"content\": content})\n return self.api_call(\"files.upload\", data=data)", "response": "Uploads or creates a file."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef groups_createChild(self, *, channel: str, **kwargs) -> SlackResponse:\n self._validate_xoxp_token()\n kwargs.update({\"channel\": channel})\n return self.api_call(\"groups.createChild\", http_verb=\"GET\", params=kwargs)", "response": "Creates a new private channel."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef groups_invite(self, *, channel: str, user: str, **kwargs) -> SlackResponse:\n self._validate_xoxp_token()\n kwargs.update({\"channel\": channel, \"user\": user})\n return self.api_call(\"groups.invite\", json=kwargs)", "response": "Invites a user to a private channel."} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nretrieve a thread of messages posted to a private channel.", "response": "def groups_replies(self, *, channel: str, thread_ts: str, **kwargs) -> SlackResponse:\n \"\"\"Retrieve a thread of messages posted to a private channel\n\n Args:\n channel (str): The channel id. e.g. 'C1234567890'\n thread_ts (str): The timestamp of an existing message with 0 or more replies.\n e.g. '1234567890.123456'\n \"\"\"\n self._validate_xoxp_token()\n kwargs.update({\"channel\": channel, \"thread_ts\": thread_ts})\n return self.api_call(\"groups.replies\", http_verb=\"GET\", params=kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nsets the purpose for a private channel.", "response": "def groups_setPurpose(self, *, channel: str, purpose: str, **kwargs) -> SlackResponse:\n \"\"\"Sets the purpose for a private channel.\n\n Args:\n channel (str): The channel id. e.g. 'G1234567890'\n purpose (str): The new purpose for the channel. e.g. 'My Purpose'\n \"\"\"\n kwargs.update({\"channel\": channel, \"purpose\": purpose})\n return self.api_call(\"groups.setPurpose\", json=kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef im_open(self, *, user: str, **kwargs) -> SlackResponse:\n kwargs.update({\"user\": user})\n return self.api_call(\"im.open\", json=kwargs)", "response": "Opens a direct message channel."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef mpim_close(self, *, channel: str, **kwargs) -> SlackResponse:\n kwargs.update({\"channel\": channel})\n return self.api_call(\"mpim.close\", json=kwargs)", "response": "Closes a multiparty direct message channel."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nfetch the history of messages and events from a multiparty direct message.", "response": "def mpim_history(self, *, channel: str, **kwargs) -> SlackResponse:\n \"\"\"Fetches history of messages and events from a multiparty direct message.\n\n Args:\n channel (str): Multiparty direct message to fetch history for. e.g. 'G1234567890'\n \"\"\"\n kwargs.update({\"channel\": channel})\n return self.api_call(\"mpim.history\", http_verb=\"GET\", params=kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef oauth_access(\n self, *, client_id: str, client_secret: str, code: str, **kwargs\n ) -> SlackResponse:\n \"\"\"Exchanges a temporary OAuth verifier code for an access token.\n\n Args:\n client_id (str): Issued when you created your application. e.g. '4b39e9-752c4'\n client_secret (str): Issued when you created your application. e.g. '33fea0113f5b1'\n code (str): The code param returned via the OAuth callback. e.g. 'ccdaa72ad'\n \"\"\"\n kwargs.update(\n {\"client_id\": client_id, \"client_secret\": client_secret, \"code\": code}\n )\n return self.api_call(\"oauth.access\", data=kwargs)", "response": "Exchanges a temporary OAuth verifier code for an access token."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nadd a reaction to an item.", "response": "def reactions_add(self, *, name: str, **kwargs) -> SlackResponse:\n \"\"\"Adds a reaction to an item.\n\n Args:\n name (str): Reaction (emoji) name. e.g. 'thumbsup'\n channel (str): Channel where the message to add reaction to was posted.\n e.g. 'C1234567890'\n timestamp (str): Timestamp of the message to add reaction to. e.g. '1234567890.123456'\n \"\"\"\n kwargs.update({\"name\": name})\n return self.api_call(\"reactions.add\", json=kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ncreating a reminder. Args: text (str): The content of the reminder. e.g. 'eat a banana' time (str): When this reminder should happen: the Unix timestamp (up to five years from now e.g. '1602288000'), the number of seconds until the reminder (if within 24 hours), or a natural language description (Ex. 'in 15 minutes' or 'every Thursday')", "response": "def reminders_add(self, *, text: str, time: str, **kwargs) -> SlackResponse:\n \"\"\"Creates a reminder.\n\n Args:\n text (str): The content of the reminder. e.g. 'eat a banana'\n time (str): When this reminder should happen:\n the Unix timestamp (up to five years from now e.g. '1602288000'),\n the number of seconds until the reminder (if within 24 hours),\n or a natural language description (Ex. 'in 15 minutes' or 'every Thursday')\n \"\"\"\n self._validate_xoxp_token()\n kwargs.update({\"text\": text, \"time\": time})\n return self.api_call(\"reminders.add\", json=kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef reminders_complete(self, *, reminder: str, **kwargs) -> SlackResponse:\n self._validate_xoxp_token()\n kwargs.update({\"reminder\": reminder})\n return self.api_call(\"reminders.complete\", json=kwargs)", "response": "Marks a reminder as complete."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef reminders_info(self, *, reminder: str, **kwargs) -> SlackResponse:\n self._validate_xoxp_token()\n kwargs.update({\"reminder\": reminder})\n return self.api_call(\"reminders.info\", http_verb=\"GET\", params=kwargs)", "response": "Gets information about a reminder."} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nlists all reminders created by or for a given user.", "response": "def reminders_list(self, **kwargs) -> SlackResponse:\n \"\"\"Lists all reminders created by or for a given user.\"\"\"\n self._validate_xoxp_token()\n return self.api_call(\"reminders.list\", http_verb=\"GET\", params=kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef search_messages(self, *, query: str, **kwargs) -> SlackResponse:\n self._validate_xoxp_token()\n kwargs.update({\"query\": query})\n return self.api_call(\"search.messages\", http_verb=\"GET\", params=kwargs)", "response": "Searches for messages matching a query."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef stars_list(self, **kwargs) -> SlackResponse:\n self._validate_xoxp_token()\n return self.api_call(\"stars.list\", http_verb=\"GET\", params=kwargs)", "response": "Lists stars for a user."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef team_accessLogs(self, **kwargs) -> SlackResponse:\n self._validate_xoxp_token()\n return self.api_call(\"team.accessLogs\", http_verb=\"GET\", params=kwargs)", "response": "Gets the access logs for the current team."} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ngets billable users information for the current team.", "response": "def team_billableInfo(self, **kwargs) -> SlackResponse:\n \"\"\"Gets billable users information for the current team.\"\"\"\n self._validate_xoxp_token()\n return self.api_call(\"team.billableInfo\", http_verb=\"GET\", params=kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ngetting the integration logs for the current team.", "response": "def team_integrationLogs(self, **kwargs) -> SlackResponse:\n \"\"\"Gets the integration logs for the current team.\"\"\"\n self._validate_xoxp_token()\n return self.api_call(\"team.integrationLogs\", http_verb=\"GET\", params=kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef team_profile_get(self, **kwargs) -> SlackResponse:\n self._validate_xoxp_token()\n return self.api_call(\"team.profile.get\", http_verb=\"GET\", params=kwargs)", "response": "Retrieve a team s profile."} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ncreates a User Group", "response": "def usergroups_create(self, *, name: str, **kwargs) -> SlackResponse:\n \"\"\"Create a User Group\n\n Args:\n name (str): A name for the User Group. Must be unique among User Groups.\n e.g. 'My Test Team'\n \"\"\"\n self._validate_xoxp_token()\n kwargs.update({\"name\": name})\n return self.api_call(\"usergroups.create\", json=kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef usergroups_disable(self, *, usergroup: str, **kwargs) -> SlackResponse:\n self._validate_xoxp_token()\n kwargs.update({\"usergroup\": usergroup})\n return self.api_call(\"usergroups.disable\", json=kwargs)", "response": "Disable an existing User Group"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef usergroups_list(self, **kwargs) -> SlackResponse:\n self._validate_xoxp_token()\n return self.api_call(\"usergroups.list\", http_verb=\"GET\", params=kwargs)", "response": "List all User Groups for a team"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nlisting all users in a User Group", "response": "def usergroups_users_list(self, *, usergroup: str, **kwargs) -> SlackResponse:\n \"\"\"List all users in a User Group\n\n Args:\n usergroup (str): The encoded ID of the User Group to update.\n e.g. 'S0604QSJC'\n \"\"\"\n self._validate_xoxp_token()\n kwargs.update({\"usergroup\": usergroup})\n return self.api_call(\"usergroups.users.list\", http_verb=\"GET\", params=kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nupdates the list of users for a User Group", "response": "def usergroups_users_update(\n self, *, usergroup: str, users: List[str], **kwargs\n ) -> SlackResponse:\n \"\"\"Update the list of users for a User Group\n\n Args:\n usergroup (str): The encoded ID of the User Group to update.\n e.g. 'S0604QSJC'\n users (list): A list user IDs that represent the entire list of\n users for the User Group. e.g. ['U060R4BJ4', 'U060RNRCZ']\n \"\"\"\n self._validate_xoxp_token()\n kwargs.update({\"usergroup\": usergroup, \"users\": users})\n return self.api_call(\"usergroups.users.update\", json=kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef users_deletePhoto(self, **kwargs) -> SlackResponse:\n self._validate_xoxp_token()\n return self.api_call(\"users.deletePhoto\", http_verb=\"GET\", params=kwargs)", "response": "Deletes the user profile photo"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ngetting user presence information.", "response": "def users_getPresence(self, *, user: str, **kwargs) -> SlackResponse:\n \"\"\"Gets user presence information.\n\n Args:\n user (str): User to get presence info on. Defaults to the authed user.\n e.g. 'W1234567890'\n \"\"\"\n kwargs.update({\"user\": user})\n return self.api_call(\"users.getPresence\", http_verb=\"GET\", params=kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef users_identity(self, **kwargs) -> SlackResponse:\n self._validate_xoxp_token()\n return self.api_call(\"users.identity\", http_verb=\"GET\", params=kwargs)", "response": "Get a user s identity."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef users_lookupByEmail(self, *, email: str, **kwargs) -> SlackResponse:\n kwargs.update({\"email\": email})\n return self.api_call(\"users.lookupByEmail\", http_verb=\"GET\", params=kwargs)", "response": "Searches for a user with an email address."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nsetting the user profile photo", "response": "def users_setPhoto(self, *, image: Union[str, IOBase], **kwargs) -> SlackResponse:\n \"\"\"Set the user profile photo\n\n Args:\n image (str): Supply the path of the image you'd like to upload.\n e.g. 'myimage.png'\n \"\"\"\n self._validate_xoxp_token()\n return self.api_call(\"users.setPhoto\", files={\"image\": image}, data=kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef users_setPresence(self, *, presence: str, **kwargs) -> SlackResponse:\n kwargs.update({\"presence\": presence})\n return self.api_call(\"users.setPresence\", json=kwargs)", "response": "Manually sets user presence."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef users_profile_get(self, **kwargs) -> SlackResponse:\n self._validate_xoxp_token()\n return self.api_call(\"users.profile.get\", http_verb=\"GET\", params=kwargs)", "response": "Retrieves a user s profile information."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef users_profile_set(self, **kwargs) -> SlackResponse:\n self._validate_xoxp_token()\n return self.api_call(\"users.profile.set\", json=kwargs)", "response": "Sets the profile information for a user."} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nexecutes the Slack API call to the specified resource.", "response": "def api_call(\n self,\n api_method: str,\n *,\n http_verb: str = \"POST\",\n files: dict = None,\n data: dict = None,\n params: dict = None,\n json: dict = None,\n ):\n \"\"\"Create a request and execute the API call to Slack.\n\n Args:\n api_method (str): The target Slack API method.\n e.g. 'chat.postMessage'\n http_verb (str): HTTP Verb. e.g. 'POST'\n files (dict): Files to multipart upload.\n e.g. {imageORfile: file_objectORfile_path}\n data: The body to attach to the request. If a dictionary is\n provided, form-encoding will take place.\n e.g. {'key1': 'value1', 'key2': 'value2'}\n params (dict): The URL parameters to append to the URL.\n e.g. {'key1': 'value1', 'key2': 'value2'}\n json (dict): JSON for the body to attach to the request\n (if files or data is not specified).\n e.g. {'key1': 'value1', 'key2': 'value2'}\n\n Returns:\n (SlackResponse)\n The server's response to an HTTP request. Data\n from the response can be accessed like a dict.\n If the response included 'next_cursor' it can\n be iterated on to execute subsequent requests.\n\n Raises:\n SlackApiError: The following Slack API call failed:\n 'chat.postMessage'.\n SlackRequestError: Json data can only be submitted as\n POST requests.\n \"\"\"\n if json is not None and http_verb != \"POST\":\n msg = \"Json data can only be submitted as POST requests. GET requests should use the 'params' argument.\"\n raise err.SlackRequestError(msg)\n\n api_url = self._get_url(api_method)\n headers = {\n \"User-Agent\": self._get_user_agent(),\n \"Authorization\": \"Bearer {}\".format(self.token),\n }\n if files is not None:\n form_data = aiohttp.FormData()\n for k, v in files.items():\n if isinstance(v, str):\n form_data.add_field(k, open(v, \"rb\"))\n else:\n form_data.add_field(k, v)\n\n if data is not None:\n for k, v in data.items():\n form_data.add_field(k, str(v))\n\n data = form_data\n\n req_args = {\n \"headers\": headers,\n \"data\": data,\n \"params\": params,\n \"json\": json,\n \"ssl\": self.ssl,\n \"proxy\": self.proxy,\n }\n\n if self._event_loop is None:\n self._set_event_loop()\n\n future = asyncio.ensure_future(\n self._send(http_verb=http_verb, api_url=api_url, req_args=req_args),\n loop=self._event_loop,\n )\n\n if self.run_async:\n return future\n\n return self._event_loop.run_until_complete(future)"} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nensures that an xoxp token is used when the specified API method is called.", "response": "def _validate_xoxp_token(self):\n \"\"\"Ensures that an xoxp token is used when the specified method is called.\n\n Raises:\n BotUserAccessError: If the API method is called with a Bot User OAuth Access Token.\n \"\"\"\n\n if self.token.startswith(\"xoxb\"):\n method_name = inspect.stack()[1][3]\n msg = \"The method '{}' cannot be called with a Bot Token.\".format(\n method_name\n )\n raise err.BotUserAccessError(msg)"} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nsends the request out for transmission.", "response": "async def _send(self, http_verb, api_url, req_args):\n \"\"\"Sends the request out for transmission.\n\n Args:\n http_verb (str): The HTTP verb. e.g. 'GET' or 'POST'.\n api_url (str): The Slack API url. e.g. 'https://slack.com/api/chat.postMessage'\n req_args (dict): The request arguments to be attached to the request.\n e.g.\n {\n json: {\n 'attachments': [{\"pretext\": \"pre-hello\", \"text\": \"text-world\"}],\n 'channel': '#random'\n }\n }\n Returns:\n The response parsed into a SlackResponse object.\n \"\"\"\n res = await self._request(\n http_verb=http_verb, api_url=api_url, req_args=req_args\n )\n data = {\n \"client\": self,\n \"http_verb\": http_verb,\n \"api_url\": api_url,\n \"req_args\": req_args,\n }\n return SlackResponse(**{**data, **res}).validate()"} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nsubmits the HTTP request with the running session or a new session.", "response": "async def _request(self, *, http_verb, api_url, req_args):\n \"\"\"Submit the HTTP request with the running session or a new session.\n Returns:\n A dictionary of the response data.\n \"\"\"\n if self.session and not self.session.closed:\n async with self.session.request(http_verb, api_url, **req_args) as res:\n self._logger.debug(\"Ran the request with existing session.\")\n return {\n \"data\": await res.json(),\n \"headers\": res.headers,\n \"status_code\": res.status,\n }\n async with aiohttp.ClientSession(\n loop=self._event_loop, timeout=aiohttp.ClientTimeout(total=self.timeout)\n ) as session:\n async with session.request(http_verb, api_url, **req_args) as res:\n self._logger.debug(\"Ran the request with a new session.\")\n return {\n \"data\": await res.json(),\n \"headers\": res.headers,\n \"status_code\": res.status,\n }"} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nconstructs the user - agent string with the package info Python version and OS version.", "response": "def _get_user_agent():\n \"\"\"Construct the user-agent header with the package info,\n Python version and OS version.\n\n Returns:\n The user agent string.\n e.g. 'Python/3.6.7 slack/2.0.0 Darwin/17.7.0'\n \"\"\"\n # __name__ returns all classes, we only want the client\n client = \"{0}/{1}\".format(__name__.split(\".\")[0], ver.__version__)\n python_version = \"Python/{v.major}.{v.minor}.{v.micro}\".format(\n v=sys.version_info\n )\n system_info = \"{0}/{1}\".format(platform.system(), platform.release())\n user_agent_string = \" \".join([python_version, client, system_info])\n return user_agent_string"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef validate(self):\n if self.status_code == 200 and self.data.get(\"ok\", False):\n self._logger.debug(\"Received the following response: %s\", self.data)\n return self\n msg = \"The request to the Slack API failed.\"\n raise e.SlackApiError(message=msg, response=self.data)", "response": "Check if the response from Slack was successful."} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncreating and send an onboarding welcome message to new users.", "response": "def onboarding_message(**payload):\n \"\"\"Create and send an onboarding welcome message to new users. Save the\n time stamp of this message so we can update this message in the future.\n \"\"\"\n # Get WebClient so you can communicate back to Slack.\n web_client = payload[\"web_client\"]\n\n # Get the id of the Slack user associated with the incoming event\n user_id = payload[\"data\"][\"user\"][\"id\"]\n\n # Open a DM with the new user.\n response = web_client.im_open(user_id)\n channel = response[\"channel\"][\"id\"]\n\n # Post the onboarding message.\n start_onboarding(web_client, user_id, channel)"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nlog the time it takes to run a function.", "response": "def log_time(func: Callable[..., Any]) -> Callable[..., Any]:\n \"\"\"Log the time it takes to run a function. It's sort of like timeit, but\n prettier.\n \"\"\"\n\n def wrapper(*args, **kwargs):\n start_time = time.time()\n log.info(\"%s starting...\", func.__name__.title())\n ret = func(*args, **kwargs)\n log.info(\n \"%s finished (%s)\",\n func.__name__.title(),\n datetime.timedelta(seconds=int(time.time() - start_time)),\n )\n return ret\n\n return wrapper"} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nforks the daemon process.", "response": "def daemonize(self) -> None:\n \"\"\"We double-fork here to detach the daemon process from the parent.\n If we were to just fork the child as a daemon, we'd have to worry about the\n parent process exiting zombifying the daemon.\"\"\"\n if os.fork() == 0:\n pid = os.fork()\n if pid == 0:\n try:\n # Closing the sys.stdout and stderr file descriptors here causes\n # the program to crash when attempting to log.\n os.close(sys.stdout.fileno())\n os.close(sys.stderr.fileno())\n self._run()\n sys.exit(0)\n except Exception as exception:\n LOG.info(\"Not running %s due to %s\", self._name, str(exception))\n sys.exit(1)\n else:\n sys.exit(0)"} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\npopulates the internal state of this object from the given graph.", "response": "def populate_from_trace_graph(self, graph: TraceGraph) -> None:\n \"\"\"Populates this graph from the given one based on affected_files\n \"\"\"\n # Track which trace frames have been visited as we populate the full\n # traces of the graph.\n self._visited_trace_frame_ids: Set[int] = set()\n\n self._populate_affected_issues(graph)\n\n if not self._affected_issues_only:\n # Finds issues from the conditions and saves them.\n # Also saves traces that have been trimmed to the affected\n # conditions.\n self._populate_issues_from_affected_trace_frames(graph)\n\n # Traces populated above may be missing all traces because\n # _populate_issues_from_affected_trace_frames only populates\n # traces that reach the affected conditions in one direction. We\n # may need to populate traces in other directions too.\n #\n # For example:\n #\n # Issue_x reaches affected_file_x via postcondition_x (forward\n # trace, i.e. trace leading to source). None of its backward\n # traces (leading to sinks) reach the affected files.\n #\n # _populate_issues_from_affected_trace_frames would have copied its\n # forward traces and trimmed it to those reaching postcondition_x.\n # We cannot blindly populate all forward traces in this case as\n # branches not leading to postcondition_x are unnecessary.\n #\n # However, in this specific example, all backward traces are needed\n # to give a complete picture of which sinks the issue reaches.\n # The following ensures that.\n for instance_id in self._issue_instances.keys():\n first_hop_ids = self._issue_instance_trace_frame_assoc[instance_id]\n fwd_trace_ids = {\n tf_id\n for tf_id in first_hop_ids\n if self._trace_frames[tf_id].kind == TraceKind.POSTCONDITION\n }\n bwd_trace_ids = {\n tf_id\n for tf_id in first_hop_ids\n if self._trace_frames[tf_id].kind == TraceKind.PRECONDITION\n }\n\n if len(fwd_trace_ids) == 0:\n self._populate_issue_trace(\n graph, instance_id, TraceKind.POSTCONDITION\n )\n\n if len(bwd_trace_ids) == 0:\n self._populate_issue_trace(\n graph, instance_id, TraceKind.PRECONDITION\n )"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _populate_affected_issues(self, graph: TraceGraph) -> None:\n affected_instance_ids = [\n instance.id.local_id\n for instance in graph._issue_instances.values()\n if self._is_filename_prefixed_with(\n graph.get_text(instance.filename_id), self._affected_files\n )\n ]\n\n for instance_id in affected_instance_ids:\n if instance_id in self._issue_instances:\n continue\n self._populate_issue_and_traces(graph, instance_id)", "response": "Populates the trimmed graph with issues whose locations are in\n affected_files based on data in the input graph."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _populate_issues_from_affected_trace_frames(self, graph: TraceGraph) -> None:\n\n initial_trace_frames = [\n trace_frame\n for trace_frame in graph._trace_frames.values()\n if self._is_filename_prefixed_with(\n graph.get_text(trace_frame.filename_id), self._affected_files\n )\n ]\n\n self._populate_issues_from_affected_conditions(\n initial_trace_frames,\n graph,\n lambda trace_frame_id: (\n graph._trace_frame_issue_instance_assoc[trace_frame_id]\n ),\n lambda trace_frame: (\n [\n graph._trace_frames[trace_frame_id]\n for trace_frame_id in graph._trace_frames_rev_map[\n (trace_frame.caller_id.local_id, trace_frame.caller_port)\n ]\n if graph._trace_frames[trace_frame_id].kind == trace_frame.kind\n ]\n ),\n lambda instance_id: (self._get_leaf_names(graph, instance_id)),\n lambda trace_frame_id: (\n self._get_leaf_names_from_pairs(\n graph, graph._trace_frame_leaf_assoc[trace_frame_id]\n )\n ),\n lambda instance, trace_frame: (\n self.add_issue_instance_trace_frame_assoc(instance, trace_frame)\n ),\n lambda trace_frame_id: (\n self._add_trace_frame(graph, graph._trace_frames[trace_frame_id])\n ),\n lambda initial_trace_frame_ids: (\n self._populate_trace(graph, initial_trace_frame_ids)\n ),\n )", "response": "Populate this TrimmedGraph with the issues that are affected by the affected_files."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _populate_issues_from_affected_conditions(\n self,\n initial_conditions,\n graph: TraceGraph,\n get_issue_instances_from_condition_id,\n get_condition_parent,\n get_instance_leaves,\n get_condition_leaves,\n add_instance_condition_assoc,\n add_condition,\n add_traces,\n ) -> None:\n \"\"\"Helper for populating reachable issue instances from the initial\n pre/postconditions. Also populates conditions/traces reachable from\n these instances. Traces are populated only in the direction this is\n called from: i.e. if initial_conditions are preconditions, only the\n backward trace is populated.\n\n Params:\n initial_conditions: The initial collection of pre/postconditions to\n start searching for issues from.\n\n graph: The trace graph to search for issues. Nodes/edges in this graph\n will be copied over to the local state\n\n get_issue_instances_from_condition_id: Function that returns all\n issue instances associated with a given a pre/postcondition id.\n\n get_condition_parent: Function that returns the parent condition of\n a given pre/postcondition. Given a pre/postcondition, p, its parent p',\n is the pre/postcondition that calls it, i.e. p.caller = p'.callee\n\n get_instance_leaves: Function that returns a collection of leaf names\n associated with the given issue instance ID.\n\n get_condition_leaves: Function that returns a collection of leaf names\n associated with the given (pre/post)condition ID.\n\n add_instance_condition_assoc: Function that takes in the issue\n instance and condition and adds the assoc between them.\n\n add_condition: Function that adds a condition to the graph given its\n id. This must add all the condition's assocs with the leaves because\n we don't filter out any condition-leaf assocs.\n\n add_traces: Function that takes a list of initial conditions and\n adds all conditions reachable from these to the graph.\n \"\"\"\n visited: Set[int] = set()\n que = list(zip(initial_conditions, initial_conditions))\n\n while len(que) > 0:\n condition, initial_condition = que.pop()\n cond_id = condition.id.local_id\n if cond_id in visited:\n continue\n visited.add(cond_id)\n\n # Found instance(s) related to the current condition. Yay.\n # This instance may have been found before, but process it again\n # anyway because we need to add the assoc with this condition.\n for instance_id in get_issue_instances_from_condition_id(cond_id):\n # Check if the leaves (sources/sinks) of the issue reach\n # the same leaves as the initial conditions. The issue is\n # relevant only if the conditions intersect.\n instance = graph._issue_instances[instance_id]\n issue_leaves = set(get_instance_leaves(instance.id.local_id))\n initial_leaves = set(\n get_condition_leaves(initial_condition.id.local_id)\n )\n leaves = issue_leaves.intersection(initial_leaves)\n if len(leaves) > 0:\n if instance_id not in self._issue_instances:\n self._populate_issue(graph, instance_id)\n add_instance_condition_assoc(instance, condition)\n\n # Conditions that call this may have originated from other issues,\n # keep searching for parent conditions leading to this one.\n que.extend(\n [\n (cond, initial_condition)\n for cond in get_condition_parent(condition)\n if cond.id.local_id not in visited\n ]\n )\n\n # Add traces leading out from initial_conditions, and all visited\n # conditions leading back towards the issues.\n initial_condition_ids = [\n condition.id.local_id for condition in initial_conditions\n ]\n add_traces(initial_condition_ids)\n for condition_id in visited:\n add_condition(condition_id)", "response": "Helper function to populate the issues that are reachable from the given conditions."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ncopying an issue over from the given trace graph including all its traces and assocs.", "response": "def _populate_issue_and_traces(self, graph: TraceGraph, instance_id: int) -> None:\n \"\"\" Copies an issue over from the given trace graph, including all its\n traces and assocs.\n \"\"\"\n self._populate_issue(graph, instance_id)\n self._populate_issue_trace(graph, instance_id)"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _populate_issue(self, graph: TraceGraph, instance_id: int) -> None:\n instance = graph._issue_instances[instance_id]\n issue = graph._issues[instance.issue_id.local_id]\n self._populate_shared_text(graph, instance.message_id)\n self._populate_shared_text(graph, instance.filename_id)\n self._populate_shared_text(graph, instance.callable_id)\n\n self.add_issue_instance(instance)\n self.add_issue(issue)\n\n if instance_id in graph._issue_instance_fix_info:\n issue_fix_info = graph._issue_instance_fix_info[instance_id]\n self.add_issue_instance_fix_info(instance, issue_fix_info)\n\n for shared_text_id in graph._issue_instance_shared_text_assoc[instance_id]:\n shared_text = graph._shared_texts[shared_text_id]\n if shared_text_id not in self._shared_texts:\n self.add_shared_text(shared_text)\n self.add_issue_instance_shared_text_assoc(instance, shared_text)", "response": "Adds an issue to the trace graph along with relevant information about the issue."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _populate_trace(self, graph: TraceGraph, trace_frame_ids: List[int]) -> None:\n while len(trace_frame_ids) > 0:\n trace_frame_id = trace_frame_ids.pop()\n if trace_frame_id in self._visited_trace_frame_ids:\n continue\n\n trace_frame = graph._trace_frames[trace_frame_id]\n self._add_trace_frame(graph, trace_frame)\n self._visited_trace_frame_ids.add(trace_frame_id)\n\n key = (trace_frame.callee_id.local_id, trace_frame.callee_port)\n trace_frame_ids.extend(\n [\n trace_frame_id\n for trace_frame_id in graph._trace_frames_map[key]\n if trace_frame_id not in self._visited_trace_frame_ids\n and graph._trace_frames[trace_frame_id].kind == trace_frame.kind\n ]\n )", "response": "Populates the forward and backward traces of the given trace graph."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _add_trace_frame(self, graph: TraceGraph, trace_frame: TraceFrame) -> None:\n trace_frame_id = trace_frame.id.local_id\n self.add_trace_frame(trace_frame)\n self._populate_shared_text(graph, trace_frame.filename_id)\n self._populate_shared_text(graph, trace_frame.caller_id)\n self._populate_shared_text(graph, trace_frame.callee_id)\n for (leaf_id, depth) in graph._trace_frame_leaf_assoc[trace_frame_id]:\n leaf = graph._shared_texts[leaf_id]\n if leaf_id not in self._shared_texts:\n self.add_shared_text(leaf)\n self.add_trace_frame_leaf_assoc(trace_frame, leaf, depth)", "response": "Adds the trace frame to this trace_graph."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _compute_symbolic_link_mapping(\n directory: str, extensions: Iterable[str]\n) -> Dict[str, str]:\n \"\"\"\n Given a shared analysis directory, produce a mapping from actual source files\n to files contained within this directory. Only includes files which have\n one of the provided extensions.\n\n Watchman watches actual source files, so when a change is detected to a\n file, this mapping can be used to identify what file changed from Pyre's\n perspective.\n \"\"\"\n symbolic_links = {}\n try:\n for symbolic_link in find_paths_with_extensions(directory, extensions):\n symbolic_links[os.path.realpath(symbolic_link)] = symbolic_link\n except subprocess.CalledProcessError as error:\n LOG.warning(\n \"Exception encountered trying to find source files \"\n \"in the analysis directory: `%s`\",\n error,\n )\n LOG.warning(\"Starting with an empty set of tracked files.\")\n return symbolic_links", "response": "Compute a mapping from actual source files\nAttributeNames to files contained within this analysis directory."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nacquire the lock file.", "response": "def acquire_lock(path: str, blocking: bool) -> Generator[Optional[int], None, None]:\n \"\"\"Raises an OSError if the lock can't be acquired\"\"\"\n try:\n with open(path, \"w+\") as lockfile:\n if not blocking:\n lock_command = fcntl.LOCK_EX | fcntl.LOCK_NB\n else:\n lock_command = fcntl.LOCK_EX\n\n fcntl.lockf(lockfile.fileno(), lock_command)\n yield lockfile.fileno()\n fcntl.lockf(lockfile.fileno(), fcntl.LOCK_UN)\n\n except FileNotFoundError:\n yield"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef process_updated_files(self, paths: List[str]) -> List[str]:\n return [path for path in paths if self._is_tracked(path)]", "response": "Processes a list of paths which were added removed or deleted files and returns a list of files that need to be tracking."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef process_updated_files(self, paths: List[str]) -> List[str]:\n tracked_files = []\n\n deleted_paths = [path for path in paths if not os.path.isfile(path)]\n # TODO(T40580762) use buck targets to properly identify what new files belong\n # to the project rather than checking if they are within the current directory\n new_paths = [\n path\n for path in paths\n if path not in self._symbolic_links\n and os.path.isfile(path)\n and is_parent(os.getcwd(), path)\n ]\n updated_paths = [\n path\n for path in paths\n if path not in deleted_paths and path not in new_paths\n ]\n\n if deleted_paths:\n LOG.info(\"Detected deleted paths: `%s`.\", \"`,`\".join(deleted_paths))\n for path in deleted_paths:\n link = self._symbolic_links.pop(path, None)\n if link:\n try:\n _delete_symbolic_link(link)\n tracked_files.append(link)\n except OSError:\n LOG.warning(\"Failed to delete link at `%s`.\", link)\n\n if new_paths:\n LOG.info(\"Detected new paths: %s.\", \",\".join(new_paths))\n try:\n for path, relative_link in buck.resolve_relative_paths(\n new_paths\n ).items():\n link = os.path.join(self.get_root(), relative_link)\n try:\n add_symbolic_link(link, path)\n self._symbolic_links[path] = link\n tracked_files.append(link)\n except OSError:\n LOG.warning(\"Failed to add link at %s.\", link)\n except buck.BuckException as error:\n LOG.error(\"Exception occurred when querying buck: %s\", error)\n LOG.error(\"No new paths will be added to the analysis directory.\")\n\n for path in updated_paths:\n if path in self._symbolic_links:\n tracked_files.append(self._symbolic_links[path])\n elif self._is_tracked(path):\n tracked_files.append(path)\n\n return tracked_files", "response": "Return the paths in the analysis directory that have been modified or deleted."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef list(\n self, root: str, patterns: List[str], exclude: Optional[List[str]] = None\n ) -> List[str]:\n \"\"\"\n Return the list of files that match any of the patterns within root.\n If exclude is provided, files that match an exclude pattern are omitted.\n\n Note: The `find` command does not understand globs properly.\n e.g. 'a/*.py' will match 'a/b/c.py'\n For this reason, avoid calling this method with glob patterns.\n \"\"\"\n\n command = [\"find\", \".\"]\n command += self._match_any(patterns)\n if exclude:\n command += [\"-and\", \"!\"]\n command += self._match_any(exclude)\n return (\n subprocess.run(command, stdout=subprocess.PIPE, cwd=root)\n .stdout.decode(\"utf-8\")\n .split()\n )", "response": "Return the list of files that match any of the patterns within root."} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\niterating over all issues and count the number of times each callable is seen.", "response": "def _compute_callables_count(self, iters: Dict[str, Any]):\n \"\"\"Iterate over all issues and count the number of times each callable\n is seen.\"\"\"\n count = dict.fromkeys([issue[\"callable\"] for issue in iters[\"issues\"]], 0)\n for issue in iters[\"issues\"]:\n count[issue[\"callable\"]] += 1\n\n return count"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nset boilerplate when creating a Run object", "response": "def _create_empty_run(\n self, status=RunStatus.FINISHED, status_description=None\n ) -> Run:\n \"\"\"setting boilerplate when creating a Run object\"\"\"\n run = Run(\n job_id=self.summary[\"job_id\"],\n issue_instances=[],\n date=datetime.datetime.now(),\n status=status,\n status_description=status_description,\n repository=self.summary[\"repository\"],\n branch=self.summary[\"branch\"],\n commit_hash=self.summary[\"commit_hash\"],\n kind=self.summary[\"run_kind\"],\n )\n return run"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ngenerate the issue instance for a given entry.", "response": "def _generate_issue(self, run, entry, callablesCount):\n \"\"\"Insert the issue instance into a run. This includes creating (for\n new issues) or finding (for existing issues) Issue objects to associate\n with the instances.\n Also create sink entries and associate related issues\"\"\"\n\n trace_frames = []\n\n for p in entry[\"preconditions\"]:\n tf = self._generate_issue_precondition(run, entry, p)\n trace_frames.append(tf)\n\n for p in entry[\"postconditions\"]:\n tf = self._generate_issue_postcondition(run, entry, p)\n trace_frames.append(tf)\n\n features = set()\n for f in entry[\"features\"]:\n features.update(self._generate_issue_feature_contents(entry, f))\n\n callable = entry[\"callable\"]\n handle = self._get_issue_handle(entry)\n # TODO: record depth in issue_sink and issue_source assoc, but this can\n # be different per instance, so should be stored with the instance.\n initial_sources = {\n self._get_shared_text(SharedTextKind.SOURCE, s)\n for (s, _) in entry[\"initial_sources\"]\n }\n final_sinks = {\n self._get_shared_text(SharedTextKind.SINK, s)\n for (s, _) in entry[\"final_sinks\"]\n }\n\n issue = Issue.Record(\n id=IssueDBID(),\n code=entry[\"code\"],\n handle=handle,\n status=IssueStatus.UNCATEGORIZED,\n first_seen=run.date,\n run_id=run.id,\n )\n\n self.graph.add_issue(issue)\n\n fix_info = None\n fix_info_id = None\n if entry.get(\"fix_info\") is not None:\n fix_info = IssueInstanceFixInfo.Record(\n id=DBID(), fix_info=json.dumps(entry[\"fix_info\"])\n )\n fix_info_id = fix_info.id\n\n message = self._get_shared_text(SharedTextKind.MESSAGE, entry[\"message\"])\n filename_record = self._get_shared_text(\n SharedTextKind.FILENAME, entry[\"filename\"]\n )\n callable_record = self._get_shared_text(SharedTextKind.CALLABLE, callable)\n\n instance = IssueInstance.Record(\n id=DBID(),\n issue_id=issue.id,\n location=self.get_location(entry),\n filename_id=filename_record.id,\n callable_id=callable_record.id,\n run_id=run.id,\n fix_info_id=fix_info_id,\n message_id=message.id,\n rank=0,\n min_trace_length_to_sources=self._get_minimum_trace_length(\n entry[\"postconditions\"]\n ),\n min_trace_length_to_sinks=self._get_minimum_trace_length(\n entry[\"preconditions\"]\n ),\n callable_count=callablesCount[callable],\n )\n\n for sink in final_sinks:\n self.graph.add_issue_instance_shared_text_assoc(instance, sink)\n for source in initial_sources:\n self.graph.add_issue_instance_shared_text_assoc(instance, source)\n\n if fix_info is not None:\n self.graph.add_issue_instance_fix_info(instance, fix_info)\n\n for trace_frame in trace_frames:\n self.graph.add_issue_instance_trace_frame_assoc(instance, trace_frame)\n\n for feature in features:\n feature = self._get_shared_text(SharedTextKind.FEATURE, feature)\n self.graph.add_issue_instance_shared_text_assoc(instance, feature)\n\n self.graph.add_issue_instance(instance)"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_history_without_ipython_calls(self):\n line_count = 0\n filtered_history = []\n for line in self.history:\n if \"get_ipython().\" not in line:\n filtered_history.append(line)\n line_count += 1\n return filtered_history, line_count", "response": "Return a list of lines in the execution history that are not in iPython calls."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _prep_save(self):\n log.info(\"Preparing bulk save.\")\n self.graph.update_bulk_saver(self.bulk_saver)\n\n log.info(\n \"Dropped %d unused preconditions, %d are missing\",\n sum(len(v) for v in self.summary[\"precondition_entries\"].values()),\n len(self.summary[\"missing_preconditions\"]),\n )\n\n log.info(\n \"Dropped %d unused postconditions, %d are missing\",\n sum(len(v) for v in self.summary[\"postcondition_entries\"].values()),\n len(self.summary[\"missing_postconditions\"]),\n )\n del self.summary[\"postcondition_entries\"]", "response": "Prepares the bulk saver to load the trace graph info into the trace graph database."} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nsaves the current run into the database.", "response": "def _save(self) -> RunSummary:\n \"\"\" Saves bulk saver's info into the databases in bulk.\n \"\"\"\n assert self.summary[\"run\"] is not None, \"Must have called process before\"\n\n trace_frames = self.bulk_saver.get_items_to_add(TraceFrame)\n log.info(\n \"Saving %d issues, %d trace frames, %d trace annotations\",\n len(self.bulk_saver.get_items_to_add(Issue)),\n len(self.bulk_saver.get_items_to_add(TraceFrame)),\n len(self.bulk_saver.get_items_to_add(TraceFrameAnnotation)),\n )\n\n num_pre = 0\n num_post = 0\n for frame in trace_frames:\n if frame.kind == TraceKind.PRECONDITION:\n num_pre += 1\n elif frame.kind == TraceKind.POSTCONDITION:\n num_post += 1\n log.info(\n \"Within trace frames: %d preconditions, %d postconditions\",\n num_pre,\n num_post,\n )\n\n with self.database.make_session() as session:\n pk_gen = self.primary_key_generator.reserve(\n session, [Run], use_lock=self.use_lock\n )\n self.summary[\"run\"].id.resolve(id=pk_gen.get(Run), is_new=True)\n session.add(self.summary[\"run\"])\n session.commit()\n\n run_id = self.summary[\"run\"].id.resolved()\n self.summary[\"run\"] = None # Invalidate it\n\n self.bulk_saver.save_all(self.database, self.use_lock)\n\n # Now that the run is finished, fetch it from the DB again and set its\n # status to FINISHED.\n with self.database.make_session() as session:\n run = session.query(self.RUN_MODEL).filter_by(id=run_id).one()\n run.status = RunStatus.FINISHED\n session.add(run)\n session.commit()\n run_summary = run.get_summary()\n\n run_summary.num_invisible_issues = 0\n run_summary.num_missing_preconditions = len(\n self.summary[\"missing_preconditions\"]\n )\n run_summary.num_missing_postconditions = len(\n self.summary[\"missing_postconditions\"]\n )\n\n return run_summary"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning a list of pairs ( leaf_name distance )", "response": "def _parse_leaves(self, leaves) -> List[Tuple[str, int]]:\n \"\"\"Returns a list of pairs (leaf_name, distance)\"\"\"\n return [(self._leaf_name(leaf), 0) for leaf in leaves]"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ngenerates all file handles represented by the analysis.", "response": "def file_handles(self) -> Iterable[IO[str]]:\n \"\"\"Generates all file handles represented by the analysis.\n Callee owns file handle and closes it when the next is yielded or the\n generator ends.\n \"\"\"\n if self.file_handle:\n yield self.file_handle\n self.file_handle.close()\n self.file_handle = None\n else:\n for name in self.file_names():\n with open(name, \"r\") as f:\n yield f"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef file_names(self) -> Iterable[str]:\n if self.is_sharded():\n yield from ShardedFile(self.filename_spec).get_filenames()\n elif self.filename_spec:\n yield self.filename_spec", "response": "Generates all file names that are used to generate file_handles."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef prepare(cls, session, pkgen, items):\n for item in cls.merge(session, items):\n if hasattr(item, \"id\"):\n item.id.resolve(id=pkgen.get(cls), is_new=True)\n yield cls.to_dict(item)", "response": "This is called immediately before the items are written to the the\n database."} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nconverts a string into a SourceLocation object.", "response": "def process_result_value(self, value, dialect):\n \"\"\"\n SQLAlchemy uses this to convert a string into a SourceLocation object.\n We separate the fields by a |\n \"\"\"\n if value is None:\n return None\n\n p = value.split(\"|\")\n\n if len(p) == 0:\n return None\n return SourceLocation(*map(int, p))"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreserving all the items in the given record set.", "response": "def reserve(\n self,\n session: Session,\n saving_classes: List[Type],\n item_counts: Optional[Dict[str, int]] = None,\n use_lock: bool = False,\n ) -> \"PrimaryKeyGenerator\":\n \"\"\"\n session - Session for DB operations.\n saving_classes - class objects that need to be saved e.g. Issue, Run\n item_counts - map from class name to the number of items, for preallocating\n id ranges\n \"\"\"\n query_classes = {cls for cls in saving_classes if cls in self.QUERY_CLASSES}\n for cls in query_classes:\n if item_counts and cls.__name__ in item_counts:\n count = item_counts[cls.__name__]\n else:\n count = 1\n self._reserve_id_range(session, cls, count, use_lock)\n\n return self"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncomputing the diff handle for a specific issue.", "response": "def compute_diff_handle(filename, old_line, code):\n \"\"\"Uses the absolute line and ignores the callable/character offsets.\n Used only in determining whether new issues are old issues.\n \"\"\"\n key = \"{filename}:{old_line}:{code}\".format(\n filename=filename, old_line=old_line, code=code\n )\n return BaseParser.compute_handle_from_key(key)"} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ndetermines whether we are connected to a capable terminal.", "response": "def is_capable_terminal(file=sys.stderr) -> bool:\n \"\"\"\n Determine whether we are connected to a capable terminal.\n \"\"\"\n if not os.isatty(file.fileno()):\n return False\n terminal = os.getenv(\"TERM\", \"dumb\")\n # Hardcoded list of non-capable terminals.\n return terminal not in [\"dumb\", \"emacs\"]"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _find_directory_upwards(base: str, target: str) -> Optional[str]:\n while True:\n step = os.path.join(base, target)\n LOG.debug(\"Trying with: `%s`\", step)\n if os.path.isdir(step):\n return step\n parent_directory = os.path.dirname(base)\n if parent_directory == base:\n # We have reached the root.\n break\n base = parent_directory\n return None", "response": "Walk directories upwards from base until the root directory is reached."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef compute_targets_to_build(self, targets: Iterable[str]) -> Iterable[BuildTarget]:\n normalized_targets = set()\n for target in targets:\n normalized_targets.update(self._normalize_target(target))\n\n targets_to_parse = deque(normalized_targets)\n targets_seen = set(normalized_targets)\n targets_not_found = []\n build_targets = set()\n\n while targets_to_parse:\n next_target_string = targets_to_parse.popleft()\n next_target = self._parse_target(next_target_string)\n build_file = self.parser.parse_file(next_target.build_file_directory)\n if next_target.name not in build_file.targets:\n targets_not_found.append(next_target_string)\n continue\n target_to_build = build_file.targets[next_target.name]\n build_targets.add(target_to_build)\n\n new_targets = [\n dependency\n for dependency in target_to_build.dependencies\n if dependency not in targets_seen\n ]\n targets_to_parse.extend(new_targets)\n targets_seen.update(new_targets)\n\n if targets_not_found and self.fail_on_unbuilt_target:\n raise BuilderException(\n \"Target(s) could not be built: {}\".format(\", \".join(targets_not_found)),\n targets=targets_not_found,\n )\n return build_targets", "response": "Compute the set of targets to build for the given targets."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef compute_reverse_dependencies(\n self, targets: Iterable[BuildTarget]\n ) -> Mapping[str, Iterable[BuildTarget]]:\n \"\"\"\n Compute the set of targets which depend on each target.\n \"\"\"\n result = defaultdict(list)\n for target in targets:\n for dependency in target.dependencies:\n result[dependency].append(target)\n return result", "response": "Compute the set of targets which depend on each target."} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nconsumes a key from the configuration file.", "response": "def consume(self, key, default=None, current=None, print_on_success=False):\n \"\"\"\n Consume a key from the configuration. When a key is consumed, it\n is removed from the configuration.\n\n If not found, the default is returned. If the current value is not\n None, it will be returned instead, but the key will still be\n considered consumed.\n \"\"\"\n\n value = self._configuration.pop(key, default)\n if current:\n return current\n if value and print_on_success:\n LOG.debug(\"Found %s: `%s`\", key, \", \".join(value))\n if value and key in self._deprecated:\n LOG.warning(\n \"Configuration file uses deprecated item `%s`: \"\n \"please migrate to its replacement `%s`\",\n key,\n self._deprecated[key],\n )\n return value"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nsetting the location of the analysis output from the static analysis tool.", "response": "def analysis_output(self, location: Optional[str] = None) -> None:\n \"\"\"Sets the location of the output from the static analysis tool.\n\n Parameters:\n location: str Filesystem location for the results.\n \"\"\"\n try:\n if not location:\n location = self.prompt(\n \"Analysis results: \",\n history_key=\"analysis_results\",\n completer=PathCompleter(),\n )\n self.current_analysis_output = AnalysisOutput.from_str(location)\n except AnalysisOutputError as e:\n raise UserError(f\"Error loading results: {e}\")"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef issue(self, issue_instance_id):\n with self.db.make_session() as session:\n selected_issue = (\n session.query(IssueInstance)\n .filter(IssueInstance.id == issue_instance_id)\n .scalar()\n )\n\n if selected_issue is None:\n self.warning(\n f\"Issue {issue_instance_id} doesn't exist. \"\n \"Type 'issues' for available issues.\"\n )\n return\n\n self.sources = self._get_leaves_issue_instance(\n session, issue_instance_id, SharedTextKind.SOURCE\n )\n\n self.sinks = self._get_leaves_issue_instance(\n session, issue_instance_id, SharedTextKind.SINK\n )\n\n self.current_issue_instance_id = int(selected_issue.id)\n self.current_frame_id = -1\n self.current_trace_frame_index = 1 # first one after the source\n\n print(f\"Set issue to {issue_instance_id}.\")\n if int(selected_issue.run_id) != self.current_run_id:\n self.current_run_id = int(selected_issue.run_id)\n print(f\"Set run to {self.current_run_id}.\")\n print()\n\n self._generate_trace_from_issue()\n self.show()", "response": "Select an issue instance."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nlist the issues for the selected run.", "response": "def issues(\n self,\n use_pager: bool = None,\n *,\n codes: Optional[Union[int, List[int]]] = None,\n callables: Optional[Union[str, List[str]]] = None,\n filenames: Optional[Union[str, List[str]]] = None,\n ):\n \"\"\"Lists issues for the selected run.\n\n Parameters (all optional):\n use_pager: bool use a unix style pager for output\n codes: int or list[int] issue codes to filter on\n callables: str or list[str] callables to filter on (supports wildcards)\n filenames: str or list[str] filenames to filter on (supports wildcards)\n\n String filters support LIKE wildcards (%, _) from SQL:\n % matches anything (like .* in regex)\n _ matches 1 character (like . in regex)\n\n For example:\n callables=[\n \"%occurs.anywhere%\",\n \"%at.end\",\n \"at.start%\",\n \"etc.\",\n ])\n \"\"\"\n pager = self._resolve_pager(use_pager)\n\n with self.db.make_session() as session:\n query = (\n session.query(\n IssueInstance.id,\n FilenameText.contents.label(\"filename\"),\n IssueInstance.location,\n Issue.code,\n CallableText.contents.label(\"callable\"),\n MessageText.contents.label(\"message\"),\n )\n .filter(IssueInstance.run_id == self.current_run_id)\n .join(FilenameText, FilenameText.id == IssueInstance.filename_id)\n .join(CallableText, CallableText.id == IssueInstance.callable_id)\n )\n\n if codes is not None:\n query = self._add_list_or_int_filter_to_query(\n codes, query, Issue.code, \"codes\"\n )\n\n if callables is not None:\n query = self._add_list_or_string_filter_to_query(\n callables, query, CallableText.contents, \"callables\"\n )\n\n if filenames is not None:\n query = self._add_list_or_string_filter_to_query(\n filenames, query, FilenameText.contents, \"filenames\"\n )\n\n issues = query.join(Issue, IssueInstance.issue_id == Issue.id).join(\n MessageText, MessageText.id == IssueInstance.message_id\n )\n\n sources_list = [\n self._get_leaves_issue_instance(\n session, int(issue.id), SharedTextKind.SOURCE\n )\n for issue in issues\n ]\n sinks_list = [\n self._get_leaves_issue_instance(\n session, int(issue.id), SharedTextKind.SINK\n )\n for issue in issues\n ]\n\n issue_strings = [\n self._create_issue_output_string(issue, sources, sinks)\n for issue, sources, sinks in zip(issues, sources_list, sinks_list)\n ]\n issue_output = f\"\\n{'-' * 80}\\n\".join(issue_strings)\n pager(issue_output)\n print(f\"Found {len(issue_strings)} issues with run_id {self.current_run_id}.\")"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef frames(\n self,\n *,\n callers: Optional[Union[str, List[str]]] = None,\n callees: Optional[Union[str, List[str]]] = None,\n kind: Optional[TraceKind] = None,\n limit: Optional[int] = 10,\n ):\n \"\"\"Display trace frames independent of the current issue.\n\n Parameters (all optional):\n callers: str or list[str] filter traces by this caller name\n callees: str or list[str] filter traces by this callee name\n kind: precondition|postcondition the type of trace frames to show\n limit: int (default: 10) how many trace frames to display\n (specify limit=None for all)\n\n Sample usage:\n frames callers=\"module.function\", kind=postcondition\n\n String filters support LIKE wildcards (%, _) from SQL:\n % matches anything (like .* in regex)\n _ matches 1 character (like . in regex)\n \"\"\"\n with self.db.make_session() as session:\n query = (\n session.query(\n TraceFrame.id,\n CallerText.contents.label(\"caller\"),\n TraceFrame.caller_port,\n CalleeText.contents.label(\"callee\"),\n TraceFrame.callee_port,\n )\n .filter(TraceFrame.run_id == self.current_run_id)\n .join(CallerText, CallerText.id == TraceFrame.caller_id)\n .join(CalleeText, CalleeText.id == TraceFrame.callee_id)\n )\n\n if callers is not None:\n query = self._add_list_or_string_filter_to_query(\n callers, query, CallerText.contents, \"callers\"\n )\n\n if callees is not None:\n query = self._add_list_or_string_filter_to_query(\n callees, query, CalleeText.contents, \"callees\"\n )\n\n if kind is not None:\n if kind not in {TraceKind.PRECONDITION, TraceKind.POSTCONDITION}:\n raise UserError(\n \"Try 'frames kind=postcondition'\"\n \" or 'frames kind=precondition'.\"\n )\n query = query.filter(TraceFrame.kind == kind)\n\n if limit is not None and not isinstance(limit, int):\n raise UserError(\"'limit' should be an int or None.\")\n\n trace_frames = query.group_by(TraceFrame.id).order_by(\n CallerText.contents, CalleeText.contents\n )\n\n total_trace_frames = trace_frames.count()\n limit = limit or total_trace_frames\n\n self._output_trace_frames(\n self._group_trace_frames(trace_frames, limit), limit, total_trace_frames\n )", "response": "Display trace frames independent of the current issue."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef next_cursor_location(self):\n self._verify_entrypoint_selected()\n self.current_trace_frame_index = min(\n self.current_trace_frame_index + 1, len(self.trace_tuples) - 1\n )\n self.trace()", "response": "Move cursor to the next trace frame."} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nmoves cursor to the previous trace frame.", "response": "def prev_cursor_location(self):\n \"\"\"Move cursor to the previous trace frame.\n \"\"\"\n self._verify_entrypoint_selected()\n self.current_trace_frame_index = max(self.current_trace_frame_index - 1, 0)\n self.trace()"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\njump to a specific trace frame in a trace.", "response": "def jump(self, selected_number: int) -> None:\n \"\"\"Jump to a specific trace frame in a trace.\n\n Parameters:\n selected_number: int the trace frame number from trace output\n \"\"\"\n self._verify_entrypoint_selected()\n if selected_number < 1 or selected_number > len(self.trace_tuples):\n raise UserError(\n \"Trace frame number out of bounds \"\n f\"(expected 1-{len(self.trace_tuples)} but got {selected_number}).\"\n )\n\n self.current_trace_frame_index = selected_number - 1\n self.trace()"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nshowing and select branches for a branched trace.", "response": "def branch(self, selected_number: Optional[int] = None) -> None:\n \"\"\"Show and select branches for a branched trace.\n - [*] signifies the current branch that is selected\n - will prompt for branch selection if called with no argument\n - will automatically select a branch if called with an argument\n\n Parameters (optional):\n selected_number: int branch number from expand output\n\n Example output:\n\n Suppose we have the trace output:\n \u2387 [callable] [port] [location]\n --> +2 leaf source module/main.py:26|4|8\n module.main root module/helper.py:76|5|10\n module.helper.process root module/helper.py:76|5|10\n +3 leaf sink module/main.py:74|1|9\n\n Calling expand will result in the output:\n [*] leaf\n [0 hops: source]\n [module/main.py:26|4|8]\n [1] module.helper.preprocess\n [1 hops: source]\n [module/main.py:21|4|8]\n \"\"\"\n self._verify_entrypoint_selected()\n self._verify_multiple_branches()\n\n with self.db.make_session() as session:\n branches, leaves_strings = self._get_branch_options(session)\n\n if selected_number is None:\n selected_number = self._select_branch_trace_frame(\n branches, leaves_strings\n )\n\n if (\n not isinstance(selected_number, int)\n or selected_number < 1\n or selected_number > len(branches)\n ):\n raise UserError(\n \"Branch number invalid \"\n f\"(expected 1-{len(branches)} but got {selected_number}).\"\n )\n\n new_navigation = self._navigate_trace_frames(\n session, branches, selected_number - 1\n )\n\n new_trace_tuples = self._create_trace_tuples(new_navigation)\n\n if self._is_before_root():\n new_trace_tuples.reverse()\n self.trace_tuples = (\n new_trace_tuples\n + self.trace_tuples[self.current_trace_frame_index + 1 :]\n )\n\n # If length of prefix changes, it will change some indices\n trace_frame_index_delta = (\n len(new_navigation) - self.current_trace_frame_index - 1\n )\n self.current_trace_frame_index += trace_frame_index_delta\n else:\n self.trace_tuples = (\n self.trace_tuples[: self.current_trace_frame_index] + new_trace_tuples\n )\n\n self.trace()"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nshowing source code around the current trace frame location.", "response": "def list_source_code(self, context: int = 5) -> None:\n \"\"\"Show source code around the current trace frame location.\n\n Parameters:\n context: int number of lines to show above and below trace location\n (default: 5)\n \"\"\"\n self._verify_entrypoint_selected()\n\n current_trace_frame = self.trace_tuples[\n self.current_trace_frame_index\n ].trace_frame\n\n filename = os.path.join(self.repository_directory, current_trace_frame.filename)\n file_lines: List[str] = []\n\n try:\n # Use readlines instead of enumerate(file) because mock_open\n # doesn't support __iter__ until python 3.7.1.\n with open(filename, \"r\") as file:\n file_lines = file.readlines()\n except FileNotFoundError:\n self.warning(f\"Couldn't open {filename}.\")\n return\n\n self._output_file_lines(current_trace_frame, file_lines, context)"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef details(\n self, *, limit: Optional[int] = 5, kind: Optional[TraceKind] = None\n ) -> None:\n \"\"\"Show additional info about the current trace frame.\n\n Parameters (all optional):\n limit: int number of related post/pre conditions to show\n (pass limit=None to show all)\n \"\"\"\n self._verify_entrypoint_selected()\n if limit is not None and not isinstance(limit, int):\n raise UserError(\"'limit' should be an int or None.\")\n if kind not in {TraceKind.PRECONDITION, TraceKind.POSTCONDITION, None}:\n raise UserError(\n \"Try 'details kind=postcondition'\" \" or 'details kind=precondition'.\"\n )\n\n current_trace_tuple = self.trace_tuples[self.current_trace_frame_index]\n current_trace_frame = current_trace_tuple.trace_frame\n callable, _port = self._get_callable_from_trace_tuple(current_trace_tuple)\n\n print(self._create_trace_frame_output_string(current_trace_frame))\n print(\n f\"\\nIssues in callable ({callable}):\",\n self._num_issues_with_callable(current_trace_frame.callee),\n )\n\n if kind is None or kind == TraceKind.POSTCONDITION:\n print(f\"\\nPostconditions with caller ({callable}):\")\n self.frames(callers=callable, kind=TraceKind.POSTCONDITION, limit=limit)\n\n if kind is None or kind == TraceKind.PRECONDITION:\n print(f\"\\nPreconditions with caller ({callable}):\")\n self.frames(callers=callable, kind=TraceKind.PRECONDITION, limit=limit)", "response": "Show additional info about the current trace frame."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ngroups the trace frames into a list of lists of tuples.", "response": "def _group_trace_frames(\n self, trace_frames: Iterable[TraceFrameQueryResult], limit: int\n ) -> Dict[Tuple[str, str], List[TraceFrameQueryResult]]:\n \"\"\"Buckets together trace frames that have the same caller:caller_port.\n \"\"\"\n # pyre-fixme[9]: caller_buckets has type `DefaultDict[Tuple[str, str], List[T...\n caller_buckets: DefaultDict[\n Tuple[str, str], List[TraceFrameQueryResult]\n ] = defaultdict(list)\n for trace_frame in itertools.islice(trace_frames, limit):\n caller_buckets[(trace_frame.caller, trace_frame.caller_port)].append(\n trace_frame\n )\n return caller_buckets"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _next_trace_frames(\n self,\n session: Session,\n trace_frame: TraceFrameQueryResult,\n visited_ids: Set[int],\n backwards: bool = False,\n ) -> List[TraceFrameQueryResult]:\n \"\"\"Finds all trace frames that the given trace_frame flows to.\n\n When backwards=True, the result will include the parameter trace_frame,\n since we are filtering on the parameter's callee.\n \"\"\"\n query = (\n session.query(\n TraceFrame.id,\n TraceFrame.caller_id,\n CallerText.contents.label(\"caller\"),\n TraceFrame.caller_port,\n TraceFrame.callee_id,\n CalleeText.contents.label(\"callee\"),\n TraceFrame.callee_port,\n TraceFrame.callee_location,\n TraceFrame.kind,\n FilenameText.contents.label(\"filename\"),\n TraceFrameLeafAssoc.trace_length,\n )\n .filter(TraceFrame.run_id == self.current_run_id)\n .filter(TraceFrame.kind == trace_frame.kind)\n .join(CallerText, CallerText.id == TraceFrame.caller_id)\n .join(CalleeText, CalleeText.id == TraceFrame.callee_id)\n .join(FilenameText, FilenameText.id == TraceFrame.filename_id)\n .filter(\n TraceFrame.caller_id != TraceFrame.callee_id\n ) # skip recursive calls for now\n )\n if backwards:\n query = query.filter(TraceFrame.callee_id == trace_frame.caller_id).filter(\n TraceFrame.callee_port == trace_frame.caller_port\n )\n else:\n query = query.filter(TraceFrame.caller_id == trace_frame.callee_id).filter(\n TraceFrame.caller_port == trace_frame.callee_port\n )\n\n results = (\n query.join(\n TraceFrameLeafAssoc, TraceFrameLeafAssoc.trace_frame_id == TraceFrame.id\n )\n .group_by(TraceFrame.id)\n .order_by(TraceFrameLeafAssoc.trace_length, TraceFrame.callee_location)\n )\n filter_leaves = (\n self.sources if trace_frame.kind == TraceKind.POSTCONDITION else self.sinks\n )\n\n filtered_results = []\n for frame in results:\n if int(frame.id) not in visited_ids and filter_leaves.intersection(\n set(\n self._get_leaves_trace_frame(\n session,\n int(frame.id),\n self._trace_kind_to_shared_text_kind(frame.kind),\n )\n )\n ):\n filtered_results.append(frame)\n\n return filtered_results", "response": "Finds all the next trace frames that the given trace_frame flows to."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nshows the name of the current callable in the trace", "response": "def callable(self) -> Optional[str]:\n \"\"\"Show the name of the current callable in the trace\"\"\"\n if self.current_trace_frame_index != -1:\n return self._get_callable_from_trace_tuple(\n self.trace_tuples[self.current_trace_frame_index]\n )[0]\n return None"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _get_callable_from_trace_tuple(\n self, trace_tuple: TraceTuple\n ) -> Tuple[str, str]:\n \"\"\"Returns either (caller, caller_port) or (callee, callee_port).\n \"\"\"\n trace_frame = trace_tuple.trace_frame\n if trace_tuple.placeholder:\n return trace_frame.caller, trace_frame.caller_port\n return trace_frame.callee, trace_frame.callee_port", "response": "Returns either caller callee or caller_port depending on the trace_tuple."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef require_option(current_ctx: click.Context, param_name: str) -> None:\n\n ctx = current_ctx\n param_definition = None\n while ctx is not None:\n # ctx.command.params has the actual definition of the param. We use\n # this when raising the exception.\n param_definition = next(\n (p for p in ctx.command.params if p.name == param_name), None\n )\n\n # ctx.params has the current value of the parameter, as set by the user.\n if ctx.params.get(param_name):\n return\n ctx = ctx.parent\n\n assert param_definition, f\"unknown parameter {param_name}\"\n raise click.MissingParameter(ctx=current_ctx, param=param_definition)", "response": "Raise an exception if an option isn t required."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef default_database(ctx: click.Context, _param: Parameter, value: Optional[str]):\n if value:\n return value\n\n if ctx.params[\"repository\"]:\n return os.path.join(ctx.params[\"repository\"], DB.DEFAULT_DB_FILE)\n\n raise click.BadParameter(\"Could not guess a database location\")", "response": "Try to guess a reasonable database name by looking at the repository path"} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\noutputs DB models in a lint - friendly format", "response": "def lint(click_ctx: click.Context, run_id: int, filenames: List[str]) -> None:\n \"\"\"Output DB models in a lint-friendly format\"\"\"\n ctx = click_ctx.obj\n require_option(click_ctx, \"repository\")\n\n paths = [Path(p).resolve() for p in filenames]\n root = Path(ctx.repository).resolve()\n relative = [str(Path(f).relative_to(root)) for f in paths]\n\n with ctx.database.make_session() as session:\n instances = (\n session.query(\n IssueInstance.filename,\n IssueInstance.location,\n SharedText.contents,\n Issue.code,\n )\n .filter(IssueInstance.run_id == run_id)\n .filter(IssueInstance.filename.in_(relative))\n .join(Issue, Issue.id == IssueInstance.issue_id)\n .join(SharedText, SharedText.id == IssueInstance.message_id)\n .all()\n )\n\n with ctx.database.make_session() as session:\n frames = (\n session.query(\n CallerText.contents.label(\"caller\"),\n CalleeText.contents.label(\"callee\"),\n FilenameText.contents.label(\"filename\"),\n TraceFrame.callee_location,\n TraceFrame.kind,\n TraceFrame.callee_port,\n TraceFrame.caller_port,\n )\n .filter(TraceFrame.run_id == run_id)\n .join(FilenameText, FilenameText.id == TraceFrame.filename_id)\n .filter(FilenameText.contents.in_(relative))\n .join(CallerText, CallerText.id == TraceFrame.caller_id)\n .join(CalleeText, CalleeText.id == TraceFrame.callee_id)\n .all()\n )\n\n def entry(filename, code, message, location):\n return {\n \"filename\": str(root / filename),\n \"code\": code,\n \"message\": message,\n \"line\": location.line_no,\n \"col\": location.begin_column,\n \"length\": location.begin_column + location.end_column + 1,\n }\n\n lints = [\n entry(i.filename, str(i.code), i.contents, i.location) for i in instances\n ] + [\n entry(\n i.filename,\n i.kind.name,\n f\"{i.caller}:{i.caller_port} -> {i.callee}->{i.callee_port}\",\n i.callee_location,\n )\n for i in frames\n ]\n\n for l in sorted(lints, key=itemgetter(\"filename\", \"line\", \"code\", \"col\")):\n click.echo(json.dumps(l))"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns a mapping from absolute source path to absolute output path as specified by the sources object. Files are not guaranteed to exist.", "response": "def resolve_source_mapping(\n source_directory: str, output_directory: str, sources: Sources\n) -> Mapping[str, str]:\n \"\"\"\n Returns a mapping from absolute source path to absolute output path as specified\n by the sources object. Files are not guaranteed to exist.\n \"\"\"\n result = {\n os.path.join(source_directory, source_file): os.path.join(\n output_directory, output_file\n )\n for source_file, output_file in sources.files.items()\n }\n\n filesystem = get_filesystem()\n for glob in sources.globs:\n matches = filesystem.list(source_directory, glob.patterns, exclude=glob.exclude)\n result.update(\n {\n os.path.join(source_directory, match): os.path.join(\n output_directory, match\n )\n for match in matches\n }\n )\n return result"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nquerying buck to obtain a mapping from each absolute path to the relative version of the analysis directory.", "response": "def resolve_relative_paths(paths: List[str]) -> Dict[str, str]:\n \"\"\"\n Query buck to obtain a mapping from each absolute path to the relative\n location in the analysis directory.\n \"\"\"\n buck_root = find_buck_root(os.getcwd())\n if buck_root is None:\n LOG.error(\n \"Buck root couldn't be found. Returning empty analysis directory mapping.\"\n )\n return {}\n command = [\n \"buck\",\n \"query\",\n \"--json\",\n \"--output-attribute\",\n \".*\",\n \"owner(%s)\",\n *paths,\n ]\n try:\n output = json.loads(\n subprocess.check_output(command, timeout=30, stderr=subprocess.DEVNULL)\n .decode()\n .strip()\n )\n except (\n subprocess.TimeoutExpired,\n subprocess.CalledProcessError,\n json.decoder.JSONDecodeError,\n ) as error:\n raise BuckException(\"Querying buck for relative paths failed: {}\".format(error))\n # TODO(T40580762) we should use the owner name to determine which files are a\n # part of the pyre project\n results = {}\n for path in paths:\n # For each path, search for the target that owns it.\n for owner in output.values():\n prefix = os.path.join(buck_root, owner[\"buck.base_path\"]) + os.sep\n\n if not path.startswith(prefix):\n continue\n\n suffix = path[len(prefix) :]\n\n if suffix not in owner[\"srcs\"]:\n continue\n\n if \"buck.base_module\" in owner:\n base_path = os.path.join(*owner[\"buck.base_module\"].split(\".\"))\n else:\n base_path = owner[\"buck.base_path\"]\n\n results[path] = os.path.join(base_path, owner[\"srcs\"][suffix])\n break # move on to next path\n return results"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nbuilding the target directories and yield the paths to the link trees.", "response": "def build(self, targets: Iterable[str]) -> Iterable[str]:\n \"\"\"\n Shell out to buck to build the targets, then yield the paths to the\n link trees.\n \"\"\"\n return generate_source_directories(\n targets, build=self._build, prompt=self._prompt\n )"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _get_parameter_string(self) -> str:\n parameters = []\n for parameter in self.parameters:\n name = parameter[\"name\"]\n if parameter[\"type\"]:\n name += \": \" + dequalify(parameter[\"type\"])\n if parameter[\"value\"]:\n name += \" = \" + parameter[\"value\"]\n elif parameter[\"value\"]:\n name += \"=\" + parameter[\"value\"]\n parameters.append(name)\n return \", \".join(parameters)", "response": "Returns a string representation of the parameters."} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ndetermines if a stub completely types a function.", "response": "def is_complete(self) -> bool:\n \"\"\" Determines if a stub completely types a function \"\"\"\n if not self.actual:\n return False\n for parameter in self.parameters:\n if parameter[\"name\"] != \"self\" and not parameter[\"type\"]:\n return False\n return True"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef to_string(self) -> str:\n classes = defaultdict(list)\n typing_imports = set()\n contents = \"\"\n\n # import necessary modules from typing\n for stub in self._stubs:\n typing_imports.update(stub.get_typing_imports())\n alphabetical_imports = sorted(list(typing_imports))\n if alphabetical_imports:\n contents += \"from typing import {}\\n\\n\".format(\n \", \".join(str(type_import) for type_import in alphabetical_imports)\n )\n\n for stub in self._fields:\n parent = _relativize_access(stub.parent, stub.path)\n # Ignore nested classes\n if len(parent) == 1:\n classes[parent[0]].append(stub)\n else:\n contents += stub.to_string() + \"\\n\"\n\n for stub in self._methods:\n parent = _relativize_access(stub.parent, stub.path)\n # Ignore nested classes\n if len(parent) == 1:\n classes[parent[0]].append(stub)\n\n for stub in self._functions:\n contents += stub.to_string() + \"\\n\"\n\n for parent, stubs in classes.items():\n contents += \"\\nclass {}:\\n\".format(parent)\n for stub in stubs:\n contents += \" {}\\n\".format(stub.to_string().replace(\"\\n\", \"\\n \"))\n return contents", "response": "Return a string representation of the object."} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ndownloading a trained weights config and preprocessor.", "response": "def download(url):\n \"\"\"Download a trained weights, config and preprocessor.\n\n Args:\n url (str): target url.\n \"\"\"\n filepath = get_file(fname='tmp.zip', origin=url, extract=True)\n base_dir = os.path.dirname(filepath)\n weights_file = os.path.join(base_dir, 'weights.h5')\n params_file = os.path.join(base_dir, 'params.json')\n preprocessor_file = os.path.join(base_dir, 'preprocessor.pickle')\n\n return weights_file, params_file, preprocessor_file"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef load_data_and_labels(filename, encoding='utf-8'):\n sents, labels = [], []\n words, tags = [], []\n with open(filename, encoding=encoding) as f:\n for line in f:\n line = line.rstrip()\n if line:\n word, tag = line.split('\\t')\n words.append(word)\n tags.append(tag)\n else:\n sents.append(words)\n labels.append(tags)\n words, tags = [], []\n\n return sents, labels", "response": "Loads data and labels from a file."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef filter_embeddings(embeddings, vocab, dim):\n if not isinstance(embeddings, dict):\n return\n _embeddings = np.zeros([len(vocab), dim])\n for word in vocab:\n if word in embeddings:\n word_idx = vocab[word]\n _embeddings[word_idx] = embeddings[word]\n\n return _embeddings", "response": "Loads word vectors in numpy array."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef load_glove(file):\n model = {}\n with open(file, encoding=\"utf8\", errors='ignore') as f:\n for line in f:\n line = line.split(' ')\n word = line[0]\n vector = np.array([float(val) for val in line[1:]])\n model[word] = vector\n\n return model", "response": "Loads GloVe vectors in numpy array."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef add_token(self, token):\n token = self.process_token(token)\n self._token_count.update([token])", "response": "Add a token to the vocabulary."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ngetting the list of token_id given doc.", "response": "def doc2id(self, doc):\n \"\"\"Get the list of token_id given doc.\n\n Args:\n doc (list): document.\n\n Returns:\n list: int id of doc.\n \"\"\"\n doc = map(self.process_token, doc)\n return [self.token_to_id(token) for token in doc]"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ngets the token_id of given token.", "response": "def token_to_id(self, token):\n \"\"\"Get the token_id of given token.\n\n Args:\n token (str): token from vocabulary.\n\n Returns:\n int: int id of token.\n \"\"\"\n token = self.process_token(token)\n return self._token2id.get(token, len(self._token2id) - 1)"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef predict_proba(self, text):\n assert isinstance(text, str)\n\n words = self.tokenizer(text)\n X = self.preprocessor.transform([words])\n y = self.model.predict(X)\n y = y[0] # reduce batch dimension.\n\n return y", "response": "Predict the probability of each class in the input text."} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nanalyze text and return pretty format.", "response": "def analyze(self, text):\n \"\"\"Analyze text and return pretty format.\n\n Args:\n text: string, the input text.\n\n Returns:\n res: dict.\n\n Examples:\n >>> text = 'President Obama is speaking at the White House.'\n >>> model.analyze(text)\n {\n \"words\": [\n \"President\",\n \"Obama\",\n \"is\",\n \"speaking\",\n \"at\",\n \"the\",\n \"White\",\n \"House.\"\n ],\n \"entities\": [\n {\n \"beginOffset\": 1,\n \"endOffset\": 2,\n \"score\": 1,\n \"text\": \"Obama\",\n \"type\": \"PER\"\n },\n {\n \"beginOffset\": 6,\n \"endOffset\": 8,\n \"score\": 1,\n \"text\": \"White House.\",\n \"type\": \"ORG\"\n }\n ]\n }\n \"\"\"\n pred = self.predict_proba(text)\n tags = self._get_tags(pred)\n prob = self._get_prob(pred)\n res = self._build_response(text, tags, prob)\n\n return res"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef predict(self, text):\n pred = self.predict_proba(text)\n tags = self._get_tags(pred)\n\n return tags", "response": "Predict using the model."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\npad nested sequences to the same length.", "response": "def pad_nested_sequences(sequences, dtype='int32'):\n \"\"\"Pads nested sequences to the same length.\n\n This function transforms a list of list sequences\n into a 3D Numpy array of shape `(num_samples, max_sent_len, max_word_len)`.\n\n Args:\n sequences: List of lists of lists.\n dtype: Type of the output sequences.\n\n # Returns\n x: Numpy array.\n \"\"\"\n max_sent_len = 0\n max_word_len = 0\n for sent in sequences:\n max_sent_len = max(len(sent), max_sent_len)\n for word in sent:\n max_word_len = max(len(word), max_word_len)\n\n x = np.zeros((len(sequences), max_sent_len, max_word_len)).astype(dtype)\n for i, sent in enumerate(sequences):\n for j, word in enumerate(sent):\n x[i, j, :len(word)] = word\n\n return x"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nlearns vocabulary from training set.", "response": "def fit(self, X, y):\n \"\"\"Learn vocabulary from training set.\n\n Args:\n X : iterable. An iterable which yields either str, unicode or file objects.\n\n Returns:\n self : IndexTransformer.\n \"\"\"\n self._word_vocab.add_documents(X)\n self._label_vocab.add_documents(y)\n if self._use_char:\n for doc in X:\n self._char_vocab.add_documents(doc)\n\n self._word_vocab.build()\n self._char_vocab.build()\n self._label_vocab.build()\n\n return self"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ntransforming documents to document ids.", "response": "def transform(self, X, y=None):\n \"\"\"Transform documents to document ids.\n\n Uses the vocabulary learned by fit.\n\n Args:\n X : iterable\n an iterable which yields either str, unicode or file objects.\n y : iterabl, label strings.\n\n Returns:\n features: document id matrix.\n y: label id matrix.\n \"\"\"\n word_ids = [self._word_vocab.doc2id(doc) for doc in X]\n word_ids = pad_sequences(word_ids, padding='post')\n\n if self._use_char:\n char_ids = [[self._char_vocab.doc2id(w) for w in doc] for doc in X]\n char_ids = pad_nested_sequences(char_ids)\n features = [word_ids, char_ids]\n else:\n features = word_ids\n\n if y is not None:\n y = [self._label_vocab.doc2id(doc) for doc in y]\n y = pad_sequences(y, padding='post')\n y = to_categorical(y, self.label_size).astype(int)\n # In 2018/06/01, to_categorical is a bit strange.\n # >>> to_categorical([[1,3]], num_classes=4).shape\n # (1, 2, 4)\n # >>> to_categorical([[1]], num_classes=4).shape\n # (1, 4)\n # So, I expand dimensions when len(y.shape) == 2.\n y = y if len(y.shape) == 3 else np.expand_dims(y, axis=0)\n return features, y\n else:\n return features"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nlearning vocabulary and return document id matrix.", "response": "def fit_transform(self, X, y=None, **params):\n \"\"\"Learn vocabulary and return document id matrix.\n\n This is equivalent to fit followed by transform.\n\n Args:\n X : iterable\n an iterable which yields either str, unicode or file objects.\n\n Returns:\n list : document id matrix.\n list: label id matrix.\n \"\"\"\n return self.fit(X, y).transform(X, y)"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn label strings. Args: y: label id matrix. lengths: sentences length. Returns: list: list of list of strings.", "response": "def inverse_transform(self, y, lengths=None):\n \"\"\"Return label strings.\n\n Args:\n y: label id matrix.\n lengths: sentences length.\n\n Returns:\n list: list of list of strings.\n \"\"\"\n y = np.argmax(y, -1)\n inverse_y = [self._label_vocab.id2doc(ids) for ids in y]\n if lengths is not None:\n inverse_y = [iy[:l] for iy, l in zip(inverse_y, lengths)]\n\n return inverse_y"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ntransforms documents to document ids.", "response": "def transform(self, X, y=None):\n \"\"\"Transform documents to document ids.\n\n Uses the vocabulary learned by fit.\n\n Args:\n X : iterable\n an iterable which yields either str, unicode or file objects.\n y : iterabl, label strings.\n\n Returns:\n features: document id matrix.\n y: label id matrix.\n \"\"\"\n word_ids = [self._word_vocab.doc2id(doc) for doc in X]\n word_ids = pad_sequences(word_ids, padding='post')\n\n char_ids = [[self._char_vocab.doc2id(w) for w in doc] for doc in X]\n char_ids = pad_nested_sequences(char_ids)\n\n character_ids = batch_to_ids(X)\n elmo_embeddings = self._elmo(character_ids)['elmo_representations'][1]\n elmo_embeddings = elmo_embeddings.detach().numpy()\n\n features = [word_ids, char_ids, elmo_embeddings]\n\n if y is not None:\n y = [self._label_vocab.doc2id(doc) for doc in y]\n y = pad_sequences(y, padding='post')\n y = to_categorical(y, self.label_size).astype(int)\n # In 2018/06/01, to_categorical is a bit strange.\n # >>> to_categorical([[1,3]], num_classes=4).shape\n # (1, 2, 4)\n # >>> to_categorical([[1]], num_classes=4).shape\n # (1, 4)\n # So, I expand dimensions when len(y.shape) == 2.\n y = y if len(y.shape) == 3 else np.expand_dims(y, axis=0)\n return features, y\n else:\n return features"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ntraining the model for a fixed number of epochs.", "response": "def train(self, x_train, y_train, x_valid=None, y_valid=None,\n epochs=1, batch_size=32, verbose=1, callbacks=None, shuffle=True):\n \"\"\"Trains the model for a fixed number of epochs (iterations on a dataset).\n\n Args:\n x_train: list of training data.\n y_train: list of training target (label) data.\n x_valid: list of validation data.\n y_valid: list of validation target (label) data.\n batch_size: Integer.\n Number of samples per gradient update.\n If unspecified, `batch_size` will default to 32.\n epochs: Integer. Number of epochs to train the model.\n verbose: Integer. 0, 1, or 2. Verbosity mode.\n 0 = silent, 1 = progress bar, 2 = one line per epoch.\n callbacks: List of `keras.callbacks.Callback` instances.\n List of callbacks to apply during training.\n shuffle: Boolean (whether to shuffle the training data\n before each epoch). `shuffle` will default to True.\n \"\"\"\n\n train_seq = NERSequence(x_train, y_train, batch_size, self._preprocessor.transform)\n\n if x_valid and y_valid:\n valid_seq = NERSequence(x_valid, y_valid, batch_size, self._preprocessor.transform)\n f1 = F1score(valid_seq, preprocessor=self._preprocessor)\n callbacks = [f1] + callbacks if callbacks else [f1]\n\n self._model.fit_generator(generator=train_seq,\n epochs=epochs,\n callbacks=callbacks,\n verbose=verbose,\n shuffle=shuffle)"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef fit(self, x_train, y_train, x_valid=None, y_valid=None,\n epochs=1, batch_size=32, verbose=1, callbacks=None, shuffle=True):\n \"\"\"Fit the model for a fixed number of epochs.\n\n Args:\n x_train: list of training data.\n y_train: list of training target (label) data.\n x_valid: list of validation data.\n y_valid: list of validation target (label) data.\n batch_size: Integer.\n Number of samples per gradient update.\n If unspecified, `batch_size` will default to 32.\n epochs: Integer. Number of epochs to train the model.\n verbose: Integer. 0, 1, or 2. Verbosity mode.\n 0 = silent, 1 = progress bar, 2 = one line per epoch.\n callbacks: List of `keras.callbacks.Callback` instances.\n List of callbacks to apply during training.\n shuffle: Boolean (whether to shuffle the training data\n before each epoch). `shuffle` will default to True.\n \"\"\"\n p = IndexTransformer(initial_vocab=self.initial_vocab, use_char=self.use_char)\n p.fit(x_train, y_train)\n embeddings = filter_embeddings(self.embeddings, p._word_vocab.vocab, self.word_embedding_dim)\n\n model = BiLSTMCRF(char_vocab_size=p.char_vocab_size,\n word_vocab_size=p.word_vocab_size,\n num_labels=p.label_size,\n word_embedding_dim=self.word_embedding_dim,\n char_embedding_dim=self.char_embedding_dim,\n word_lstm_size=self.word_lstm_size,\n char_lstm_size=self.char_lstm_size,\n fc_dim=self.fc_dim,\n dropout=self.dropout,\n embeddings=embeddings,\n use_char=self.use_char,\n use_crf=self.use_crf)\n model, loss = model.build()\n model.compile(loss=loss, optimizer=self.optimizer)\n\n trainer = Trainer(model, preprocessor=p)\n trainer.train(x_train, y_train, x_valid, y_valid,\n epochs=epochs, batch_size=batch_size,\n verbose=verbose, callbacks=callbacks,\n shuffle=shuffle)\n\n self.p = p\n self.model = model", "response": "Fit the model for a fixed number of epochs."} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns the prediction of the model on the given test data.", "response": "def predict(self, x_test):\n \"\"\"Returns the prediction of the model on the given test data.\n\n Args:\n x_test : array-like, shape = (n_samples, sent_length)\n Test samples.\n\n Returns:\n y_pred : array-like, shape = (n_smaples, sent_length)\n Prediction labels for x.\n \"\"\"\n if self.model:\n lengths = map(len, x_test)\n x_test = self.p.transform(x_test)\n y_pred = self.model.predict(x_test)\n y_pred = self.p.inverse_transform(y_pred, lengths)\n return y_pred \n else:\n raise OSError('Could not find a model. Call load(dir_path).')"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef analyze(self, text, tokenizer=str.split):\n if not self.tagger:\n self.tagger = Tagger(self.model,\n preprocessor=self.p,\n tokenizer=tokenizer)\n\n return self.tagger.analyze(text)", "response": "Analyze text and return pretty format."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_log_normalization_constant(self, input_energy, mask, **kwargs):\n # should have logZ[:, i] == logZ[:, j] for any i, j\n logZ = self.recursion(input_energy, mask, return_sequences=False, **kwargs)\n return logZ[:, 0]", "response": "Compute the logarithm of the normalization constant Z where the input_energy is the sum of the logarithm of the input_energy"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get_energy(self, y_true, input_energy, mask):\n input_energy = K.sum(input_energy * y_true, 2) # (B, T)\n chain_energy = K.sum(K.dot(y_true[:, :-1, :], self.chain_kernel) * y_true[:, 1:, :], 2) # (B, T-1)\n\n if mask is not None:\n mask = K.cast(mask, K.floatx())\n chain_mask = mask[:, :-1] * mask[:, 1:] # (B, T-1), mask[:,:-1]*mask[:,1:] makes it work with any padding\n input_energy = input_energy * mask\n chain_energy = chain_energy * chain_mask\n total_energy = K.sum(input_energy, -1) + K.sum(chain_energy, -1) # (B, )\n\n return total_energy", "response": "Get the total energy of the logarithm of the logarithm of the input and chain."} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncomputes the loss i. e. negative log likelihood", "response": "def get_negative_log_likelihood(self, y_true, X, mask):\n \"\"\"Compute the loss, i.e., negative log likelihood (normalize by number of time steps)\n likelihood = 1/Z * exp(-E) -> neg_log_like = - log(1/Z * exp(-E)) = logZ + E\n \"\"\"\n input_energy = self.activation(K.dot(X, self.kernel) + self.bias)\n if self.use_boundary:\n input_energy = self.add_boundary_energy(input_energy, mask, self.left_boundary, self.right_boundary)\n energy = self.get_energy(y_true, input_energy, mask)\n logZ = self.get_log_normalization_constant(input_energy, mask, input_length=K.int_shape(X)[1])\n nloglik = logZ + energy\n if mask is not None:\n nloglik = nloglik / K.sum(K.cast(mask, K.floatx()), 1)\n else:\n nloglik = nloglik / K.cast(K.shape(X)[1], K.floatx())\n return nloglik"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef recursion(self, input_energy, mask=None, go_backwards=False, return_sequences=True, return_logZ=True, input_length=None):\n chain_energy = self.chain_kernel\n chain_energy = K.expand_dims(chain_energy, 0) # shape=(1, F, F): F=num of output features. 1st F is for t-1, 2nd F for t\n prev_target_val = K.zeros_like(input_energy[:, 0, :]) # shape=(B, F), dtype=float32\n\n if go_backwards:\n input_energy = K.reverse(input_energy, 1)\n if mask is not None:\n mask = K.reverse(mask, 1)\n\n initial_states = [prev_target_val, K.zeros_like(prev_target_val[:, :1])]\n constants = [chain_energy]\n\n if mask is not None:\n mask2 = K.cast(K.concatenate([mask, K.zeros_like(mask[:, :1])], axis=1), K.floatx())\n constants.append(mask2)\n\n def _step(input_energy_i, states):\n return self.step(input_energy_i, states, return_logZ)\n\n target_val_last, target_val_seq, _ = K.rnn(_step, input_energy, initial_states, constants=constants,\n input_length=input_length, unroll=self.unroll)\n\n if return_sequences:\n if go_backwards:\n target_val_seq = K.reverse(target_val_seq, 1)\n return target_val_seq\n else:\n return target_val_last", "response": "This function is used to compute the Viterbi s best path lookup table for a given input energy."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef extract_signature(sql):\n sql = force_text(sql)\n sql = sql.strip()\n first_space = sql.find(\" \")\n if first_space < 0:\n return sql\n\n second_space = sql.find(\" \", first_space + 1)\n\n sql_type = sql[0:first_space].upper()\n\n if sql_type in [\"INSERT\", \"DELETE\"]:\n keyword = \"INTO\" if sql_type == \"INSERT\" else \"FROM\"\n sql_type = sql_type + \" \" + keyword\n\n table_name = look_for_table(sql, keyword)\n elif sql_type in [\"CREATE\", \"DROP\"]:\n # 2nd word is part of SQL type\n sql_type = sql_type + sql[first_space:second_space]\n table_name = \"\"\n elif sql_type == \"UPDATE\":\n table_name = look_for_table(sql, \"UPDATE\")\n elif sql_type == \"SELECT\":\n # Name is first table\n try:\n sql_type = \"SELECT FROM\"\n table_name = look_for_table(sql, \"FROM\")\n except Exception:\n table_name = \"\"\n else:\n # No name\n table_name = \"\"\n\n signature = \" \".join(filter(bool, [sql_type, table_name]))\n return signature", "response": "Extracts a minimal signature from a given SQL statement."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _timed_queue_join(self, timeout):\n deadline = time.time() + timeout\n\n with self._queue.all_tasks_done:\n while self._queue.unfinished_tasks:\n delay = deadline - time.time()\n if delay <= 0:\n # timed out\n return False\n\n self._queue.all_tasks_done.wait(timeout=delay)\n\n return True", "response": "implementation of Queue. join which takes a timeout argument returns True on success False on timeout"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef start(self):\n self._lock.acquire()\n try:\n if not self._thread:\n self._thread = Thread(target=self._target)\n self._thread.setDaemon(True)\n self._thread.name = \"elasticapm sender thread\"\n self._thread.start()\n finally:\n self._lock.release()", "response": "Starts the task thread."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef stop(self, timeout=None):\n self._lock.acquire()\n try:\n if self._thread:\n self._queue.put_nowait(self._terminator)\n self._thread.join(timeout=timeout)\n self._thread = None\n finally:\n self._lock.release()", "response": "Stops the task thread."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nget version without importing from elasticapm", "response": "def get_version():\n \"\"\"\n Get version without importing from elasticapm. This avoids any side effects\n from importing while installing and/or building the module\n :return: a string, indicating the version\n \"\"\"\n version_file = open(os.path.join(\"elasticapm\", \"version.py\"), encoding=\"utf-8\")\n for line in version_file:\n if line.startswith(\"__version__\"):\n version_tuple = ast.literal_eval(line.split(\" = \")[1])\n return \".\".join(map(str, version_tuple))\n return \"unknown\""} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn only proper HTTP headers.", "response": "def get_headers(environ):\n \"\"\"\n Returns only proper HTTP headers.\n \"\"\"\n for key, value in compat.iteritems(environ):\n key = str(key)\n if key.startswith(\"HTTP_\") and key not in (\"HTTP_CONTENT_TYPE\", \"HTTP_CONTENT_LENGTH\"):\n yield key[5:].replace(\"_\", \"-\").lower(), value\n elif key in (\"CONTENT_TYPE\", \"CONTENT_LENGTH\"):\n yield key.replace(\"_\", \"-\").lower(), value"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nflush the queue. This method should only be called from the event processing queue :param sync: if true, flushes the queue synchronously in the current thread :return: None", "response": "def _flush(self, buffer):\n \"\"\"\n Flush the queue. This method should only be called from the event processing queue\n :param sync: if true, flushes the queue synchronously in the current thread\n :return: None\n \"\"\"\n if not self.state.should_try():\n logger.error(\"dropping flushed data due to transport failure back-off\")\n else:\n fileobj = buffer.fileobj # get a reference to the fileobj before closing the gzip file\n buffer.close()\n\n # StringIO on Python 2 does not have getbuffer, so we need to fall back to getvalue\n data = fileobj.getbuffer() if hasattr(fileobj, \"getbuffer\") else fileobj.getvalue()\n try:\n self.send(data)\n self.handle_transport_success()\n except Exception as e:\n self.handle_transport_fail(e)"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nclose the connection to the cache.", "response": "def close(self):\n \"\"\"\n Cleans up resources and closes connection\n :return:\n \"\"\"\n if self._closed:\n return\n self._closed = True\n self.queue(\"close\", None)\n if not self._flushed.wait(timeout=self._max_flush_time):\n raise ValueError(\"close timed out\")"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef flush(self):\n self.queue(None, None, flush=True)\n if not self._flushed.wait(timeout=self._max_flush_time):\n raise ValueError(\"flush timed out\")", "response": "Trigger a flush of the queue."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncalls by the transport when a send failure occurs.", "response": "def handle_transport_fail(self, exception=None, **kwargs):\n \"\"\"\n Failure handler called by the transport on send failure\n \"\"\"\n message = str(exception)\n logger.error(\"Failed to submit message: %r\", message, exc_info=getattr(exception, \"print_trace\", True))\n self.state.set_fail()"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nputs an item into the queue.", "response": "def put(self, item, block=True, timeout=None, chill=True):\n \"\"\"Put an item into the queue.\n\n If optional args 'block' is true and 'timeout' is None (the default),\n block if necessary until a free slot is available. If 'timeout' is\n a non-negative number, it blocks at most 'timeout' seconds and raises\n the Full exception if no free slot was available within that time.\n Otherwise ('block' is false), put an item on the queue if a free slot\n is immediately available, else raise the Full exception ('timeout'\n is ignored in that case).\n \"\"\"\n with self.not_full:\n if self.maxsize > 0:\n if not block:\n if self._qsize() >= self.maxsize:\n raise compat.queue.Full\n elif timeout is None:\n while self._qsize() >= self.maxsize:\n self.not_full.wait()\n elif timeout < 0:\n raise ValueError(\"'timeout' must be a non-negative number\")\n else:\n endtime = time.time() + timeout\n while self._qsize() >= self.maxsize:\n remaining = endtime - time.time()\n if remaining <= 0.0:\n raise compat.queue.Full\n self.not_full.wait(remaining)\n self._put(item)\n self.unfinished_tasks += 1\n if (\n not chill\n or self._qsize() > self._chill_until\n or (time.time() - self._last_unchill) > self._max_chill_time\n ):\n self.not_empty.notify()\n self._last_unchill = time.time()"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn request - host as defined by RFC 2965.", "response": "def request_host(request):\n \"\"\"Return request-host, as defined by RFC 2965.\n\n Variation from RFC: returned value is lowercased, for convenient\n comparison.\n\n \"\"\"\n url = request.get_full_url()\n parse_result = compat.urlparse.urlparse(url)\n scheme, host, port = parse_result.scheme, parse_result.hostname, parse_result.port\n try:\n port = int(port)\n except ValueError:\n pass\n if host == \"\":\n host = request.get_header(\"Host\", \"\")\n\n if port != default_ports.get(scheme):\n host = \"%s:%s\" % (host, port)\n return host"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nget an ElasticAPM client.", "response": "def get_client(client=None):\n \"\"\"\n Get an ElasticAPM client.\n\n :param client:\n :return:\n :rtype: elasticapm.base.Client\n \"\"\"\n global _client\n\n tmp_client = client is not None\n if not tmp_client:\n config = getattr(django_settings, \"ELASTIC_APM\", {})\n client = config.get(\"CLIENT\", default_client_class)\n\n if _client[0] != client:\n client_class = import_string(client)\n instance = client_class()\n if not tmp_client:\n _client = (client, instance)\n return instance\n return _client[1]"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _get_installed_apps_paths():\n out = set()\n for app in django_settings.INSTALLED_APPS:\n out.add(app)\n return out", "response": "Generate a list of modules in settings. INSTALLED_APPS."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _get_stack_info_for_trace(\n self,\n frames,\n library_frame_context_lines=None,\n in_app_frame_context_lines=None,\n with_locals=True,\n locals_processor_func=None,\n ):\n \"\"\"If the stacktrace originates within the elasticapm module, it will skip\n frames until some other module comes up.\"\"\"\n return list(\n iterate_with_template_sources(\n frames,\n with_locals=with_locals,\n library_frame_context_lines=library_frame_context_lines,\n in_app_frame_context_lines=in_app_frame_context_lines,\n include_paths_re=self.include_paths_re,\n exclude_paths_re=self.exclude_paths_re,\n locals_processor_func=locals_processor_func,\n )\n )", "response": "Returns a list of stack info for the given stack trace."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef send(self, url, **kwargs):\n if self.config.server_url:\n return super(DjangoClient, self).send(url, **kwargs)\n else:\n self.error_logger.error(\"No server configured, and elasticapm not installed. Cannot send message\")\n return None", "response": "Sends a message to the server."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef deprecated(alternative=None):\n\n def real_decorator(func):\n @functools.wraps(func)\n def new_func(*args, **kwargs):\n msg = \"Call to deprecated function {0}.\".format(func.__name__)\n if alternative:\n msg += \" Use {0} instead\".format(alternative)\n warnings.warn_explicit(\n msg,\n category=DeprecationWarning,\n filename=compat.get_function_code(func).co_filename,\n lineno=compat.get_function_code(func).co_firstlineno + 1,\n )\n return func(*args, **kwargs)\n\n return new_func\n\n return real_decorator", "response": "This is a decorator which marks functions\n as deprecated. It will result in a warning being emitted\n when the function is used."} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nregistering a new metric set.", "response": "def register(self, class_path):\n \"\"\"\n Register a new metric set\n :param class_path: a string with the import path of the metricset class\n \"\"\"\n if class_path in self._metricsets:\n return\n else:\n try:\n class_obj = import_string(class_path)\n self._metricsets[class_path] = class_obj(self)\n except ImportError as e:\n logger.warning(\"Could not register %s metricset: %s\", class_path, compat.text_type(e))"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncollects metrics from all registered metric sets.", "response": "def collect(self):\n \"\"\"\n Collect metrics from all registered metric sets\n :return:\n \"\"\"\n logger.debug(\"Collecting metrics\")\n\n for name, metricset in compat.iteritems(self._metricsets):\n data = metricset.collect()\n if data:\n self._queue_func(constants.METRICSET, data)"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef counter(self, name):\n with self._lock:\n if name not in self._counters:\n if self._registry._ignore_patterns and any(\n pattern.match(name) for pattern in self._registry._ignore_patterns\n ):\n counter = noop_metric\n else:\n counter = Counter(name)\n self._counters[name] = counter\n return self._counters[name]", "response": "Returns an existing Counter object or creates a new Counter object if it does not exist."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef gauge(self, name):\n with self._lock:\n if name not in self._gauges:\n if self._registry._ignore_patterns and any(\n pattern.match(name) for pattern in self._registry._ignore_patterns\n ):\n gauge = noop_metric\n else:\n gauge = Gauge(name)\n self._gauges[name] = gauge\n return self._gauges[name]", "response": "Returns an existing gauge object or creates a new one if it does not exist."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncollecting all metrics attached to this metricset and returns it together with a timestamp in microsecond precision.", "response": "def collect(self):\n \"\"\"\n Collects all metrics attached to this metricset, and returns it as a list, together with a timestamp\n in microsecond precision.\n\n The format of the return value should be\n\n {\n \"samples\": {\"metric.name\": {\"value\": some_float}, ...},\n \"timestamp\": unix epoch in microsecond precision\n }\n \"\"\"\n samples = {}\n if self._counters:\n samples.update(\n {label: {\"value\": c.val} for label, c in compat.iteritems(self._counters) if c is not noop_metric}\n )\n if self._gauges:\n samples.update(\n {label: {\"value\": g.val} for label, g in compat.iteritems(self._gauges) if g is not noop_metric}\n )\n if samples:\n return {\"samples\": samples, \"timestamp\": int(time.time() * 1000000)}"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef tag(**tags):\n transaction = execution_context.get_transaction()\n if not transaction:\n error_logger.warning(\"Ignored tags %s. No transaction currently active.\", \", \".join(tags.keys()))\n else:\n transaction.tag(**tags)", "response": "Tags the current page with the given tags."} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nbegins a new span with the given name and type.", "response": "def begin_span(self, name, span_type, context=None, leaf=False, tags=None):\n \"\"\"\n Begin a new span\n :param name: name of the span\n :param span_type: type of the span\n :param context: a context dict\n :param leaf: True if this is a leaf span\n :param tags: a flat string/string dict of tags\n :return: the Span object\n \"\"\"\n return self._begin_span(name, span_type, context=context, leaf=leaf, tags=tags, parent_span_id=None)"} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nensures that the parent trace_parent has a span_id generate one and return it.", "response": "def ensure_parent_id(self):\n \"\"\"If current trace_parent has no span_id, generate one, then return it\n\n This is used to generate a span ID which the RUM agent will use to correlate\n the RUM transaction with the backend transaction.\n \"\"\"\n if self.trace_parent.span_id == self.id:\n self.trace_parent.span_id = \"%016x\" % random.getrandbits(64)\n logger.debug(\"Set parent id to generated %s\", self.trace_parent.span_id)\n return self.trace_parent.span_id"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ntag this transaction with one or multiple key - value tags.", "response": "def tag(self, **tags):\n \"\"\"\n Tag this transaction with one or multiple key/value tags. Both the values should be strings\n\n transaction_obj.tag(key1=\"value1\", key2=\"value2\")\n\n Note that keys will be dedotted, replacing dot (.), star (*) and double quote (\") with an underscore (_)\n \"\"\"\n for key in tags.keys():\n self.tags[TAG_RE.sub(\"_\", compat.text_type(key))] = encoding.keyword_field(compat.text_type(tags[key]))"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef begin_transaction(self, transaction_type, trace_parent=None):\n if trace_parent:\n is_sampled = bool(trace_parent.trace_options.recorded)\n else:\n is_sampled = self._sample_rate == 1.0 or self._sample_rate > random.random()\n transaction = Transaction(self, transaction_type, trace_parent=trace_parent, is_sampled=is_sampled)\n if trace_parent is None:\n transaction.trace_parent = TraceParent(\n constants.TRACE_CONTEXT_VERSION,\n \"%032x\" % random.getrandbits(128),\n transaction.id,\n TracingOptions(recorded=is_sampled),\n )\n execution_context.set_transaction(transaction)\n return transaction", "response": "Starts a new transaction and bind it in a thread - local variable"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef handle_check(self, command, **options):\n passed = True\n client = DjangoClient(metrics_interval=\"0ms\")\n\n def is_set(x):\n return x and x != \"None\"\n\n # check if org/app is set:\n if is_set(client.config.service_name):\n self.write(\"Service name is set, good job!\", green)\n else:\n passed = False\n self.write(\"Configuration errors detected!\", red, ending=\"\\n\\n\")\n self.write(\" * SERVICE_NAME not set! \", red, ending=\"\\n\")\n self.write(CONFIG_EXAMPLE)\n\n # secret token is optional but recommended\n if not is_set(client.config.secret_token):\n self.write(\" * optional SECRET_TOKEN not set\", yellow, ending=\"\\n\")\n self.write(\"\")\n\n server_url = client.config.server_url\n if server_url:\n parsed_url = urlparse.urlparse(server_url)\n if parsed_url.scheme.lower() in (\"http\", \"https\"):\n # parse netloc, making sure people did not supply basic auth\n if \"@\" in parsed_url.netloc:\n credentials, _, path = parsed_url.netloc.rpartition(\"@\")\n passed = False\n self.write(\"Configuration errors detected!\", red, ending=\"\\n\\n\")\n if \":\" in credentials:\n self.write(\" * SERVER_URL cannot contain authentication \" \"credentials\", red, ending=\"\\n\")\n else:\n self.write(\n \" * SERVER_URL contains an unexpected at-sign!\"\n \" This is usually used for basic authentication, \"\n \"but the colon is left out\",\n red,\n ending=\"\\n\",\n )\n else:\n self.write(\"SERVER_URL {0} looks fine\".format(server_url), green)\n # secret token in the clear not recommended\n if is_set(client.config.secret_token) and parsed_url.scheme.lower() == \"http\":\n self.write(\" * SECRET_TOKEN set but server not using https\", yellow, ending=\"\\n\")\n else:\n self.write(\n \" * SERVER_URL has scheme {0} and we require \" \"http or https!\".format(parsed_url.scheme),\n red,\n ending=\"\\n\",\n )\n passed = False\n else:\n self.write(\"Configuration errors detected!\", red, ending=\"\\n\\n\")\n self.write(\" * SERVER_URL appears to be empty\", red, ending=\"\\n\")\n passed = False\n self.write(\"\")\n\n # check if we're disabled due to DEBUG:\n if settings.DEBUG:\n if getattr(settings, \"ELASTIC_APM\", {}).get(\"DEBUG\"):\n self.write(\n \"Note: even though you are running in DEBUG mode, we will \"\n 'send data to the APM Server, because you set ELASTIC_APM[\"DEBUG\"] to '\n \"True. You can disable ElasticAPM while in DEBUG mode like this\"\n \"\\n\\n\",\n yellow,\n )\n self.write(\n \" ELASTIC_APM = {\\n\"\n ' \"DEBUG\": False,\\n'\n \" # your other ELASTIC_APM settings\\n\"\n \" }\"\n )\n else:\n self.write(\n \"Looks like you're running in DEBUG mode. ElasticAPM will NOT \"\n \"gather any data while DEBUG is set to True.\\n\\n\",\n red,\n )\n self.write(\n \"If you want to test ElasticAPM while DEBUG is set to True, you\"\n \" can force ElasticAPM to gather data by setting\"\n ' ELASTIC_APM[\"DEBUG\"] to True, like this\\n\\n'\n \" ELASTIC_APM = {\\n\"\n ' \"DEBUG\": True,\\n'\n \" # your other ELASTIC_APM settings\\n\"\n \" }\"\n )\n passed = False\n else:\n self.write(\"DEBUG mode is disabled! Looking good!\", green)\n self.write(\"\")\n\n # check if middleware is set, and if it is at the first position\n middleware_attr = \"MIDDLEWARE\" if getattr(settings, \"MIDDLEWARE\", None) is not None else \"MIDDLEWARE_CLASSES\"\n middleware = list(getattr(settings, middleware_attr))\n try:\n pos = middleware.index(\"elasticapm.contrib.django.middleware.TracingMiddleware\")\n if pos == 0:\n self.write(\"Tracing middleware is configured! Awesome!\", green)\n else:\n self.write(\"Tracing middleware is configured, but not at the first position\\n\", yellow)\n self.write(\"ElasticAPM works best if you add it at the top of your %s setting\" % middleware_attr)\n except ValueError:\n self.write(\"Tracing middleware not configured!\", red)\n self.write(\n \"\\n\"\n \"Add it to your %(name)s setting like this:\\n\\n\"\n \" %(name)s = (\\n\"\n ' \"elasticapm.contrib.django.middleware.TracingMiddleware\",\\n'\n \" # your other middleware classes\\n\"\n \" )\\n\" % {\"name\": middleware_attr}\n )\n self.write(\"\")\n if passed:\n self.write(\"Looks like everything should be ready!\", green)\n else:\n self.write(\"Please fix the above errors.\", red)\n self.write(\"\")\n client.close()\n return passed", "response": "Check if the configuration of a specific service is set and if not print a message to the console."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef write(self, msg, style_func=None, ending=None, stream=None):\n if stream is None:\n stream = self.stdout\n if OutputWrapper is None:\n ending = \"\\n\" if ending is None else ending\n msg += ending\n stream.write(msg)\n else:\n stream.write(msg, style_func=style_func, ending=ending)", "response": "Write a message to the output stream."} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncall when an exception has been raised in the code run by ZeroRPC.", "response": "def server_inspect_exception(self, req_event, rep_event, task_ctx, exc_info):\n \"\"\"Called when an exception has been raised in the code run by ZeroRPC\"\"\"\n\n # Hide the zerorpc internal frames for readability, for a REQ/REP or\n # REQ/STREAM server the frames to hide are:\n # - core.ServerBase._async_task\n # - core.Pattern*.process_call\n # - core.DecoratorBase.__call__\n #\n # For a PUSH/PULL or PUB/SUB server the frame to hide is:\n # - core.Puller._receiver\n if self._hide_zerorpc_frames:\n traceback = exc_info[2]\n while traceback:\n zerorpc_frame = traceback.tb_frame\n zerorpc_frame.f_locals[\"__traceback_hide__\"] = True\n frame_info = inspect.getframeinfo(zerorpc_frame)\n # Is there a better way than this (or looking up the filenames\n # or hardcoding the number of frames to skip) to know when we\n # are out of zerorpc?\n if frame_info.function == \"__call__\" or frame_info.function == \"_receiver\":\n break\n traceback = traceback.tb_next\n\n self._elasticapm_client.capture_exception(exc_info, extra=task_ctx, handled=False)"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef mutate_unsampled_call_args(self, module, method, wrapped, instance, args, kwargs, transaction):\n return args, kwargs", "response": "This method is used to mutate the args and kwargs of unsampled wrapped calls."} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ninstruments the n - grams of the current language.", "response": "def instrument(client):\n \"\"\"\n Auto-instruments code to get nice spans\n \"\"\"\n from elasticapm.instrumentation.control import instrument\n\n instrument()\n try:\n import celery # noqa F401\n from elasticapm.contrib.celery import register_instrumentation\n\n register_instrumentation(client)\n except ImportError:\n client.logger.debug(\"Not instrumenting Celery, couldn't import\")"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nset up logging to pipe to Elastic APM.", "response": "def setup_logging(handler, exclude=(\"gunicorn\", \"south\", \"elasticapm.errors\")):\n \"\"\"\n Configures logging to pipe to Elastic APM.\n\n - ``exclude`` is a list of loggers that shouldn't go to ElasticAPM.\n\n For a typical Python install:\n\n >>> from elasticapm.handlers.logging import LoggingHandler\n >>> client = ElasticAPM(...)\n >>> setup_logging(LoggingHandler(client))\n\n Within Django:\n\n >>> from elasticapm.contrib.django.handlers import LoggingHandler\n >>> setup_logging(LoggingHandler())\n\n Returns a boolean based on if logging was configured or not.\n \"\"\"\n logger = logging.getLogger()\n if handler.__class__ in map(type, logger.handlers):\n return False\n\n logger.addHandler(handler)\n\n return True"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nregister exception handlers with an ElasticAPM client and an RQ worker.", "response": "def register_elasticapm(client, worker):\n \"\"\"Given an ElasticAPM client and an RQ worker, registers exception handlers\n with the worker so exceptions are logged to the apm server.\n\n E.g.:\n\n from elasticapm.contrib.django.models import client\n from elasticapm.contrib.rq import register_elasticapm\n\n worker = Worker(map(Queue, listen))\n register_elasticapm(client, worker)\n worker.work()\n\n \"\"\"\n\n def send_to_server(job, *exc_info):\n client.capture_exception(\n exc_info=exc_info,\n extra={\n \"job_id\": job.id,\n \"func\": job.func_name,\n \"args\": job.args,\n \"kwargs\": job.kwargs,\n \"description\": job.description,\n },\n )\n\n worker.push_exc_handler(send_to_server)"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get_transaction(self, clear=False):\n transaction = getattr(self.thread_local, \"transaction\", None)\n if clear:\n self.thread_local.transaction = None\n return transaction", "response": "Get the transaction registered for the current thread."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_lines_from_file(filename, lineno, context_lines, loader=None, module_name=None):\n lineno = lineno - 1\n lower_bound = max(0, lineno - context_lines)\n upper_bound = lineno + context_lines\n\n source = None\n if loader is not None and hasattr(loader, \"get_source\"):\n result = get_source_lines_from_loader(loader, module_name, lineno, lower_bound, upper_bound)\n if result is not None:\n return result\n\n if source is None:\n try:\n with open(filename, \"rb\") as file_obj:\n encoding = \"utf8\"\n # try to find encoding of source file by \"coding\" header\n # if none is found, utf8 is used as a fallback\n for line in itertools.islice(file_obj, 0, 2):\n match = _coding_re.search(line.decode(\"utf8\"))\n if match:\n encoding = match.group(1)\n break\n file_obj.seek(0)\n lines = [\n compat.text_type(line, encoding, \"replace\")\n for line in itertools.islice(file_obj, lower_bound, upper_bound + 1)\n ]\n offset = lineno - lower_bound\n return (\n [l.strip(\"\\r\\n\") for l in lines[0:offset]],\n lines[offset].strip(\"\\r\\n\"),\n [l.strip(\"\\r\\n\") for l in lines[offset + 1 :]] if len(lines) > offset else [],\n )\n except (OSError, IOError, IndexError):\n pass\n return None, None, None", "response": "Returns a tuple of lines before and after lineno from a file."} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\niterates over all the traceback frames in the traceback object.", "response": "def iter_traceback_frames(tb):\n \"\"\"\n Given a traceback object, it will iterate over all\n frames that do not contain the ``__traceback_hide__``\n local variable.\n \"\"\"\n while tb:\n # support for __traceback_hide__ which is used by a few libraries\n # to hide internal frames.\n frame = tb.tb_frame\n f_locals = getattr(frame, \"f_locals\", {})\n if not _getitem_from_frame(f_locals, \"__traceback_hide__\"):\n yield frame, getattr(tb, \"tb_lineno\", None)\n tb = tb.tb_next"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef iter_stack_frames(frames=None, start_frame=None, skip=0, skip_top_modules=()):\n if not frames:\n frame = start_frame if start_frame is not None else inspect.currentframe().f_back\n frames = _walk_stack(frame)\n stop_ignoring = False\n for i, frame in enumerate(frames):\n if i < skip:\n continue\n f_globals = getattr(frame, \"f_globals\", {})\n if not stop_ignoring and f_globals.get(\"__name__\", \"\").startswith(skip_top_modules):\n continue\n stop_ignoring = True\n f_locals = getattr(frame, \"f_locals\", {})\n if not _getitem_from_frame(f_locals, \"__traceback_hide__\"):\n yield frame, frame.f_lineno", "response": "Iterate over the stack frames."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_stack_info(\n frames,\n with_locals=True,\n library_frame_context_lines=None,\n in_app_frame_context_lines=None,\n include_paths_re=None,\n exclude_paths_re=None,\n locals_processor_func=None,\n):\n \"\"\"\n Given a list of frames, returns a list of stack information\n dictionary objects that are JSON-ready.\n\n We have to be careful here as certain implementations of the\n _Frame class do not contain the necessary data to lookup all\n of the information we want.\n\n :param frames: a list of (Frame, lineno) tuples\n :param with_locals: boolean to indicate if local variables should be collected\n :param include_paths_re: a regex to determine if a frame is not a library frame\n :param exclude_paths_re: a regex to exclude frames from not being library frames\n :param locals_processor_func: a function to call on all local variables\n :return:\n \"\"\"\n results = []\n for frame, lineno in frames:\n result = get_frame_info(\n frame,\n lineno,\n library_frame_context_lines=library_frame_context_lines,\n in_app_frame_context_lines=in_app_frame_context_lines,\n with_locals=with_locals,\n include_paths_re=include_paths_re,\n exclude_paths_re=exclude_paths_re,\n locals_processor_func=locals_processor_func,\n )\n if result:\n results.append(result)\n return results", "response": "Given a list of frames returns a list of stack information dicts that are JSON - ready."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef atexit_register(func):\n try:\n import uwsgi\n\n orig = getattr(uwsgi, \"atexit\", None)\n\n def uwsgi_atexit():\n if callable(orig):\n orig()\n func()\n\n uwsgi.atexit = uwsgi_atexit\n except ImportError:\n atexit.register(func)", "response": "Register a function to be called at exit from uwsgi"} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns a list of glob paths depending on the used platform.", "response": "def get_default_library_patters():\n \"\"\"\n Returns library paths depending on the used platform.\n\n :return: a list of glob paths\n \"\"\"\n python_version = platform.python_version_tuple()\n python_implementation = platform.python_implementation()\n system = platform.system()\n if python_implementation == \"PyPy\":\n if python_version[0] == \"2\":\n return [\"*/lib-python/%s.%s/*\" % python_version[:2], \"*/site-packages/*\"]\n else:\n return [\"*/lib-python/%s/*\" % python_version[0], \"*/site-packages/*\"]\n else:\n if system == \"Windows\":\n return [r\"*\\lib\\*\"]\n return [\"*/lib/python%s.%s/*\" % python_version[:2], \"*/lib64/python%s.%s/*\" % python_version[:2]]"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef multidict_to_dict(d):\n return dict((k, v[0] if len(v) == 1 else v) for k, v in iterlists(d))", "response": "Turn a werkzeug. MultiDict or django. MultiValueDict into a dict with\n list values\n "} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _is_ignorable_404(uri):\n urls = getattr(django_settings, \"IGNORABLE_404_URLS\", ())\n return any(pattern.search(uri) for pattern in urls)", "response": "Returns True if the given request should not notify the site managers."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get_cgroup_container_metadata():\n if not os.path.exists(CGROUP_PATH):\n return {}\n with open(CGROUP_PATH) as f:\n return parse_cgroups(f) or {}", "response": "Reads docker - kubernetes metadata from the cgroups directory"} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreads a file handle and tries to parse kubernetes Container IDs and kubernetes Pod IDs.", "response": "def parse_cgroups(filehandle):\n \"\"\"\n Reads lines from a file handle and tries to parse docker container IDs and kubernetes Pod IDs.\n\n See tests.utils.docker_tests.test_cgroup_parsing for a set of test cases\n\n :param filehandle:\n :return: nested dictionary or None\n \"\"\"\n for line in filehandle:\n parts = line.strip().split(\":\")\n if len(parts) != 3:\n continue\n cgroup_path = parts[2]\n\n # Depending on the filesystem driver used for cgroup\n # management, the paths in /proc/pid/cgroup will have\n # one of the following formats in a Docker container:\n #\n # systemd: /system.slice/docker-.scope\n # cgroupfs: /docker/\n #\n # In a Kubernetes pod, the cgroup path will look like:\n #\n # systemd:\n # /kubepods.slice/kubepods-.slice/kubepods--pod.slice/.scope\n # cgroupfs:\n # /kubepods//pod/\n\n directory, container_id = os.path.split(cgroup_path)\n if container_id.endswith(SYSTEMD_SCOPE_SUFFIX):\n container_id = container_id[: -len(SYSTEMD_SCOPE_SUFFIX)]\n if \"-\" in container_id:\n container_id = container_id.split(\"-\", 1)[1]\n kubepods_match = kubepods_regexp.match(directory)\n if kubepods_match:\n pod_id = kubepods_match.group(1)\n if not pod_id:\n pod_id = kubepods_match.group(2)\n return {\"container\": {\"id\": container_id}, \"kubernetes\": {\"pod\": {\"uid\": pod_id}}}\n elif container_id_regexp.match(container_id):\n return {\"container\": {\"id\": container_id}}"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef remove_stacktrace_locals(client, event):\n func = lambda frame: frame.pop(\"vars\", None)\n return _process_stack_frames(event, func)", "response": "Removes local variables from any frames in the event."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef sanitize_stacktrace_locals(client, event):\n\n def func(frame):\n if \"vars\" in frame:\n frame[\"vars\"] = varmap(_sanitize, frame[\"vars\"])\n\n return _process_stack_frames(event, func)", "response": "Sanitize local variables in all frames in the event."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef sanitize_http_request_cookies(client, event):\n\n # sanitize request.cookies dict\n try:\n cookies = event[\"context\"][\"request\"][\"cookies\"]\n event[\"context\"][\"request\"][\"cookies\"] = varmap(_sanitize, cookies)\n except (KeyError, TypeError):\n pass\n\n # sanitize request.header.cookie string\n try:\n cookie_string = event[\"context\"][\"request\"][\"headers\"][\"cookie\"]\n event[\"context\"][\"request\"][\"headers\"][\"cookie\"] = _sanitize_string(cookie_string, \"; \", \"=\")\n except (KeyError, TypeError):\n pass\n return event", "response": "Sanitize the http request cookies dict and header. cookie string."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nsanitize the set - cookie header of the response object.", "response": "def sanitize_http_response_cookies(client, event):\n \"\"\"\n Sanitizes the set-cookie header of the response\n :param client: an ElasticAPM client\n :param event: a transaction or error event\n :return: The modified event\n \"\"\"\n try:\n cookie_string = event[\"context\"][\"response\"][\"headers\"][\"set-cookie\"]\n event[\"context\"][\"response\"][\"headers\"][\"set-cookie\"] = _sanitize_string(cookie_string, \";\", \"=\")\n except (KeyError, TypeError):\n pass\n return event"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef sanitize_http_headers(client, event):\n # request headers\n try:\n headers = event[\"context\"][\"request\"][\"headers\"]\n event[\"context\"][\"request\"][\"headers\"] = varmap(_sanitize, headers)\n except (KeyError, TypeError):\n pass\n\n # response headers\n try:\n headers = event[\"context\"][\"response\"][\"headers\"]\n event[\"context\"][\"response\"][\"headers\"] = varmap(_sanitize, headers)\n except (KeyError, TypeError):\n pass\n\n return event", "response": "Sanitize the http request and response headers."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nsanitizing the WSGI environment variables in an ElasticAPM event.", "response": "def sanitize_http_wsgi_env(client, event):\n \"\"\"\n Sanitizes WSGI environment variables\n\n :param client: an ElasticAPM client\n :param event: a transaction or error event\n :return: The modified event\n \"\"\"\n try:\n env = event[\"context\"][\"request\"][\"env\"]\n event[\"context\"][\"request\"][\"env\"] = varmap(_sanitize, env)\n except (KeyError, TypeError):\n pass\n return event"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nsanitizing the http request query string and return the modified event", "response": "def sanitize_http_request_querystring(client, event):\n \"\"\"\n Sanitizes http request query string\n\n :param client: an ElasticAPM client\n :param event: a transaction or error event\n :return: The modified event\n \"\"\"\n try:\n query_string = force_text(event[\"context\"][\"request\"][\"url\"][\"search\"], errors=\"replace\")\n except (KeyError, TypeError):\n return event\n if \"=\" in query_string:\n sanitized_query_string = _sanitize_string(query_string, \"&\", \"=\")\n full_url = event[\"context\"][\"request\"][\"url\"][\"full\"]\n event[\"context\"][\"request\"][\"url\"][\"search\"] = sanitized_query_string\n event[\"context\"][\"request\"][\"url\"][\"full\"] = full_url.replace(query_string, sanitized_query_string)\n return event"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef sanitize_http_request_body(client, event):\n try:\n body = force_text(event[\"context\"][\"request\"][\"body\"], errors=\"replace\")\n except (KeyError, TypeError):\n return event\n if \"=\" in body:\n sanitized_query_string = _sanitize_string(body, \"&\", \"=\")\n event[\"context\"][\"request\"][\"body\"] = sanitized_query_string\n return event", "response": "Sanitize the http request body."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _sanitize_string(unsanitized, itemsep, kvsep):\n sanitized = []\n kvs = unsanitized.split(itemsep)\n for kv in kvs:\n kv = kv.split(kvsep)\n if len(kv) == 2:\n sanitized.append((kv[0], _sanitize(kv[0], kv[1])))\n else:\n sanitized.append(kv)\n return itemsep.join(kvsep.join(kv) for kv in sanitized)", "response": "sanitizes a string that contains multiple key value items and returns a sanitized version of the unsanitized string"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncapturing and processes an event and pipes it off to Client. send.", "response": "def capture(self, event_type, date=None, context=None, custom=None, stack=None, handled=True, **kwargs):\n \"\"\"\n Captures and processes an event and pipes it off to Client.send.\n \"\"\"\n if event_type == \"Exception\":\n # never gather log stack for exceptions\n stack = False\n data = self._build_msg_for_logging(\n event_type, date=date, context=context, custom=custom, stack=stack, handled=handled, **kwargs\n )\n\n if data:\n # queue data, and flush the queue if this is an unhandled exception\n self.queue(ERROR, data, flush=not handled)\n return data[\"id\"]"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncreates an event from a message.", "response": "def capture_message(self, message=None, param_message=None, **kwargs):\n \"\"\"\n Creates an event from ``message``.\n\n >>> client.capture_message('My event just happened!')\n \"\"\"\n return self.capture(\"Message\", message=message, param_message=param_message, **kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncreate an event from an exception.", "response": "def capture_exception(self, exc_info=None, handled=True, **kwargs):\n \"\"\"\n Creates an event from an exception.\n\n >>> try:\n >>> exc_info = sys.exc_info()\n >>> client.capture_exception(exc_info)\n >>> finally:\n >>> del exc_info\n\n If exc_info is not provided, or is set to True, then this method will\n perform the ``exc_info = sys.exc_info()`` and the requisite clean-up\n for you.\n \"\"\"\n return self.capture(\"Exception\", exc_info=exc_info, handled=handled, **kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nregister the start of a transaction on the client", "response": "def begin_transaction(self, transaction_type, trace_parent=None):\n \"\"\"Register the start of a transaction on the client\n \"\"\"\n return self.tracer.begin_transaction(transaction_type, trace_parent=trace_parent)"} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nbuilds a message for logging.", "response": "def _build_msg_for_logging(\n self, event_type, date=None, context=None, custom=None, stack=None, handled=True, **kwargs\n ):\n \"\"\"\n Captures, processes and serializes an event into a dict object\n \"\"\"\n transaction = execution_context.get_transaction()\n if transaction:\n transaction_context = deepcopy(transaction.context)\n else:\n transaction_context = {}\n event_data = {}\n if custom is None:\n custom = {}\n if date is not None:\n warnings.warn(\n \"The date argument is no longer evaluated and will be removed in a future release\", DeprecationWarning\n )\n date = time.time()\n if stack is None:\n stack = self.config.auto_log_stacks\n if context:\n transaction_context.update(context)\n context = transaction_context\n else:\n context = transaction_context\n event_data[\"context\"] = context\n if transaction and transaction.tags:\n context[\"tags\"] = deepcopy(transaction.tags)\n\n # if '.' not in event_type:\n # Assume it's a builtin\n event_type = \"elasticapm.events.%s\" % event_type\n\n handler = self.get_handler(event_type)\n result = handler.capture(self, **kwargs)\n if self._filter_exception_type(result):\n return\n # data (explicit) culprit takes over auto event detection\n culprit = result.pop(\"culprit\", None)\n if custom.get(\"culprit\"):\n culprit = custom.pop(\"culprit\")\n\n for k, v in compat.iteritems(result):\n if k not in event_data:\n event_data[k] = v\n\n log = event_data.get(\"log\", {})\n if stack and \"stacktrace\" not in log:\n if stack is True:\n frames = stacks.iter_stack_frames(skip=3)\n else:\n frames = stack\n frames = stacks.get_stack_info(\n frames,\n with_locals=self.config.collect_local_variables in (\"errors\", \"all\"),\n library_frame_context_lines=self.config.source_lines_error_library_frames,\n in_app_frame_context_lines=self.config.source_lines_error_app_frames,\n include_paths_re=self.include_paths_re,\n exclude_paths_re=self.exclude_paths_re,\n locals_processor_func=lambda local_var: varmap(\n lambda k, v: shorten(\n v,\n list_length=self.config.local_var_list_max_length,\n string_length=self.config.local_var_max_length,\n ),\n local_var,\n ),\n )\n log[\"stacktrace\"] = frames\n\n if \"stacktrace\" in log and not culprit:\n culprit = stacks.get_culprit(log[\"stacktrace\"], self.config.include_paths, self.config.exclude_paths)\n\n if \"level\" in log and isinstance(log[\"level\"], compat.integer_types):\n log[\"level\"] = logging.getLevelName(log[\"level\"]).lower()\n\n if log:\n event_data[\"log\"] = log\n\n if culprit:\n event_data[\"culprit\"] = culprit\n\n if \"custom\" in context:\n context[\"custom\"].update(custom)\n else:\n context[\"custom\"] = custom\n\n # Make sure all data is coerced\n event_data = transform(event_data)\n if \"exception\" in event_data:\n event_data[\"exception\"][\"handled\"] = bool(handled)\n\n event_data[\"timestamp\"] = int(date * 1000000)\n\n if transaction:\n if transaction.trace_parent:\n event_data[\"trace_id\"] = transaction.trace_parent.trace_id\n event_data[\"parent_id\"] = transaction.id\n event_data[\"transaction_id\"] = transaction.id\n event_data[\"transaction\"] = {\"sampled\": transaction.is_sampled, \"type\": transaction.transaction_type}\n\n return event_data"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef papermill(\n notebook_path,\n output_path,\n parameters,\n parameters_raw,\n parameters_file,\n parameters_yaml,\n parameters_base64,\n inject_input_path,\n inject_output_path,\n inject_paths,\n engine,\n prepare_only,\n kernel,\n cwd,\n progress_bar,\n log_output,\n log_level,\n start_timeout,\n report_mode,\n):\n \"\"\"This utility executes a single notebook on a container.\n\n Papermill takes a source notebook, applies parameters to the source\n notebook, executes the notebook with the specified kernel, and saves the\n output in the destination notebook.\n\n \"\"\"\n logging.basicConfig(level=log_level, format=\"%(message)s\")\n\n if progress_bar is None:\n progress_bar = not log_output\n\n # Read in Parameters\n parameters_final = {}\n if inject_input_path or inject_paths:\n parameters_final['PAPERMILL_INPUT_PATH'] = notebook_path\n if inject_output_path or inject_paths:\n parameters_final['PAPERMILL_OUTPUT_PATH'] = output_path\n for params in parameters_base64 or []:\n parameters_final.update(yaml.load(base64.b64decode(params)))\n for files in parameters_file or []:\n parameters_final.update(read_yaml_file(files))\n for params in parameters_yaml or []:\n parameters_final.update(yaml.load(params))\n for name, value in parameters or []:\n parameters_final[name] = _resolve_type(value)\n for name, value in parameters_raw or []:\n parameters_final[name] = value\n\n execute_notebook(\n notebook_path,\n output_path,\n parameters_final,\n engine_name=engine,\n prepare_only=prepare_only,\n kernel_name=kernel,\n progress_bar=progress_bar,\n log_output=log_output,\n start_timeout=start_timeout,\n report_mode=report_mode,\n cwd=cwd,\n )", "response": "This utility executes a single notebook on a container."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nwrap to catch `nb` keyword arguments This helps catch `nb` keyword arguments and assign onto self when passed to the wrapped function. Used for callback methods when the caller may optionally have a new copy of the originally wrapped `nb` object.", "response": "def catch_nb_assignment(func):\n \"\"\"\n Wrapper to catch `nb` keyword arguments\n\n This helps catch `nb` keyword arguments and assign onto self when passed to\n the wrapped function.\n\n Used for callback methods when the caller may optionally have a new copy\n of the originally wrapped `nb` object.\n \"\"\"\n\n @wraps(func)\n def wrapper(self, *args, **kwargs):\n nb = kwargs.get('nb')\n if nb:\n # Reassign if executing notebook object was replaced\n self.nb = nb\n return func(self, *args, **kwargs)\n\n return wrapper"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nregistering entrypoints for an engine Load handlers provided by other packages", "response": "def register_entry_points(self):\n \"\"\"Register entrypoints for an engine\n\n Load handlers provided by other packages\n \"\"\"\n for entrypoint in entrypoints.get_group_all(\"papermill.engine\"):\n self.register(entrypoint.name, entrypoint.load())"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nretrieving an engine by name.", "response": "def get_engine(self, name=None):\n \"\"\"Retrieves an engine by name.\"\"\"\n engine = self._engines.get(name)\n if not engine:\n raise PapermillException(\"No engine named '{}' found\".format(name))\n return engine"} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nfetch a named engine and execute the nb object against it.", "response": "def execute_notebook_with_engine(self, engine_name, nb, kernel_name, **kwargs):\n \"\"\"Fetch a named engine and execute the nb object against it.\"\"\"\n return self.get_engine(engine_name).execute_notebook(nb, kernel_name, **kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef save(self, **kwargs):\n if self.output_path:\n write_ipynb(self.nb, self.output_path)", "response": "Saves the wrapped notebook state to the output path."} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ninitializes a notebook and clear the metadata for all cells and save it.", "response": "def notebook_start(self, **kwargs):\n \"\"\"\n Initialize a notebook, clearing its metadata, and save it.\n\n When starting a notebook, this initializes and clears the metadata for\n the notebook and its cells, and saves the notebook to the given\n output path.\n\n Called by Engine when execution begins.\n \"\"\"\n self.set_timer()\n\n self.nb.metadata.papermill['start_time'] = self.start_time.isoformat()\n self.nb.metadata.papermill['end_time'] = None\n self.nb.metadata.papermill['duration'] = None\n self.nb.metadata.papermill['exception'] = None\n\n for cell in self.nb.cells:\n # Reset the cell execution counts.\n if cell.get(\"execution_count\") is not None:\n cell.execution_count = None\n\n # Clear out the papermill metadata for each cell.\n cell.metadata.papermill = dict(\n exception=None,\n start_time=None,\n end_time=None,\n duration=None,\n status=self.PENDING, # pending, running, completed\n )\n if cell.get(\"outputs\") is not None:\n cell.outputs = []\n\n self.save()"} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nsets and save a cell s start state.", "response": "def cell_start(self, cell, cell_index=None, **kwargs):\n \"\"\"\n Set and save a cell's start state.\n\n Optionally called by engines during execution to initialize the\n metadata for a cell and save the notebook to the output path.\n \"\"\"\n if self.log_output:\n ceel_num = cell_index + 1 if cell_index is not None else ''\n logger.info('Executing Cell {:-<40}'.format(ceel_num))\n\n cell.metadata.papermill['start_time'] = self.now().isoformat()\n cell.metadata.papermill[\"status\"] = self.RUNNING\n cell.metadata.papermill['exception'] = False\n\n self.save()"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef cell_exception(self, cell, cell_index=None, **kwargs):\n cell.metadata.papermill['exception'] = True\n cell.metadata.papermill['status'] = self.FAILED\n self.nb.metadata.papermill['exception'] = True", "response": "Set metadata when an exception is raised within a notebook cell."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nfinalize the metadata for a cell and saves the notebook.", "response": "def cell_complete(self, cell, cell_index=None, **kwargs):\n \"\"\"\n Finalize metadata for a cell and save notebook.\n\n Optionally called by engines during execution to finalize the\n metadata for a cell and save the notebook to the output path.\n \"\"\"\n end_time = self.now()\n\n if self.log_output:\n ceel_num = cell_index + 1 if cell_index is not None else ''\n logger.info('Ending Cell {:-<43}'.format(ceel_num))\n # Ensure our last cell messages are not buffered by python\n sys.stdout.flush()\n sys.stderr.flush()\n\n cell.metadata.papermill['end_time'] = end_time.isoformat()\n if cell.metadata.papermill.get('start_time'):\n start_time = dateutil.parser.parse(cell.metadata.papermill['start_time'])\n cell.metadata.papermill['duration'] = (end_time - start_time).total_seconds()\n if cell.metadata.papermill['status'] != self.FAILED:\n cell.metadata.papermill['status'] = self.COMPLETED\n\n self.save()\n if self.pbar:\n self.pbar.update(1)"} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nfinalizes the notebook metadata and save the notebook to the output path.", "response": "def notebook_complete(self, **kwargs):\n \"\"\"\n Finalize the metadata for a notebook and save the notebook to\n the output path.\n\n Called by Engine when execution concludes, regardless of exceptions.\n \"\"\"\n self.end_time = self.now()\n self.nb.metadata.papermill['end_time'] = self.end_time.isoformat()\n if self.nb.metadata.papermill.get('start_time'):\n self.nb.metadata.papermill['duration'] = (\n self.end_time - self.start_time\n ).total_seconds()\n\n # Cleanup cell statuses in case callbacks were never called\n for cell in self.nb.cells:\n if cell.metadata.papermill['status'] == self.FAILED:\n break\n elif cell.metadata.papermill['status'] == self.PENDING:\n cell.metadata.papermill['status'] = self.COMPLETED\n\n self.complete_pbar()\n self.cleanup_pbar()\n\n # Force a final sync\n self.save()"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef cleanup_pbar(self):\n if hasattr(self, 'pbar') and self.pbar:\n self.pbar.close()\n self.pbar = None", "response": "Clean up a progress bar"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef execute_notebook(\n cls, nb, kernel_name, output_path=None, progress_bar=True, log_output=False, **kwargs\n ):\n \"\"\"\n A wrapper to handle notebook execution tasks.\n\n Wraps the notebook object in a `NotebookExecutionManager` in order to track\n execution state in a uniform manner. This is meant to help simplify\n engine implementations. This allows a developer to just focus on\n iterating and executing the cell contents.\n \"\"\"\n nb_man = NotebookExecutionManager(\n nb, output_path=output_path, progress_bar=progress_bar, log_output=log_output\n )\n\n nb_man.notebook_start()\n try:\n nb = cls.execute_managed_notebook(nb_man, kernel_name, log_output=log_output, **kwargs)\n # Update the notebook object in case the executor didn't do it for us\n if nb:\n nb_man.nb = nb\n finally:\n nb_man.cleanup_pbar()\n nb_man.notebook_complete()\n\n return nb_man.nb", "response": "A wrapper to handle notebook execution tasks."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nexecute the managed notebook locally.", "response": "def execute_managed_notebook(\n cls,\n nb_man,\n kernel_name,\n log_output=False,\n start_timeout=60,\n execution_timeout=None,\n **kwargs\n ):\n \"\"\"\n Performs the actual execution of the parameterized notebook locally.\n\n Args:\n nb (NotebookNode): Executable notebook object.\n kernel_name (str): Name of kernel to execute the notebook against.\n log_output (bool): Flag for whether or not to write notebook output to stderr.\n start_timeout (int): Duration to wait for kernel start-up.\n execution_timeout (int): Duration to wait before failing execution (default: never).\n\n\n Note: The preprocessor concept in this method is similar to what is used\n by `nbconvert`, and it is somewhat misleading here. The preprocesser\n represents a notebook processor, not a preparation object.\n \"\"\"\n preprocessor = PapermillExecutePreprocessor(\n timeout=execution_timeout,\n startup_timeout=start_timeout,\n kernel_name=kernel_name,\n log=logger,\n )\n preprocessor.log_output = log_output\n preprocessor.preprocess(nb_man, kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _split_url(self, url):\n match = re.match(r\"abs://(.*)\\.blob\\.core\\.windows\\.net\\/(.*)\\/(.*)\\?(.*)$\", url)\n if not match:\n raise Exception(\"Invalid azure blob url '{0}'\".format(url))\n else:\n params = {\n \"account\": match.group(1),\n \"container\": match.group(2),\n \"blob\": match.group(3),\n \"sas_token\": urllib.parse.unquote_plus(match.group(4)),\n }\n return params", "response": "Split the url into parameters"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef read(self, url):\n params = self._split_url(url)\n output_stream = io.BytesIO()\n block_blob_service = self._block_blob_service(\n account_name=params[\"account\"], sas_token=params[\"sas_token\"]\n )\n\n block_blob_service.get_blob_to_stream(\n container_name=params[\"container\"], blob_name=params[\"blob\"], stream=output_stream\n )\n\n output_stream.seek(0)\n return [line.decode(\"utf-8\") for line in output_stream]", "response": "Read storage at a given url"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef listdir(self, url):\n params = self._split_url(url)\n\n block_blob_service = self._block_blob_service(\n account_name=params[\"account\"], sas_token=params[\"sas_token\"]\n )\n blobs = block_blob_service.list_blobs(params[\"container\"])\n return blobs", "response": "Returns a list of the files under the specified path"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef write(self, buf, url):\n params = self._split_url(url)\n\n block_blob_service = self._block_blob_service(\n account_name=params[\"account\"], sas_token=params[\"sas_token\"]\n )\n\n block_blob_service.create_blob_from_text(\n container_name=params[\"container\"], blob_name=params[\"blob\"], text=buf\n )", "response": "Write buffer to storage at a given url"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nexecute a single notebook locally.", "response": "def execute_notebook(\n input_path,\n output_path,\n parameters=None,\n engine_name=None,\n prepare_only=False,\n kernel_name=None,\n progress_bar=True,\n log_output=False,\n start_timeout=60,\n report_mode=False,\n cwd=None,\n):\n \"\"\"Executes a single notebook locally.\n\n Parameters\n ----------\n input_path : str\n Path to input notebook\n output_path : str\n Path to save executed notebook\n parameters : dict, optional\n Arbitrary keyword arguments to pass to the notebook parameters\n engine_name : str, optional\n Name of execution engine to use\n prepare_only : bool, optional\n Flag to determine if execution should occur or not\n kernel_name : str, optional\n Name of kernel to execute the notebook against\n progress_bar : bool, optional\n Flag for whether or not to show the progress bar.\n log_output : bool, optional\n Flag for whether or not to write notebook output_path to `stderr`\n start_timeout : int, optional\n Duration in seconds to wait for kernel start-up\n report_mode : bool, optional\n Flag for whether or not to hide input.\n cwd : str, optional\n Working directory to use when executing the notebook\n\n Returns\n -------\n nb : NotebookNode\n Executed notebook object\n \"\"\"\n path_parameters = add_builtin_parameters(parameters)\n input_path = parameterize_path(input_path, path_parameters)\n output_path = parameterize_path(output_path, path_parameters)\n\n logger.info(\"Input Notebook: %s\" % get_pretty_path(input_path))\n logger.info(\"Output Notebook: %s\" % get_pretty_path(output_path))\n with local_file_io_cwd():\n if cwd is not None:\n logger.info(\"Working directory: {}\".format(get_pretty_path(cwd)))\n\n nb = load_notebook_node(input_path)\n\n # Parameterize the Notebook.\n if parameters:\n nb = parameterize_notebook(nb, parameters, report_mode)\n\n nb = prepare_notebook_metadata(nb, input_path, output_path, report_mode)\n\n if not prepare_only:\n # Fetch the kernel name if it's not supplied\n kernel_name = kernel_name or nb.metadata.kernelspec.name\n\n # Execute the Notebook in `cwd` if it is set\n with chdir(cwd):\n nb = papermill_engines.execute_notebook_with_engine(\n engine_name,\n nb,\n input_path=input_path,\n output_path=output_path,\n kernel_name=kernel_name,\n progress_bar=progress_bar,\n log_output=log_output,\n start_timeout=start_timeout,\n )\n\n # Check for errors first (it saves on error before raising)\n raise_for_execution_errors(nb, output_path)\n\n # Write final output in case the engine didn't write it on cell completion.\n write_ipynb(nb, output_path)\n\n return nb"} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nprepares notebook metadata associated with a notebook and its cells.", "response": "def prepare_notebook_metadata(nb, input_path, output_path, report_mode=False):\n \"\"\"Prepare metadata associated with a notebook and its cells\n\n Parameters\n ----------\n nb : NotebookNode\n Executable notebook object\n input_path : str\n Path to input notebook\n output_path : str\n Path to write executed notebook\n report_mode : bool, optional\n Flag to set report mode\n \"\"\"\n # Copy the nb object to avoid polluting the input\n nb = copy.deepcopy(nb)\n\n # Hide input if report-mode is set to True.\n if report_mode:\n for cell in nb.cells:\n if cell.cell_type == 'code':\n cell.metadata['jupyter'] = cell.get('jupyter', {})\n cell.metadata['jupyter']['source_hidden'] = True\n\n # Record specified environment variable values.\n nb.metadata.papermill['input_path'] = input_path\n nb.metadata.papermill['output_path'] = output_path\n\n return nb"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nraising an error if any of the output_path is not a valid notebook.", "response": "def raise_for_execution_errors(nb, output_path):\n \"\"\"Assigned parameters into the appropriate place in the input notebook\n\n Parameters\n ----------\n nb : NotebookNode\n Executable notebook object\n output_path : str\n Path to write executed notebook\n \"\"\"\n error = None\n for cell in nb.cells:\n if cell.get(\"outputs\") is None:\n continue\n\n for output in cell.outputs:\n if output.output_type == \"error\":\n error = PapermillExecutionError(\n exec_count=cell.execution_count,\n source=cell.source,\n ename=output.ename,\n evalue=output.evalue,\n traceback=output.traceback,\n )\n break\n\n if error:\n # Write notebook back out with the Error Message at the top of the Notebook.\n error_msg = ERROR_MESSAGE_TEMPLATE % str(error.exec_count)\n error_msg_cell = nbformat.v4.new_code_cell(\n source=\"%%html\\n\" + error_msg,\n outputs=[\n nbformat.v4.new_output(output_type=\"display_data\", data={\"text/html\": error_msg})\n ],\n metadata={\"inputHidden\": True, \"hide_input\": True},\n )\n nb.cells = [error_msg_cell] + nb.cells\n write_ipynb(nb, output_path)\n raise error"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nlimiting a list of Bucket s objects based on prefix and delimiter.", "response": "def list(self, prefix='', delimiter=None):\n \"\"\"Limits a list of Bucket's objects based on prefix and delimiter.\"\"\"\n return self.service._list(\n bucket=self.name, prefix=prefix, delimiter=delimiter, objects=True\n )"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef cat(\n self,\n source,\n buffersize=None,\n memsize=2 ** 24,\n compressed=False,\n encoding='UTF-8',\n raw=False,\n ):\n \"\"\"\n Returns an iterator for the data in the key or nothing if the key\n doesn't exist. Decompresses data on the fly (if compressed is True\n or key ends with .gz) unless raw is True. Pass None for encoding to\n skip encoding.\n\n \"\"\"\n assert self._is_s3(source) or isinstance(source, Key), 'source must be a valid s3 path'\n\n key = self._get_key(source) if not isinstance(source, Key) else source\n compressed = (compressed or key.name.endswith('.gz')) and not raw\n if compressed:\n decompress = zlib.decompressobj(16 + zlib.MAX_WBITS)\n\n size = 0\n bytes_read = 0\n err = None\n undecoded = ''\n if key:\n # try to read the file multiple times\n for i in range(100):\n obj = self.s3.Object(key.bucket.name, key.name)\n buffersize = buffersize if buffersize is not None else 2 ** 20\n\n if not size:\n size = obj.content_length\n elif size != obj.content_length:\n raise AwsError('key size unexpectedly changed while reading')\n\n r = obj.get(Range=\"bytes={}-\".format(bytes_read))\n\n try:\n while bytes_read < size:\n # this making this weird check because this call is\n # about 100 times slower if the amt is too high\n if size - bytes_read > buffersize:\n bytes = r['Body'].read(amt=buffersize)\n else:\n bytes = r['Body'].read()\n if compressed:\n s = decompress.decompress(bytes)\n else:\n s = bytes\n\n if encoding and not raw:\n try:\n decoded = undecoded + s.decode(encoding)\n undecoded = ''\n yield decoded\n except UnicodeDecodeError:\n undecoded += s\n if len(undecoded) > memsize:\n raise\n else:\n yield s\n\n bytes_read += len(bytes)\n\n except zlib.error:\n logger.error(\"Error while decompressing [%s]\", key.name)\n raise\n except UnicodeDecodeError:\n raise\n except Exception:\n err = True\n pass\n\n if size <= bytes_read:\n break\n\n if size != bytes_read:\n if err:\n raise Exception\n else:\n raise AwsError('Failed to fully read [%s]' % source.name)\n\n if undecoded:\n assert encoding is not None # only time undecoded is set\n\n # allow exception to be raised if one is thrown\n decoded = undecoded.decode(encoding)\n yield decoded", "response": "Returns an iterator for the data in the key."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef cp_string(self, source, dest, **kwargs):\n\n assert isinstance(source, six.string_types), \"source must be a string\"\n assert self._is_s3(dest), \"Destination must be s3 location\"\n\n return self._put_string(source, dest, **kwargs)", "response": "Copies source string into destination location."} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns a list of the files under the specified path", "response": "def list(self, name, iterator=False, **kwargs):\n \"\"\"\n Returns a list of the files under the specified path\n name must be in the form of `s3://bucket/prefix`\n\n Parameters\n ----------\n keys: optional\n if True then this will return the actual boto keys for files\n that are encountered\n objects: optional\n if True then this will return the actual boto objects for\n files or prefixes that are encountered\n delimiter: optional\n if set this\n iterator: optional\n if True return iterator rather than converting to list object\n\n \"\"\"\n assert self._is_s3(name), \"name must be in form s3://bucket/key\"\n\n it = self._list(bucket=self._bucket_name(name), prefix=self._key_name(name), **kwargs)\n return iter(it) if iterator else list(it)"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef listdir(self, name, **kwargs):\n assert self._is_s3(name), \"name must be in form s3://bucket/prefix/\"\n\n if not name.endswith('/'):\n name += \"/\"\n return self.list(name, delimiter='/', **kwargs)", "response": "Returns a list of the files under the specified path."} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreads a file in s3 and yields a list of the entries in the file.", "response": "def read(self, source, compressed=False, encoding='UTF-8'):\n \"\"\"\n Iterates over a file in s3 split on newline.\n\n Yields a line in file.\n\n \"\"\"\n buf = ''\n for block in self.cat(source, compressed=compressed, encoding=encoding):\n buf += block\n if '\\n' in buf:\n ret, buf = buf.rsplit('\\n', 1)\n for line in ret.split('\\n'):\n yield line\n\n lines = buf.split('\\n')\n for line in lines[:-1]:\n yield line\n\n # only yield the last line if the line has content in it\n if lines[-1]:\n yield lines[-1]"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef translate_escaped_str(cls, str_val):\n if isinstance(str_val, string_types):\n str_val = str_val.encode('unicode_escape')\n if sys.version_info >= (3, 0):\n str_val = str_val.decode('utf-8')\n str_val = str_val.replace('\"', r'\\\"')\n return '\"{}\"'.format(str_val)", "response": "Reusable by most interpreters"} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ntranslates each of the standard json / yaml types to appropiate objects.", "response": "def translate(cls, val):\n \"\"\"Translate each of the standard json/yaml types to appropiate objects.\"\"\"\n if val is None:\n return cls.translate_none(val)\n elif isinstance(val, string_types):\n return cls.translate_str(val)\n # Needs to be before integer checks\n elif isinstance(val, bool):\n return cls.translate_bool(val)\n elif isinstance(val, integer_types):\n return cls.translate_int(val)\n elif isinstance(val, float):\n return cls.translate_float(val)\n elif isinstance(val, dict):\n return cls.translate_dict(val)\n elif isinstance(val, list):\n return cls.translate_list(val)\n # Use this generic translation as a last resort\n return cls.translate_escaped_str(val)"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef translate_dict(cls, val):\n escaped = ', '.join(\n [\"{} -> {}\".format(cls.translate_str(k), cls.translate(v)) for k, v in val.items()]\n )\n return 'Map({})'.format(escaped)", "response": "Translate dicts to scala Maps"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef translate_list(cls, val):\n escaped = ', '.join([cls.translate(v) for v in val])\n return 'Seq({})'.format(escaped)", "response": "Translate list to scala Seq"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef load_notebook_node(notebook_path):\n nb = nbformat.reads(papermill_io.read(notebook_path), as_version=4)\n\n if not hasattr(nb.metadata, 'papermill'):\n nb.metadata['papermill'] = {\n 'parameters': dict(),\n 'environment_variables': dict(),\n 'version': __version__,\n }\n\n for cell in nb.cells:\n if not hasattr(cell.metadata, 'tags'):\n cell.metadata['tags'] = [] # Create tags attr if one doesn't exist.\n\n if not hasattr(cell.metadata, 'papermill'):\n cell.metadata['papermill'] = dict()\n return nb", "response": "Returns a notebook object with papermill metadata loaded from the specified path."} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nset the cwd during reads and writes", "response": "def cwd(self, new_path):\n '''Sets the cwd during reads and writes'''\n old_cwd = self._cwd\n self._cwd = new_path\n return old_cwd"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning a list of the files under the specified path", "response": "def listdir(self, url):\n \"\"\"Returns a list of the files under the specified path\"\"\"\n (store_name, path) = self._split_url(url)\n adapter = self._create_adapter(store_name)\n return [\n \"adl://{store_name}.azuredatalakestore.net/{path_to_child}\".format(\n store_name=store_name, path_to_child=path_to_child\n )\n for path_to_child in adapter.ls(path)\n ]"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreading a given url and return a list of all the items in the file.", "response": "def read(self, url):\n \"\"\"Read storage at a given url\"\"\"\n (store_name, path) = self._split_url(url)\n adapter = self._create_adapter(store_name)\n lines = []\n with adapter.open(path) as f:\n for line in f:\n lines.append(line.decode())\n return lines"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef write(self, buf, url):\n (store_name, path) = self._split_url(url)\n adapter = self._create_adapter(store_name)\n with adapter.open(path, 'wb') as f:\n f.write(buf.encode())", "response": "Write buffer to storage at a given url"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef preprocess(self, nb_man, resources, km=None):\n with self.setup_preprocessor(nb_man.nb, resources, km=km):\n if self.log_output:\n self.log.info(\"Executing notebook with kernel: {}\".format(self.kernel_name))\n nb, resources = self.papermill_process(nb_man, resources)\n info_msg = self._wait_for_reply(self.kc.kernel_info())\n nb.metadata['language_info'] = info_msg['content']['language_info']\n self.set_widgets_metadata()\n\n return nb, resources", "response": "This method is used to prepare the notebook with the kernel process."} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nadds built - in parameters to a dictionary of parameters-----------", "response": "def add_builtin_parameters(parameters):\n \"\"\"Add built-in parameters to a dictionary of parameters\n\n Parameters\n ----------\n parameters : dict\n Dictionary of parameters provided by the user\n \"\"\"\n with_builtin_parameters = {\n \"pm\": {\n \"run_uuid\": str(uuid4()),\n \"current_datetime_local\": datetime.now(),\n \"current_datetime_utc\": datetime.utcnow(),\n }\n }\n\n if parameters is not None:\n with_builtin_parameters.update(parameters)\n\n return with_builtin_parameters"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef parameterize_path(path, parameters):\n if parameters is None:\n parameters = {}\n\n try:\n return path.format(**parameters)\n except KeyError as key_error:\n raise PapermillMissingParameterException(\"Missing parameter {}\".format(key_error))", "response": "Format a path with optional parameters as a python format"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef parameterize_notebook(nb, parameters, report_mode=False):\n # Load from a file if 'parameters' is a string.\n if isinstance(parameters, six.string_types):\n parameters = read_yaml_file(parameters)\n\n # Copy the nb object to avoid polluting the input\n nb = copy.deepcopy(nb)\n\n kernel_name = nb.metadata.kernelspec.name\n language = nb.metadata.kernelspec.language\n\n # Generate parameter content based on the kernel_name\n param_content = translate_parameters(kernel_name, language, parameters)\n\n newcell = nbformat.v4.new_code_cell(source=param_content)\n newcell.metadata['tags'] = ['injected-parameters']\n\n if report_mode:\n newcell.metadata['jupyter'] = newcell.get('jupyter', {})\n newcell.metadata['jupyter']['source_hidden'] = True\n\n param_cell_index = _find_first_tagged_cell_index(nb, 'parameters')\n injected_cell_index = _find_first_tagged_cell_index(nb, 'injected-parameters')\n if injected_cell_index >= 0:\n # Replace the injected cell with a new version\n before = nb.cells[:injected_cell_index]\n after = nb.cells[injected_cell_index + 1 :]\n elif param_cell_index >= 0:\n # Add an injected cell after the parameter cell\n before = nb.cells[: param_cell_index + 1]\n after = nb.cells[param_cell_index + 1 :]\n else:\n # Inject to the top of the notebook\n logger.warning(\"Input notebook does not contain a cell with tag 'parameters'\")\n before = []\n after = nb.cells\n\n nb.cells = before + [newcell] + after\n nb.metadata.papermill['parameters'] = parameters\n\n return nb", "response": "Assign parameters into the appropriate place in the input notebook"} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns True if key is a modifier key.", "response": "def is_modifier(key):\n \"\"\"\n Returns True if `key` is a scan code or name of a modifier key.\n \"\"\"\n if _is_str(key):\n return key in all_modifiers\n else:\n if not _modifier_scan_codes:\n scan_codes = (key_to_scan_codes(name, False) for name in all_modifiers) \n _modifier_scan_codes.update(*scan_codes)\n return key in _modifier_scan_codes"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef key_to_scan_codes(key, error_if_missing=True):\n if _is_number(key):\n return (key,)\n elif _is_list(key):\n return sum((key_to_scan_codes(i) for i in key), ())\n elif not _is_str(key):\n raise ValueError('Unexpected key type ' + str(type(key)) + ', value (' + repr(key) + ')')\n\n normalized = normalize_name(key)\n if normalized in sided_modifiers:\n left_scan_codes = key_to_scan_codes('left ' + normalized, False)\n right_scan_codes = key_to_scan_codes('right ' + normalized, False)\n return left_scan_codes + tuple(c for c in right_scan_codes if c not in left_scan_codes)\n\n try:\n # Put items in ordered dict to remove duplicates.\n t = tuple(_collections.OrderedDict((scan_code, True) for scan_code, modifier in _os_keyboard.map_name(normalized)))\n e = None\n except (KeyError, ValueError) as exception:\n t = ()\n e = exception\n\n if not t and error_if_missing:\n raise ValueError('Key {} is not mapped to any known key.'.format(repr(key)), e)\n else:\n return t", "response": "Converts a key to a list of scan codes associated with this key."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef parse_hotkey(hotkey):\n if _is_number(hotkey) or len(hotkey) == 1:\n scan_codes = key_to_scan_codes(hotkey)\n step = (scan_codes,)\n steps = (step,)\n return steps\n elif _is_list(hotkey):\n if not any(map(_is_list, hotkey)):\n step = tuple(key_to_scan_codes(k) for k in hotkey)\n steps = (step,)\n return steps\n return hotkey\n\n steps = []\n for step in _re.split(r',\\s?', hotkey):\n keys = _re.split(r'\\s?\\+\\s?', step)\n steps.append(tuple(key_to_scan_codes(key) for key in keys))\n return tuple(steps)", "response": "Parses a user - provided hotkey into nested tuples representing the base base."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nsending events that perform the given hotkey.", "response": "def send(hotkey, do_press=True, do_release=True):\n \"\"\"\n Sends OS events that perform the given *hotkey* hotkey.\n\n - `hotkey` can be either a scan code (e.g. 57 for space), single key\n (e.g. 'space') or multi-key, multi-step hotkey (e.g. 'alt+F4, enter').\n - `do_press` if true then press events are sent. Defaults to True.\n - `do_release` if true then release events are sent. Defaults to True.\n\n send(57)\n send('ctrl+alt+del')\n send('alt+F4, enter')\n send('shift+s')\n\n Note: keys are released in the opposite order they were pressed.\n \"\"\"\n _listener.is_replaying = True\n\n parsed = parse_hotkey(hotkey)\n for step in parsed:\n if do_press:\n for scan_codes in step:\n _os_keyboard.press(scan_codes[0])\n\n if do_release:\n for scan_codes in reversed(step):\n _os_keyboard.release(scan_codes[0])\n\n _listener.is_replaying = False"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning True if the key is pressed.", "response": "def is_pressed(hotkey):\n \"\"\"\n Returns True if the key is pressed.\n\n is_pressed(57) #-> True\n is_pressed('space') #-> True\n is_pressed('ctrl+space') #-> True\n \"\"\"\n _listener.start_if_necessary()\n\n if _is_number(hotkey):\n # Shortcut.\n with _pressed_events_lock:\n return hotkey in _pressed_events\n\n steps = parse_hotkey(hotkey)\n if len(steps) > 1:\n raise ValueError(\"Impossible to check if multi-step hotkeys are pressed (`a+b` is ok, `a, b` isn't).\")\n\n # Convert _pressed_events into a set \n with _pressed_events_lock:\n pressed_scan_codes = set(_pressed_events)\n for scan_codes in steps[0]:\n if not any(scan_code in pressed_scan_codes for scan_code in scan_codes):\n return False\n return True"} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncall the provided function in a new thread after waiting some time.", "response": "def call_later(fn, args=(), delay=0.001):\n \"\"\"\n Calls the provided function in a new thread after waiting some time.\n Useful for giving the system some time to process an event, without blocking\n the current execution flow.\n \"\"\"\n thread = _Thread(target=lambda: (_time.sleep(delay), fn(*args)))\n thread.start()"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ninstalling a global listener for all available keyboards invoking callback on each key is pressed or released.", "response": "def hook(callback, suppress=False, on_remove=lambda: None):\n \"\"\"\n Installs a global listener on all available keyboards, invoking `callback`\n each time a key is pressed or released.\n \n The event passed to the callback is of type `keyboard.KeyboardEvent`,\n with the following attributes:\n\n - `name`: an Unicode representation of the character (e.g. \"&\") or\n description (e.g. \"space\"). The name is always lower-case.\n - `scan_code`: number representing the physical key, e.g. 55.\n - `time`: timestamp of the time the event occurred, with as much precision\n as given by the OS.\n\n Returns the given callback for easier development.\n \"\"\"\n if suppress:\n _listener.start_if_necessary()\n append, remove = _listener.blocking_hooks.append, _listener.blocking_hooks.remove\n else:\n append, remove = _listener.add_handler, _listener.remove_handler\n\n append(callback)\n def remove_():\n del _hooks[callback]\n del _hooks[remove_]\n remove(callback)\n on_remove()\n _hooks[callback] = _hooks[remove_] = remove_\n return remove_"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nrunning callback for every KEY_DOWN event.", "response": "def on_press(callback, suppress=False):\n \"\"\"\n Invokes `callback` for every KEY_DOWN event. For details see `hook`.\n \"\"\"\n return hook(lambda e: e.event_type == KEY_UP or callback(e), suppress=suppress)"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nrunning callback for every KEY_UP event.", "response": "def on_release(callback, suppress=False):\n \"\"\"\n Invokes `callback` for every KEY_UP event. For details see `hook`.\n \"\"\"\n return hook(lambda e: e.event_type == KEY_DOWN or callback(e), suppress=suppress)"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nhooking a key up and key down event for a single key.", "response": "def hook_key(key, callback, suppress=False):\n \"\"\"\n Hooks key up and key down events for a single key. Returns the event handler\n created. To remove a hooked key use `unhook_key(key)` or\n `unhook_key(handler)`.\n\n Note: this function shares state with hotkeys, so `clear_all_hotkeys`\n affects it aswell.\n \"\"\"\n _listener.start_if_necessary()\n store = _listener.blocking_keys if suppress else _listener.nonblocking_keys\n scan_codes = key_to_scan_codes(key)\n for scan_code in scan_codes:\n store[scan_code].append(callback)\n\n def remove_():\n del _hooks[callback]\n del _hooks[key]\n del _hooks[remove_]\n for scan_code in scan_codes:\n store[scan_code].remove(callback)\n _hooks[callback] = _hooks[key] = _hooks[remove_] = remove_\n return remove_"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef on_press_key(key, callback, suppress=False):\n return hook_key(key, lambda e: e.event_type == KEY_UP or callback(e), suppress=suppress)", "response": "Invokes callback for KEY_DOWN event related to the given key."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef on_release_key(key, callback, suppress=False):\n return hook_key(key, lambda e: e.event_type == KEY_DOWN or callback(e), suppress=suppress)", "response": "Invokes callback for KEY_UP event related to the given key."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nremove all keyboard hooks in use including hotkeys abbreviations word listeners and wait events.", "response": "def unhook_all():\n \"\"\"\n Removes all keyboard hooks in use, including hotkeys, abbreviations, word\n listeners, `record`ers and `wait`s.\n \"\"\"\n _listener.start_if_necessary()\n _listener.blocking_keys.clear()\n _listener.nonblocking_keys.clear()\n del _listener.blocking_hooks[:]\n del _listener.handlers[:]\n unhook_all_hotkeys()"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef parse_hotkey_combinations(hotkey):\n def combine_step(step):\n # A single step may be composed of many keys, and each key can have\n # multiple scan codes. To speed up hotkey matching and avoid introducing\n # event delays, we list all possible combinations of scan codes for these\n # keys. Hotkeys are usually small, and there are not many combinations, so\n # this is not as insane as it sounds.\n return (tuple(sorted(scan_codes)) for scan_codes in _itertools.product(*step))\n\n return tuple(tuple(combine_step(step)) for step in parse_hotkey(hotkey))", "response": "Parses a user - provided hotkey and returns a list of possible combinations of each scan code."} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nadds handler to the list of possible combinations of the modfiier + main key.", "response": "def _add_hotkey_step(handler, combinations, suppress):\n \"\"\"\n Hooks a single-step hotkey (e.g. 'shift+a').\n \"\"\"\n container = _listener.blocking_hotkeys if suppress else _listener.nonblocking_hotkeys\n\n # Register the scan codes of every possible combination of\n # modfiier + main key. Modifiers have to be registered in \n # filtered_modifiers too, so suppression and replaying can work.\n for scan_codes in combinations:\n for scan_code in scan_codes:\n if is_modifier(scan_code):\n _listener.filtered_modifiers[scan_code] += 1\n container[scan_codes].append(handler)\n\n def remove():\n for scan_codes in combinations:\n for scan_code in scan_codes:\n if is_modifier(scan_code):\n _listener.filtered_modifiers[scan_code] -= 1\n container[scan_codes].remove(handler)\n return remove"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nadding a callback to be invoked every time a hotkey is pressed.", "response": "def add_hotkey(hotkey, callback, args=(), suppress=False, timeout=1, trigger_on_release=False):\n \"\"\"\n Invokes a callback every time a hotkey is pressed. The hotkey must\n be in the format `ctrl+shift+a, s`. This would trigger when the user holds\n ctrl, shift and \"a\" at once, releases, and then presses \"s\". To represent\n literal commas, pluses, and spaces, use their names ('comma', 'plus',\n 'space').\n\n - `args` is an optional list of arguments to passed to the callback during\n each invocation.\n - `suppress` defines if successful triggers should block the keys from being\n sent to other programs.\n - `timeout` is the amount of seconds allowed to pass between key presses.\n - `trigger_on_release` if true, the callback is invoked on key release instead\n of key press.\n\n The event handler function is returned. To remove a hotkey call\n `remove_hotkey(hotkey)` or `remove_hotkey(handler)`.\n before the hotkey state is reset.\n\n Note: hotkeys are activated when the last key is *pressed*, not released.\n Note: the callback is executed in a separate thread, asynchronously. For an\n example of how to use a callback synchronously, see `wait`.\n\n Examples:\n\n # Different but equivalent ways to listen for a spacebar key press.\n add_hotkey(' ', print, args=['space was pressed'])\n add_hotkey('space', print, args=['space was pressed'])\n add_hotkey('Space', print, args=['space was pressed'])\n # Here 57 represents the keyboard code for spacebar; so you will be\n # pressing 'spacebar', not '57' to activate the print function.\n add_hotkey(57, print, args=['space was pressed'])\n\n add_hotkey('ctrl+q', quit)\n add_hotkey('ctrl+alt+enter, space', some_callback)\n \"\"\"\n if args:\n callback = lambda callback=callback: callback(*args)\n\n _listener.start_if_necessary()\n\n steps = parse_hotkey_combinations(hotkey)\n\n event_type = KEY_UP if trigger_on_release else KEY_DOWN\n if len(steps) == 1:\n # Deciding when to allow a KEY_UP event is far harder than I thought,\n # and any mistake will make that key \"sticky\". Therefore just let all\n # KEY_UP events go through as long as that's not what we are listening\n # for.\n handler = lambda e: (event_type == KEY_DOWN and e.event_type == KEY_UP and e.scan_code in _logically_pressed_keys) or (event_type == e.event_type and callback())\n remove_step = _add_hotkey_step(handler, steps[0], suppress)\n def remove_():\n remove_step()\n del _hotkeys[hotkey]\n del _hotkeys[remove_]\n del _hotkeys[callback]\n # TODO: allow multiple callbacks for each hotkey without overwriting the\n # remover.\n _hotkeys[hotkey] = _hotkeys[remove_] = _hotkeys[callback] = remove_\n return remove_\n\n state = _State()\n state.remove_catch_misses = None\n state.remove_last_step = None\n state.suppressed_events = []\n state.last_update = float('-inf')\n \n def catch_misses(event, force_fail=False):\n if (\n event.event_type == event_type\n and state.index\n and event.scan_code not in allowed_keys_by_step[state.index]\n ) or (\n timeout\n and _time.monotonic() - state.last_update >= timeout\n ) or force_fail: # Weird formatting to ensure short-circuit.\n\n state.remove_last_step()\n\n for event in state.suppressed_events:\n if event.event_type == KEY_DOWN:\n press(event.scan_code)\n else:\n release(event.scan_code)\n del state.suppressed_events[:]\n\n index = 0\n set_index(0)\n return True\n\n def set_index(new_index):\n state.index = new_index\n\n if new_index == 0:\n # This is done for performance reasons, avoiding a global key hook\n # that is always on.\n state.remove_catch_misses = lambda: None\n elif new_index == 1:\n state.remove_catch_misses()\n # Must be `suppress=True` to ensure `send` has priority.\n state.remove_catch_misses = hook(catch_misses, suppress=True)\n\n if new_index == len(steps) - 1:\n def handler(event):\n if event.event_type == KEY_UP:\n remove()\n set_index(0)\n accept = event.event_type == event_type and callback() \n if accept:\n return catch_misses(event, force_fail=True)\n else:\n state.suppressed_events[:] = [event]\n return False\n remove = _add_hotkey_step(handler, steps[state.index], suppress)\n else:\n # Fix value of next_index.\n def handler(event, new_index=state.index+1):\n if event.event_type == KEY_UP:\n remove()\n set_index(new_index)\n state.suppressed_events.append(event)\n return False\n remove = _add_hotkey_step(handler, steps[state.index], suppress)\n state.remove_last_step = remove\n state.last_update = _time.monotonic()\n return False\n set_index(0)\n\n allowed_keys_by_step = [\n set().union(*step)\n for step in steps\n ]\n\n def remove_():\n state.remove_catch_misses()\n state.remove_last_step()\n del _hotkeys[hotkey]\n del _hotkeys[remove_]\n del _hotkeys[callback]\n # TODO: allow multiple callbacks for each hotkey without overwriting the\n # remover.\n _hotkeys[hotkey] = _hotkeys[remove_] = _hotkeys[callback] = remove_\n return remove_"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nadd a hotkey src to the tree.", "response": "def remap_hotkey(src, dst, suppress=True, trigger_on_release=False):\n \"\"\"\n Whenever the hotkey `src` is pressed, suppress it and send\n `dst` instead.\n\n Example:\n\n remap('alt+w', 'ctrl+up')\n \"\"\"\n def handler():\n active_modifiers = sorted(modifier for modifier, state in _listener.modifier_states.items() if state == 'allowed')\n for modifier in active_modifiers:\n release(modifier)\n send(dst)\n for modifier in reversed(active_modifiers):\n press(modifier)\n return False\n return add_hotkey(src, handler, suppress=suppress, trigger_on_release=trigger_on_release)"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef stash_state():\n # TODO: stash caps lock / numlock /scrollock state.\n with _pressed_events_lock:\n state = sorted(_pressed_events)\n for scan_code in state:\n _os_keyboard.release(scan_code)\n return state", "response": "Builds a list of all currently pressed scan codes releases them and returns\n "} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nrestores the state of the current keyboard.", "response": "def restore_state(scan_codes):\n \"\"\"\n Given a list of scan_codes ensures these keys, and only these keys, are\n pressed. Pairs well with `stash_state`, alternative to `restore_modifiers`.\n \"\"\"\n _listener.is_replaying = True\n\n with _pressed_events_lock:\n current = set(_pressed_events)\n target = set(scan_codes)\n for scan_code in current - target:\n _os_keyboard.release(scan_code)\n for scan_code in target - current:\n _os_keyboard.press(scan_code)\n\n _listener.is_replaying = False"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef write(text, delay=0, restore_state_after=True, exact=None):\n if exact is None:\n exact = _platform.system() == 'Windows'\n\n state = stash_state()\n \n # Window's typing of unicode characters is quite efficient and should be preferred.\n if exact:\n for letter in text:\n if letter in '\\n\\b':\n send(letter)\n else:\n _os_keyboard.type_unicode(letter)\n if delay: _time.sleep(delay)\n else:\n for letter in text:\n try:\n entries = _os_keyboard.map_name(normalize_name(letter))\n scan_code, modifiers = next(iter(entries))\n except (KeyError, ValueError):\n _os_keyboard.type_unicode(letter)\n continue\n \n for modifier in modifiers:\n press(modifier)\n\n _os_keyboard.press(scan_code)\n _os_keyboard.release(scan_code)\n\n for modifier in modifiers:\n release(modifier)\n\n if delay:\n _time.sleep(delay)\n\n if restore_state_after:\n restore_modifiers(state)", "response": "Writes a string to the keyboard and returns the state of the keyboard."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nblock the program execution until the given hotkey is pressed or the given parameters blocks forever.", "response": "def wait(hotkey=None, suppress=False, trigger_on_release=False):\n \"\"\"\n Blocks the program execution until the given hotkey is pressed or,\n if given no parameters, blocks forever.\n \"\"\"\n if hotkey:\n lock = _Event()\n remove = add_hotkey(hotkey, lambda: lock.set(), suppress=suppress, trigger_on_release=trigger_on_release)\n lock.wait()\n remove_hotkey(remove)\n else:\n while True:\n _time.sleep(1e6)"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef get_hotkey_name(names=None):\n if names is None:\n _listener.start_if_necessary()\n with _pressed_events_lock:\n names = [e.name for e in _pressed_events.values()]\n else:\n names = [normalize_name(name) for name in names]\n clean_names = set(e.replace('left ', '').replace('right ', '').replace('+', 'plus') for e in names)\n # https://developer.apple.com/macos/human-interface-guidelines/input-and-output/keyboard/\n # > List modifier keys in the correct order. If you use more than one modifier key in a\n # > hotkey, always list them in this order: Control, Option, Shift, Command.\n modifiers = ['ctrl', 'alt', 'shift', 'windows']\n sorting_key = lambda k: (modifiers.index(k) if k in modifiers else 5, str(k))\n return '+'.join(sorted(clean_names, key=sorting_key))", "response": "Returns a string representation of the given hotkey names."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreading a keyboard event from the system.", "response": "def read_event(suppress=False):\n \"\"\"\n Blocks until a keyboard event happens, then returns that event.\n \"\"\"\n queue = _queue.Queue(maxsize=1)\n hooked = hook(queue.put, suppress=suppress)\n while True:\n event = queue.get()\n unhook(hooked)\n return event"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreading a keyboard event from the keyboard and returns its name or its scan code.", "response": "def read_key(suppress=False):\n \"\"\"\n Blocks until a keyboard event happens, then returns that event's name or,\n if missing, its scan code.\n \"\"\"\n event = read_event(suppress)\n return event.name or event.scan_code"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef read_hotkey(suppress=True):\n queue = _queue.Queue()\n fn = lambda e: queue.put(e) or e.event_type == KEY_DOWN\n hooked = hook(fn, suppress=suppress)\n while True:\n event = queue.get()\n if event.event_type == KEY_UP:\n unhook(hooked)\n with _pressed_events_lock:\n names = [e.name for e in _pressed_events.values()] + [event.name]\n return get_hotkey_name(names)", "response": "Read a single key from the keyboard and return the name of the hotkey pressed."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_typed_strings(events, allow_backspace=True):\n backspace_name = 'delete' if _platform.system() == 'Darwin' else 'backspace'\n\n shift_pressed = False\n capslock_pressed = False\n string = ''\n for event in events:\n name = event.name\n\n # Space is the only key that we _parse_hotkey to the spelled out name\n # because of legibility. Now we have to undo that.\n if event.name == 'space':\n name = ' '\n\n if 'shift' in event.name:\n shift_pressed = event.event_type == 'down'\n elif event.name == 'caps lock' and event.event_type == 'down':\n capslock_pressed = not capslock_pressed\n elif allow_backspace and event.name == backspace_name and event.event_type == 'down':\n string = string[:-1]\n elif event.event_type == 'down':\n if len(name) == 1:\n if shift_pressed ^ capslock_pressed:\n name = name.upper()\n string = string + name\n else:\n yield string\n string = ''\n yield string", "response": "Given a sequence of events returns a generator that yields the typed string of the events."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nstarts recording all keyboard events into a global variable or the given queue if any. Returns the given queue of events and the hooked function.", "response": "def start_recording(recorded_events_queue=None):\n \"\"\"\n Starts recording all keyboard events into a global variable, or the given\n queue if any. Returns the queue of events and the hooked function.\n\n Use `stop_recording()` or `unhook(hooked_function)` to stop.\n \"\"\"\n recorded_events_queue = recorded_events_queue or _queue.Queue()\n global _recording\n _recording = (recorded_events_queue, hook(recorded_events_queue.put))\n return _recording"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nstops the global recording of events and returns a list of the events captured.", "response": "def stop_recording():\n \"\"\"\n Stops the global recording of events and returns a list of the events\n captured.\n \"\"\"\n global _recording\n if not _recording:\n raise ValueError('Must call \"start_recording\" before.')\n recorded_events_queue, hooked = _recording\n unhook(hooked)\n return list(recorded_events_queue.queue)"} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nrecords all keyboard events from all keyboards until the user presses the given hotkey.", "response": "def record(until='escape', suppress=False, trigger_on_release=False):\n \"\"\"\n Records all keyboard events from all keyboards until the user presses the\n given hotkey. Then returns the list of events recorded, of type\n `keyboard.KeyboardEvent`. Pairs well with\n `play(events)`.\n\n Note: this is a blocking function.\n Note: for more details on the keyboard hook and events see `hook`.\n \"\"\"\n start_recording()\n wait(until, suppress=suppress, trigger_on_release=trigger_on_release)\n return stop_recording()"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef play(events, speed_factor=1.0):\n state = stash_state()\n\n last_time = None\n for event in events:\n if speed_factor > 0 and last_time is not None:\n _time.sleep((event.time - last_time) / speed_factor)\n last_time = event.time\n\n key = event.scan_code or event.name\n press(key) if event.event_type == KEY_DOWN else release(key)\n\n restore_modifiers(state)", "response": "Plays a sequence of recorded events maintaining the relative time\n intervals."} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nadds a callback to be invoked every time a word is typed.", "response": "def add_word_listener(word, callback, triggers=['space'], match_suffix=False, timeout=2):\n \"\"\"\n Invokes a callback every time a sequence of characters is typed (e.g. 'pet')\n and followed by a trigger key (e.g. space). Modifiers (e.g. alt, ctrl,\n shift) are ignored.\n\n - `word` the typed text to be matched. E.g. 'pet'.\n - `callback` is an argument-less function to be invoked each time the word\n is typed.\n - `triggers` is the list of keys that will cause a match to be checked. If\n the user presses some key that is not a character (len>1) and not in\n triggers, the characters so far will be discarded. By default the trigger\n is only `space`.\n - `match_suffix` defines if endings of words should also be checked instead\n of only whole words. E.g. if true, typing 'carpet'+space will trigger the\n listener for 'pet'. Defaults to false, only whole words are checked.\n - `timeout` is the maximum number of seconds between typed characters before\n the current word is discarded. Defaults to 2 seconds.\n\n Returns the event handler created. To remove a word listener use\n `remove_word_listener(word)` or `remove_word_listener(handler)`.\n\n Note: all actions are performed on key down. Key up events are ignored.\n Note: word mathes are **case sensitive**.\n \"\"\"\n state = _State()\n state.current = ''\n state.time = -1\n\n def handler(event):\n name = event.name\n if event.event_type == KEY_UP or name in all_modifiers: return\n\n if timeout and event.time - state.time > timeout:\n state.current = ''\n state.time = event.time\n\n matched = state.current == word or (match_suffix and state.current.endswith(word))\n if name in triggers and matched:\n callback()\n state.current = ''\n elif len(name) > 1:\n state.current = ''\n else:\n state.current += name\n\n hooked = hook(handler)\n def remove():\n hooked()\n del _word_listeners[word]\n del _word_listeners[handler]\n del _word_listeners[remove]\n _word_listeners[word] = _word_listeners[handler] = _word_listeners[remove] = remove\n # TODO: allow multiple word listeners and removing them correctly.\n return remove"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nadding a hotkey that replaces one typed text with another.", "response": "def add_abbreviation(source_text, replacement_text, match_suffix=False, timeout=2):\n \"\"\"\n Registers a hotkey that replaces one typed text with another. For example\n\n add_abbreviation('tm', u'\u2122')\n\n Replaces every \"tm\" followed by a space with a \u2122 symbol (and no space). The\n replacement is done by sending backspace events.\n\n - `match_suffix` defines if endings of words should also be checked instead\n of only whole words. E.g. if true, typing 'carpet'+space will trigger the\n listener for 'pet'. Defaults to false, only whole words are checked.\n - `timeout` is the maximum number of seconds between typed characters before\n the current word is discarded. Defaults to 2 seconds.\n \n For more details see `add_word_listener`.\n \"\"\"\n replacement = '\\b'*(len(source_text)+1) + replacement_text\n callback = lambda: write(replacement)\n return add_word_listener(source_text, callback, match_suffix=match_suffix, timeout=timeout)"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef direct_callback(self, event):\n # Pass through all fake key events, don't even report to other handlers.\n if self.is_replaying:\n return True\n\n if not all(hook(event) for hook in self.blocking_hooks):\n return False\n\n event_type = event.event_type\n scan_code = event.scan_code\n\n # Update tables of currently pressed keys and modifiers.\n with _pressed_events_lock:\n if event_type == KEY_DOWN:\n if is_modifier(scan_code): self.active_modifiers.add(scan_code)\n _pressed_events[scan_code] = event\n hotkey = tuple(sorted(_pressed_events))\n if event_type == KEY_UP:\n self.active_modifiers.discard(scan_code)\n if scan_code in _pressed_events: del _pressed_events[scan_code]\n\n # Mappings based on individual keys instead of hotkeys.\n for key_hook in self.blocking_keys[scan_code]:\n if not key_hook(event):\n return False\n\n # Default accept.\n accept = True\n\n if self.blocking_hotkeys:\n if self.filtered_modifiers[scan_code]:\n origin = 'modifier'\n modifiers_to_update = set([scan_code])\n else:\n modifiers_to_update = self.active_modifiers\n if is_modifier(scan_code):\n modifiers_to_update = modifiers_to_update | {scan_code}\n callback_results = [callback(event) for callback in self.blocking_hotkeys[hotkey]]\n if callback_results:\n accept = all(callback_results)\n origin = 'hotkey'\n else:\n origin = 'other'\n\n for key in sorted(modifiers_to_update):\n transition_tuple = (self.modifier_states.get(key, 'free'), event_type, origin)\n should_press, new_accept, new_state = self.transition_table[transition_tuple]\n if should_press: press(key)\n if new_accept is not None: accept = new_accept\n self.modifier_states[key] = new_state\n\n if accept:\n if event_type == KEY_DOWN:\n _logically_pressed_keys[scan_code] = event\n elif event_type == KEY_UP and scan_code in _logically_pressed_keys:\n del _logically_pressed_keys[scan_code]\n\n # Queue for handlers that won't block the event.\n self.queue.put(event)\n\n return accept", "response": "Direct callback function for all keyboard events."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nensure the scan code/virtual key code/name translation tables are filled.", "response": "def _setup_name_tables():\n \"\"\"\n Ensures the scan code/virtual key code/name translation tables are\n filled.\n \"\"\"\n with tables_lock:\n if to_name: return\n\n # Go through every possible scan code, and map them to virtual key codes.\n # Then vice-versa.\n all_scan_codes = [(sc, user32.MapVirtualKeyExW(sc, MAPVK_VSC_TO_VK_EX, 0)) for sc in range(0x100)]\n all_vks = [(user32.MapVirtualKeyExW(vk, MAPVK_VK_TO_VSC_EX, 0), vk) for vk in range(0x100)]\n for scan_code, vk in all_scan_codes + all_vks:\n # `to_name` and `from_name` entries will be a tuple (scan_code, vk, extended, shift_state).\n if (scan_code, vk, 0, 0, 0) in to_name:\n continue\n\n if scan_code not in scan_code_to_vk:\n scan_code_to_vk[scan_code] = vk\n\n # Brute force all combinations to find all possible names.\n for extended in [0, 1]:\n for modifiers in distinct_modifiers:\n entry = (scan_code, vk, extended, modifiers)\n # Get key names from ToUnicode, GetKeyNameText, MapVirtualKeyW and official virtual keys.\n names = list(get_event_names(*entry))\n if names:\n # Also map lowercased key names, but only after the properly cased ones.\n lowercase_names = [name.lower() for name in names]\n to_name[entry] = names + lowercase_names\n # Remember the \"id\" of the name, as the first techniques\n # have better results and therefore priority.\n for i, name in enumerate(map(normalize_name, names + lowercase_names)):\n from_name[name].append((i, entry))\n\n # TODO: single quotes on US INTL is returning the dead key (?), and therefore\n # not typing properly.\n\n # Alt gr is way outside the usual range of keys (0..127) and on my\n # computer is named as 'ctrl'. Therefore we add it manually and hope\n # Windows is consistent in its inconsistency.\n for extended in [0, 1]:\n for modifiers in distinct_modifiers:\n to_name[(541, 162, extended, modifiers)] = ['alt gr']\n from_name['alt gr'].append((1, (541, 162, extended, modifiers)))\n\n modifiers_preference = defaultdict(lambda: 10)\n modifiers_preference.update({(): 0, ('shift',): 1, ('alt gr',): 2, ('ctrl',): 3, ('alt',): 4})\n def order_key(line):\n i, entry = line\n scan_code, vk, extended, modifiers = entry\n return modifiers_preference[modifiers], i, extended, vk, scan_code\n for name, entries in list(from_name.items()):\n from_name[name] = sorted(set(entries), key=order_key)"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\npreparing a callback to be invoked for each high - level keyboard event.", "response": "def prepare_intercept(callback):\n \"\"\"\n Registers a Windows low level keyboard hook. The provided callback will\n be invoked for each high-level keyboard event, and is expected to return\n True if the key event should be passed to the next program, or False if\n the event is to be blocked.\n\n No event is processed until the Windows messages are pumped (see\n start_intercept).\n \"\"\"\n _setup_name_tables()\n \n def process_key(event_type, vk, scan_code, is_extended):\n global shift_is_pressed, altgr_is_pressed, ignore_next_right_alt\n #print(event_type, vk, scan_code, is_extended)\n\n # Pressing alt-gr also generates an extra \"right alt\" event\n if vk == 0xA5 and ignore_next_right_alt:\n ignore_next_right_alt = False\n return True\n\n modifiers = (\n ('shift',) * shift_is_pressed +\n ('alt gr',) * altgr_is_pressed +\n ('num lock',) * (user32.GetKeyState(0x90) & 1) +\n ('caps lock',) * (user32.GetKeyState(0x14) & 1) +\n ('scroll lock',) * (user32.GetKeyState(0x91) & 1)\n )\n entry = (scan_code, vk, is_extended, modifiers)\n if entry not in to_name:\n to_name[entry] = list(get_event_names(*entry))\n\n names = to_name[entry]\n name = names[0] if names else None\n\n # TODO: inaccurate when holding multiple different shifts.\n if vk in shift_vks:\n shift_is_pressed = event_type == KEY_DOWN\n if scan_code == 541 and vk == 162:\n ignore_next_right_alt = True\n altgr_is_pressed = event_type == KEY_DOWN\n\n is_keypad = (scan_code, vk, is_extended) in keypad_keys\n return callback(KeyboardEvent(event_type=event_type, scan_code=scan_code or -vk, name=name, is_keypad=is_keypad))\n\n def low_level_keyboard_handler(nCode, wParam, lParam):\n try:\n vk = lParam.contents.vk_code\n # Ignore the second `alt` DOWN observed in some cases.\n fake_alt = (LLKHF_INJECTED | 0x20)\n # Ignore events generated by SendInput with Unicode.\n if vk != VK_PACKET and lParam.contents.flags & fake_alt != fake_alt:\n event_type = keyboard_event_types[wParam]\n is_extended = lParam.contents.flags & 1\n scan_code = lParam.contents.scan_code\n should_continue = process_key(event_type, vk, scan_code, is_extended)\n if not should_continue:\n return -1\n except Exception as e:\n print('Error in keyboard hook:')\n traceback.print_exc()\n\n return CallNextHookEx(None, nCode, wParam, lParam)\n\n WH_KEYBOARD_LL = c_int(13)\n keyboard_callback = LowLevelKeyboardProc(low_level_keyboard_handler)\n handle = GetModuleHandleW(None)\n thread_id = DWORD(0)\n keyboard_hook = SetWindowsHookEx(WH_KEYBOARD_LL, keyboard_callback, handle, thread_id)\n\n # Register to remove the hook when the interpreter exits. Unfortunately a\n # try/finally block doesn't seem to work here.\n atexit.register(UnhookWindowsHookEx, keyboard_callback)"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nstarting the listening thread if it wans t already.", "response": "def start_if_necessary(self):\n \"\"\"\n Starts the listening thread if it wans't already.\n \"\"\"\n self.lock.acquire()\n try:\n if not self.listening:\n self.init()\n\n self.listening = True\n self.listening_thread = Thread(target=self.listen)\n self.listening_thread.daemon = True\n self.listening_thread.start()\n\n self.processing_thread = Thread(target=self.process)\n self.processing_thread.daemon = True\n self.processing_thread.start()\n finally:\n self.lock.release()"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef process(self):\n assert self.queue is not None\n while True:\n event = self.queue.get()\n if self.pre_process_event(event):\n self.invoke_handlers(event)\n self.queue.task_done()", "response": "Processes the queue of events and invoke handlers."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nremove a previously added event handler.", "response": "def remove_handler(self, handler):\n \"\"\" Removes a previously added event handler. \"\"\"\n while handler in self.handlers:\n self.handlers.remove(handler)"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef move(x, y, absolute=True, duration=0):\n x = int(x)\n y = int(y)\n\n # Requires an extra system call on Linux, but `move_relative` is measured\n # in millimiters so we would lose precision.\n position_x, position_y = get_position()\n\n if not absolute:\n x = position_x + x\n y = position_y + y\n\n if duration:\n start_x = position_x\n start_y = position_y\n dx = x - start_x\n dy = y - start_y\n\n if dx == 0 and dy == 0:\n _time.sleep(duration)\n else:\n # 120 movements per second.\n # Round and keep float to ensure float division in Python 2\n steps = max(1.0, float(int(duration * 120.0)))\n for i in range(int(steps)+1):\n move(start_x + dx*i/steps, start_y + dy*i/steps)\n _time.sleep(duration/steps)\n else:\n _os_mouse.move_to(x, y)", "response": "Moves the mouse to the specified location."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef drag(start_x, start_y, end_x, end_y, absolute=True, duration=0):\n if is_pressed():\n release()\n move(start_x, start_y, absolute, 0)\n press()\n move(end_x, end_y, absolute, duration)\n release()", "response": "Moves the mouse to the left mouse button."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef on_button(callback, args=(), buttons=(LEFT, MIDDLE, RIGHT, X, X2), types=(UP, DOWN, DOUBLE)):\n if not isinstance(buttons, (tuple, list)):\n buttons = (buttons,)\n if not isinstance(types, (tuple, list)):\n types = (types,)\n\n def handler(event):\n if isinstance(event, ButtonEvent):\n if event.event_type in types and event.button in buttons:\n callback(*args)\n _listener.add_handler(handler)\n return handler", "response": "Invokes callback with args when the specified event happens."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef wait(button=LEFT, target_types=(UP, DOWN, DOUBLE)):\n from threading import Lock\n lock = Lock()\n lock.acquire()\n handler = on_button(lock.release, (), [button], target_types)\n lock.acquire()\n _listener.remove_handler(handler)", "response": "Blocks program execution until the given button performs an event."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nrecording all mouse events until the user presses the given button.", "response": "def record(button=RIGHT, target_types=(DOWN,)):\n \"\"\"\n Records all mouse events until the user presses the given button.\n Then returns the list of events recorded. Pairs well with `play(events)`.\n\n Note: this is a blocking function.\n Note: for more details on the mouse hook and events see `hook`.\n \"\"\"\n recorded = []\n hook(recorded.append)\n wait(button=button, target_types=target_types)\n unhook(recorded.append)\n return recorded"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef play(events, speed_factor=1.0, include_clicks=True, include_moves=True, include_wheel=True):\n last_time = None\n for event in events:\n if speed_factor > 0 and last_time is not None:\n _time.sleep((event.time - last_time) / speed_factor)\n last_time = event.time\n\n if isinstance(event, ButtonEvent) and include_clicks:\n if event.event_type == UP:\n _os_mouse.release(event.button)\n else:\n _os_mouse.press(event.button)\n elif isinstance(event, MoveEvent) and include_moves:\n _os_mouse.move_to(event.x, event.y)\n elif isinstance(event, WheelEvent) and include_wheel:\n _os_mouse.wheel(event.delta)", "response": "Plays a sequence of recorded events."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef normalize_name(name):\n if not name or not isinstance(name, basestring):\n raise ValueError('Can only normalize non-empty string names. Unexpected '+ repr(name))\n\n if len(name) > 1:\n name = name.lower()\n if name != '_' and '_' in name:\n name = name.replace('_', ' ')\n\n return canonical_names.get(name, name)", "response": "Normalizes a key name."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef listen(queue):\n if not os.geteuid() == 0:\n raise OSError(\"Error 13 - Must be run as administrator\")\n listener = MouseEventListener(lambda e: queue.put(e) or is_allowed(e.name, e.event_type == KEY_UP))\n t = threading.Thread(target=listener.run, args=())\n t.daemon = True\n t.start()", "response": "Adds events to the queue."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nsend a down event for the specified button.", "response": "def press(button=LEFT):\n \"\"\" Sends a down event for the specified button, using the provided constants \"\"\"\n location = get_position()\n button_code, button_down, _, _ = _button_mapping[button]\n e = Quartz.CGEventCreateMouseEvent(\n None,\n button_down,\n location,\n button_code)\n\n # Check if this is a double-click (same location within the last 300ms)\n if _last_click[\"time\"] is not None and datetime.datetime.now() - _last_click[\"time\"] < datetime.timedelta(seconds=0.3) and _last_click[\"button\"] == button and _last_click[\"position\"] == location:\n # Repeated Click\n _last_click[\"click_count\"] = min(3, _last_click[\"click_count\"]+1)\n else:\n # Not a double-click - Reset last click\n _last_click[\"click_count\"] = 1\n Quartz.CGEventSetIntegerValueField(\n e,\n Quartz.kCGMouseEventClickState,\n _last_click[\"click_count\"])\n Quartz.CGEventPost(Quartz.kCGHIDEventTap, e)\n _button_state[button] = True\n _last_click[\"time\"] = datetime.datetime.now()\n _last_click[\"button\"] = button\n _last_click[\"position\"] = location"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef release(button=LEFT):\n location = get_position()\n button_code, _, button_up, _ = _button_mapping[button]\n e = Quartz.CGEventCreateMouseEvent(\n None,\n button_up,\n location,\n button_code)\n\n if _last_click[\"time\"] is not None and _last_click[\"time\"] > datetime.datetime.now() - datetime.timedelta(microseconds=300000) and _last_click[\"button\"] == button and _last_click[\"position\"] == location:\n # Repeated Click\n Quartz.CGEventSetIntegerValueField(\n e,\n Quartz.kCGMouseEventClickState,\n _last_click[\"click_count\"])\n Quartz.CGEventPost(Quartz.kCGHIDEventTap, e)\n _button_state[button] = False", "response": "Sends an up event for the specified button."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef wheel(delta=1):\n location = get_position()\n e = Quartz.CGEventCreateMouseEvent(\n None,\n Quartz.kCGEventScrollWheel,\n location,\n Quartz.kCGMouseButtonLeft)\n e2 = Quartz.CGEventCreateScrollWheelEvent(\n None,\n Quartz.kCGScrollEventUnitLine,\n 1,\n delta)\n Quartz.CGEventPost(Quartz.kCGHIDEventTap, e)\n Quartz.CGEventPost(Quartz.kCGHIDEventTap, e2)", "response": "Sends a wheel event for the specified number of clicks."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef move_to(x, y):\n for b in _button_state:\n if _button_state[b]:\n e = Quartz.CGEventCreateMouseEvent(\n None,\n _button_mapping[b][3], # Drag Event\n (x, y),\n _button_mapping[b][0])\n break\n else:\n e = Quartz.CGEventCreateMouseEvent(\n None,\n Quartz.kCGEventMouseMoved,\n (x, y),\n Quartz.kCGMouseButtonLeft)\n Quartz.CGEventPost(Quartz.kCGHIDEventTap, e)", "response": "Sets the mouse s location to the specified coordinates."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef get_position():\n e = Quartz.CGEventCreate(None)\n point = Quartz.CGEventGetLocation(e)\n return (point.x, point.y)", "response": "Returns the mouse s location as a tuple of x y."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef character_to_vk(self, character):\n for vk in self.non_layout_keys:\n if self.non_layout_keys[vk] == character.lower():\n return (vk, [])\n for vk in self.layout_specific_keys:\n if self.layout_specific_keys[vk][0] == character:\n return (vk, [])\n elif self.layout_specific_keys[vk][1] == character:\n return (vk, ['shift'])\n raise ValueError(\"Unrecognized character: {}\".format(character))", "response": "Convert a character to a tuple of scan code and modifiers."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef vk_to_character(self, vk, modifiers=[]):\n if vk in self.non_layout_keys:\n # Not a character\n return self.non_layout_keys[vk]\n elif vk in self.layout_specific_keys:\n if 'shift' in modifiers:\n return self.layout_specific_keys[vk][1]\n return self.layout_specific_keys[vk][0]\n else:\n # Invalid vk\n raise ValueError(\"Invalid scan code: {}\".format(vk))", "response": "Returns a character corresponding to the specified scan code."} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nsends a down event for the specified scan code.", "response": "def press(self, key_code):\n \"\"\" Sends a 'down' event for the specified scan code \"\"\"\n if key_code >= 128:\n # Media key\n ev = NSEvent.otherEventWithType_location_modifierFlags_timestamp_windowNumber_context_subtype_data1_data2_(\n 14, # type\n (0, 0), # location\n 0xa00, # flags\n 0, # timestamp\n 0, # window\n 0, # ctx\n 8, # subtype\n ((key_code-128) << 16) | (0xa << 8), # data1\n -1 # data2\n )\n Quartz.CGEventPost(0, ev.CGEvent())\n else:\n # Regular key\n # Apply modifiers if necessary\n event_flags = 0\n if self.current_modifiers[\"shift\"]:\n event_flags += Quartz.kCGEventFlagMaskShift\n if self.current_modifiers[\"caps\"]:\n event_flags += Quartz.kCGEventFlagMaskAlphaShift\n if self.current_modifiers[\"alt\"]:\n event_flags += Quartz.kCGEventFlagMaskAlternate\n if self.current_modifiers[\"ctrl\"]:\n event_flags += Quartz.kCGEventFlagMaskControl\n if self.current_modifiers[\"cmd\"]:\n event_flags += Quartz.kCGEventFlagMaskCommand\n \n # Update modifiers if necessary\n if key_code == 0x37: # cmd\n self.current_modifiers[\"cmd\"] = True\n elif key_code == 0x38 or key_code == 0x3C: # shift or right shift\n self.current_modifiers[\"shift\"] = True\n elif key_code == 0x39: # caps lock\n self.current_modifiers[\"caps\"] = True\n elif key_code == 0x3A: # alt\n self.current_modifiers[\"alt\"] = True\n elif key_code == 0x3B: # ctrl\n self.current_modifiers[\"ctrl\"] = True\n event = Quartz.CGEventCreateKeyboardEvent(None, key_code, True)\n Quartz.CGEventSetFlags(event, event_flags)\n Quartz.CGEventPost(Quartz.kCGHIDEventTap, event)\n time.sleep(0.01)"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef run(self):\n self.tap = Quartz.CGEventTapCreate(\n Quartz.kCGSessionEventTap,\n Quartz.kCGHeadInsertEventTap,\n Quartz.kCGEventTapOptionDefault,\n Quartz.CGEventMaskBit(Quartz.kCGEventKeyDown) |\n Quartz.CGEventMaskBit(Quartz.kCGEventKeyUp) |\n Quartz.CGEventMaskBit(Quartz.kCGEventFlagsChanged),\n self.handler,\n None)\n loopsource = Quartz.CFMachPortCreateRunLoopSource(None, self.tap, 0)\n loop = Quartz.CFRunLoopGetCurrent()\n Quartz.CFRunLoopAddSource(loop, loopsource, Quartz.kCFRunLoopDefaultMode)\n Quartz.CGEventTapEnable(self.tap, True)\n\n while self.listening:\n Quartz.CFRunLoopRunInMode(Quartz.kCFRunLoopDefaultMode, 5, False)", "response": "Creates a listener and loops while waiting for an event."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nconverting a dumpkey name to a standard format.", "response": "def cleanup_key(name):\n \"\"\" Formats a dumpkeys format to our standard. \"\"\"\n name = name.lstrip('+')\n is_keypad = name.startswith('KP_')\n for mod in ('Meta_', 'Control_', 'dead_', 'KP_'):\n if name.startswith(mod):\n name = name[len(mod):]\n\n # Dumpkeys is weird like that.\n if name == 'Remove':\n name = 'Delete'\n elif name == 'Delete':\n name = 'Backspace'\n\n if name.endswith('_r'):\n name = 'right ' + name[:-2]\n if name.endswith('_l'):\n name = 'left ' + name[:-2]\n\n\n return normalize_name(name), is_keypad"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning a string that represents the indented block of the given grammar.", "response": "def indentedBlock(blockStatementExpr, indentStack, indent=True):\r\n \"\"\"\r\n Helper method for defining space-delimited indentation blocks, such as\r\n those used to define block statements in Python source code.\r\n\r\n Parameters:\r\n - blockStatementExpr - expression defining syntax of statement that\r\n is repeated within the indented block\r\n - indentStack - list created by caller to manage indentation stack\r\n (multiple statementWithIndentedBlock expressions within a single grammar\r\n should share a common indentStack)\r\n - indent - boolean indicating whether block must be indented beyond the\r\n the current level; set to False for block of left-most statements\r\n (default=C{True})\r\n\r\n A valid block must contain at least one C{blockStatement}.\r\n\r\n Example::\r\n data = '''\r\n def A(z):\r\n A1\r\n B = 100\r\n G = A2\r\n A2\r\n A3\r\n B\r\n def BB(a,b,c):\r\n BB1\r\n def BBA():\r\n bba1\r\n bba2\r\n bba3\r\n C\r\n D\r\n def spam(x,y):\r\n def eggs(z):\r\n pass\r\n '''\r\n\r\n\r\n indentStack = [1]\r\n stmt = Forward()\r\n\r\n identifier = Word(alphas, alphanums)\r\n funcDecl = (\"def\" + identifier + Group( \"(\" + Optional( delimitedList(identifier) ) + \")\" ) + \":\")\r\n func_body = indentedBlock(stmt, indentStack)\r\n funcDef = Group( funcDecl + func_body )\r\n\r\n rvalue = Forward()\r\n funcCall = Group(identifier + \"(\" + Optional(delimitedList(rvalue)) + \")\")\r\n rvalue << (funcCall | identifier | Word(nums))\r\n assignment = Group(identifier + \"=\" + rvalue)\r\n stmt << ( funcDef | assignment | identifier )\r\n\r\n module_body = OneOrMore(stmt)\r\n\r\n parseTree = module_body.parseString(data)\r\n parseTree.pprint()\r\n prints::\r\n [['def',\r\n 'A',\r\n ['(', 'z', ')'],\r\n ':',\r\n [['A1'], [['B', '=', '100']], [['G', '=', 'A2']], ['A2'], ['A3']]],\r\n 'B',\r\n ['def',\r\n 'BB',\r\n ['(', 'a', 'b', 'c', ')'],\r\n ':',\r\n [['BB1'], [['def', 'BBA', ['(', ')'], ':', [['bba1'], ['bba2'], ['bba3']]]]]],\r\n 'C',\r\n 'D',\r\n ['def',\r\n 'spam',\r\n ['(', 'x', 'y', ')'],\r\n ':',\r\n [[['def', 'eggs', ['(', 'z', ')'], ':', [['pass']]]]]]] \r\n \"\"\"\r\n def checkPeerIndent(s,l,t):\r\n if l >= len(s): return\r\n curCol = col(l,s)\r\n if curCol != indentStack[-1]:\r\n if curCol > indentStack[-1]:\r\n raise ParseFatalException(s,l,\"illegal nesting\")\r\n raise ParseException(s,l,\"not a peer entry\")\r\n\r\n def checkSubIndent(s,l,t):\r\n curCol = col(l,s)\r\n if curCol > indentStack[-1]:\r\n indentStack.append( curCol )\r\n else:\r\n raise ParseException(s,l,\"not a subentry\")\r\n\r\n def checkUnindent(s,l,t):\r\n if l >= len(s): return\r\n curCol = col(l,s)\r\n if not(indentStack and curCol < indentStack[-1] and curCol <= indentStack[-2]):\r\n raise ParseException(s,l,\"not an unindent\")\r\n indentStack.pop()\r\n\r\n NL = OneOrMore(LineEnd().setWhitespaceChars(\"\\t \").suppress())\r\n INDENT = (Empty() + Empty().setParseAction(checkSubIndent)).setName('INDENT')\r\n PEER = Empty().setParseAction(checkPeerIndent).setName('')\r\n UNDENT = Empty().setParseAction(checkUnindent).setName('UNINDENT')\r\n if indent:\r\n smExpr = Group( Optional(NL) +\r\n #~ FollowedBy(blockStatementExpr) +\r\n INDENT + (OneOrMore( PEER + Group(blockStatementExpr) + Optional(NL) )) + UNDENT)\r\n else:\r\n smExpr = Group( Optional(NL) +\r\n (OneOrMore( PEER + Group(blockStatementExpr) + Optional(NL) )) )\r\n blockStatementExpr.ignore(_bslash + LineEnd())\r\n return smExpr.setName('indented block')"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef copy( self ):\r\n ret = ParseResults( self.__toklist )\r\n ret.__tokdict = self.__tokdict.copy()\r\n ret.__parent = self.__parent\r\n ret.__accumNames.update( self.__accumNames )\r\n ret.__name = self.__name\r\n return ret", "response": "Returns a copy of this object."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef asXML( self, doctag=None, namedItemsOnly=False, indent=\"\", formatted=True ):\r\n nl = \"\\n\"\r\n out = []\r\n namedItems = dict((v[1],k) for (k,vlist) in self.__tokdict.items()\r\n for v in vlist)\r\n nextLevelIndent = indent + \" \"\r\n\r\n # collapse out indents if formatting is not desired\r\n if not formatted:\r\n indent = \"\"\r\n nextLevelIndent = \"\"\r\n nl = \"\"\r\n\r\n selfTag = None\r\n if doctag is not None:\r\n selfTag = doctag\r\n else:\r\n if self.__name:\r\n selfTag = self.__name\r\n\r\n if not selfTag:\r\n if namedItemsOnly:\r\n return \"\"\r\n else:\r\n selfTag = \"ITEM\"\r\n\r\n out += [ nl, indent, \"<\", selfTag, \">\" ]\r\n\r\n for i,res in enumerate(self.__toklist):\r\n if isinstance(res,ParseResults):\r\n if i in namedItems:\r\n out += [ res.asXML(namedItems[i],\r\n namedItemsOnly and doctag is None,\r\n nextLevelIndent,\r\n formatted)]\r\n else:\r\n out += [ res.asXML(None,\r\n namedItemsOnly and doctag is None,\r\n nextLevelIndent,\r\n formatted)]\r\n else:\r\n # individual token, see if there is a name for it\r\n resTag = None\r\n if i in namedItems:\r\n resTag = namedItems[i]\r\n if not resTag:\r\n if namedItemsOnly:\r\n continue\r\n else:\r\n resTag = \"ITEM\"\r\n xmlBodyText = _xml_escape(_ustr(res))\r\n out += [ nl, nextLevelIndent, \"<\", resTag, \">\",\r\n xmlBodyText,\r\n \"\" ]\r\n\r\n out += [ nl, indent, \"\" ]\r\n return \"\".join(out)", "response": "Returns the parse results as XML."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef scanString( self, instring, maxMatches=_MAX_INT, overlap=False ):\r\n if not self.streamlined:\r\n self.streamline()\r\n for e in self.ignoreExprs:\r\n e.streamline()\r\n\r\n if not self.keepTabs:\r\n instring = _ustr(instring).expandtabs()\r\n instrlen = len(instring)\r\n loc = 0\r\n preparseFn = self.preParse\r\n parseFn = self._parse\r\n ParserElement.resetCache()\r\n matches = 0\r\n try:\r\n while loc <= instrlen and matches < maxMatches:\r\n try:\r\n preloc = preparseFn( instring, loc )\r\n nextLoc,tokens = parseFn( instring, preloc, callPreParse=False )\r\n except ParseException:\r\n loc = preloc+1\r\n else:\r\n if nextLoc > loc:\r\n matches += 1\r\n yield tokens, preloc, nextLoc\r\n if overlap:\r\n nextloc = preparseFn( instring, loc )\r\n if nextloc > loc:\r\n loc = nextLoc\r\n else:\r\n loc += 1\r\n else:\r\n loc = nextLoc\r\n else:\r\n loc = preloc+1\r\n except ParseBaseException as exc:\r\n if ParserElement.verbose_stacktrace:\r\n raise\r\n else:\r\n # catch and re-raise exception from here, clears out pyparsing internal stack trace\r\n raise exc", "response": "Scan the input string for expression matches and return the list of matching words."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nsupplementing reinitialize_command to work around http://bugs.python.org/issue20819", "response": "def reinitialize_command(self, command, reinit_subcommands=0):\n \"\"\"\n Supplement reinitialize_command to work around\n http://bugs.python.org/issue20819\n \"\"\"\n cmd = self.distribution.reinitialize_command(\n command, reinit_subcommands)\n if command in ('install', 'install_lib'):\n cmd.install_lib = None\n return cmd"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ndetermining if two paths reference the same file.", "response": "def samefile(p1, p2):\n \"\"\"\n Determine if two paths reference the same file.\n\n Augments os.path.samefile to work on Windows and\n suppresses errors if the path doesn't exist.\n \"\"\"\n both_exist = os.path.exists(p1) and os.path.exists(p2)\n use_samefile = hasattr(os.path, 'samefile') and both_exist\n if use_samefile:\n return os.path.samefile(p1, p2)\n norm_p1 = os.path.normpath(os.path.normcase(p1))\n norm_p2 = os.path.normpath(os.path.normcase(p2))\n return norm_p1 == norm_p2"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn a list of site directories.", "response": "def get_site_dirs():\n \"\"\"\n Return a list of 'site' dirs\n \"\"\"\n\n sitedirs = []\n\n # start with PYTHONPATH\n sitedirs.extend(_pythonpath())\n\n prefixes = [sys.prefix]\n if sys.exec_prefix != sys.prefix:\n prefixes.append(sys.exec_prefix)\n for prefix in prefixes:\n if prefix:\n if sys.platform in ('os2emx', 'riscos'):\n sitedirs.append(os.path.join(prefix, \"Lib\", \"site-packages\"))\n elif os.sep == '/':\n sitedirs.extend([\n os.path.join(\n prefix,\n \"lib\",\n \"python\" + sys.version[:3],\n \"site-packages\",\n ),\n os.path.join(prefix, \"lib\", \"site-python\"),\n ])\n else:\n sitedirs.extend([\n prefix,\n os.path.join(prefix, \"lib\", \"site-packages\"),\n ])\n if sys.platform == 'darwin':\n # for framework builds *only* we add the standard Apple\n # locations. Currently only per-user, but /Library and\n # /Network/Library could be added too\n if 'Python.framework' in prefix:\n home = os.environ.get('HOME')\n if home:\n home_sp = os.path.join(\n home,\n 'Library',\n 'Python',\n sys.version[:3],\n 'site-packages',\n )\n sitedirs.append(home_sp)\n lib_paths = get_path('purelib'), get_path('platlib')\n for site_lib in lib_paths:\n if site_lib not in sitedirs:\n sitedirs.append(site_lib)\n\n if site.ENABLE_USER_SITE:\n sitedirs.append(site.USER_SITE)\n\n try:\n sitedirs.extend(site.getsitepackages())\n except AttributeError:\n pass\n\n sitedirs = list(map(normalize_path, sitedirs))\n\n return sitedirs"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef expand_paths(inputs):\n\n seen = {}\n\n for dirname in inputs:\n dirname = normalize_path(dirname)\n if dirname in seen:\n continue\n\n seen[dirname] = 1\n if not os.path.isdir(dirname):\n continue\n\n files = os.listdir(dirname)\n yield dirname, files\n\n for name in files:\n if not name.endswith('.pth'):\n # We only care about the .pth files\n continue\n if name in ('easy-install.pth', 'setuptools.pth'):\n # Ignore .pth files that we control\n continue\n\n # Read the .pth file\n f = open(os.path.join(dirname, name))\n lines = list(yield_lines(f))\n f.close()\n\n # Yield existing non-dupe, non-import directory lines from it\n for line in lines:\n if not line.startswith(\"import\"):\n line = normalize_path(line.rstrip())\n if line not in seen:\n seen[line] = 1\n if not os.path.isdir(line):\n continue\n yield line, os.listdir(line)", "response": "Yields sys. path directories that might contain old - style packages"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_exe_prefixes(exe_filename):\n\n prefixes = [\n ('PURELIB/', ''),\n ('PLATLIB/pywin32_system32', ''),\n ('PLATLIB/', ''),\n ('SCRIPTS/', 'EGG-INFO/scripts/'),\n ('DATA/lib/site-packages', ''),\n ]\n z = zipfile.ZipFile(exe_filename)\n try:\n for info in z.infolist():\n name = info.filename\n parts = name.split('/')\n if len(parts) == 3 and parts[2] == 'PKG-INFO':\n if parts[1].endswith('.egg-info'):\n prefixes.insert(0, ('/'.join(parts[:2]), 'EGG-INFO/'))\n break\n if len(parts) != 2 or not name.endswith('.pth'):\n continue\n if name.endswith('-nspkg.pth'):\n continue\n if parts[0].upper() in ('PURELIB', 'PLATLIB'):\n contents = z.read(name)\n if six.PY3:\n contents = contents.decode()\n for pth in yield_lines(contents):\n pth = pth.strip().replace('\\\\', '/')\n if not pth.startswith('import'):\n prefixes.append((('%s/%s/' % (parts[0], pth)), ''))\n finally:\n z.close()\n prefixes = [(x.lower(), y) for x, y in prefixes]\n prefixes.sort()\n prefixes.reverse()\n return prefixes", "response": "Get a list of exe - > egg path translations for a given. exe file"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _first_line_re():\n if isinstance(first_line_re.pattern, str):\n return first_line_re\n\n # first_line_re in Python >=3.1.4 and >=3.2.1 is a bytes pattern.\n return re.compile(first_line_re.pattern.decode())", "response": "Return a regular expression based on first_line_re suitable for matching\n strings."} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nupdates dist_path related caches.", "response": "def update_dist_caches(dist_path, fix_zipimporter_caches):\n \"\"\"\n Fix any globally cached `dist_path` related data\n\n `dist_path` should be a path of a newly installed egg distribution (zipped\n or unzipped).\n\n sys.path_importer_cache contains finder objects that have been cached when\n importing data from the original distribution. Any such finders need to be\n cleared since the replacement distribution might be packaged differently,\n e.g. a zipped egg distribution might get replaced with an unzipped egg\n folder or vice versa. Having the old finders cached may then cause Python\n to attempt loading modules from the replacement distribution using an\n incorrect loader.\n\n zipimport.zipimporter objects are Python loaders charged with importing\n data packaged inside zip archives. If stale loaders referencing the\n original distribution, are left behind, they can fail to load modules from\n the replacement distribution. E.g. if an old zipimport.zipimporter instance\n is used to load data from a new zipped egg archive, it may cause the\n operation to attempt to locate the requested data in the wrong location -\n one indicated by the original distribution's zip archive directory\n information. Such an operation may then fail outright, e.g. report having\n read a 'bad local file header', or even worse, it may fail silently &\n return invalid data.\n\n zipimport._zip_directory_cache contains cached zip archive directory\n information for all existing zipimport.zipimporter instances and all such\n instances connected to the same archive share the same cached directory\n information.\n\n If asked, and the underlying Python implementation allows it, we can fix\n all existing zipimport.zipimporter instances instead of having to track\n them down and remove them one by one, by updating their shared cached zip\n archive directory information. This, of course, assumes that the\n replacement distribution is packaged as a zipped egg.\n\n If not asked to fix existing zipimport.zipimporter instances, we still do\n our best to clear any remaining zipimport.zipimporter related cached data\n that might somehow later get used when attempting to load data from the new\n distribution and thus cause such load operations to fail. Note that when\n tracking down such remaining stale data, we can not catch every conceivable\n usage from here, and we clear only those that we know of and have found to\n cause problems if left alive. Any remaining caches should be updated by\n whomever is in charge of maintaining them, i.e. they should be ready to\n handle us replacing their zip archives with new distributions at runtime.\n\n \"\"\"\n # There are several other known sources of stale zipimport.zipimporter\n # instances that we do not clear here, but might if ever given a reason to\n # do so:\n # * Global setuptools pkg_resources.working_set (a.k.a. 'master working\n # set') may contain distributions which may in turn contain their\n # zipimport.zipimporter loaders.\n # * Several zipimport.zipimporter loaders held by local variables further\n # up the function call stack when running the setuptools installation.\n # * Already loaded modules may have their __loader__ attribute set to the\n # exact loader instance used when importing them. Python 3.4 docs state\n # that this information is intended mostly for introspection and so is\n # not expected to cause us problems.\n normalized_path = normalize_path(dist_path)\n _uncache(normalized_path, sys.path_importer_cache)\n if fix_zipimporter_caches:\n _replace_zip_directory_cache_data(normalized_path)\n else:\n # Here, even though we do not want to fix existing and now stale\n # zipimporter cache information, we still want to remove it. Related to\n # Python's zip archive directory information cache, we clear each of\n # its stale entries in two phases:\n # 1. Clear the entry so attempting to access zip archive information\n # via any existing stale zipimport.zipimporter instances fails.\n # 2. Remove the entry from the cache so any newly constructed\n # zipimport.zipimporter instances do not end up using old stale\n # zip archive directory information.\n # This whole stale data removal step does not seem strictly necessary,\n # but has been left in because it was done before we started replacing\n # the zip archive directory information cache content if possible, and\n # there are no relevant unit tests that we can depend on to tell us if\n # this is really needed.\n _remove_and_clear_zip_directory_cache_data(normalized_path)"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _collect_zipimporter_cache_entries(normalized_path, cache):\n result = []\n prefix_len = len(normalized_path)\n for p in cache:\n np = normalize_path(p)\n if (np.startswith(normalized_path) and\n np[prefix_len:prefix_len + 1] in (os.sep, '')):\n result.append(p)\n return result", "response": "Collect the zipimporter cache entries related to a given normalized path."} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nis this string a valid Python script?", "response": "def is_python(text, filename=''):\n \"Is this string a valid Python script?\"\n try:\n compile(text, filename, 'exec')\n except (SyntaxError, TypeError):\n return False\n else:\n return True"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ndetermines if the specified executable is a. sh.", "response": "def is_sh(executable):\n \"\"\"Determine if the specified executable is a .sh (contains a #! line)\"\"\"\n try:\n with io.open(executable, encoding='latin-1') as fp:\n magic = fp.read(2)\n except (OSError, IOError):\n return executable\n return magic == '#!'"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef is_python_script(script_text, filename):\n if filename.endswith('.py') or filename.endswith('.pyw'):\n return True # extension says it's Python\n if is_python(script_text, filename):\n return True # it's syntactically valid Python\n if script_text.startswith('#!'):\n # It begins with a '#!' line, so check if 'python' is in it somewhere\n return 'python' in script_text.splitlines()[0].lower()\n\n return False", "response": "Is this text a whole a Python script?"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_win_launcher(type):\n launcher_fn = '%s.exe' % type\n if is_64bit():\n launcher_fn = launcher_fn.replace(\".\", \"-64.\")\n else:\n launcher_fn = launcher_fn.replace(\".\", \"-32.\")\n return resource_string('setuptools', launcher_fn)", "response": "Load the Windows launcher suitable for launching a script."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nrendering the Setuptools version and installation details then exit.", "response": "def _render_version():\n \"\"\"\n Render the Setuptools version and installation details, then exit.\n \"\"\"\n ver = sys.version[:3]\n dist = get_distribution('setuptools')\n tmpl = 'setuptools {dist.version} from {dist.location} (Python {ver})'\n print(tmpl.format(**locals()))\n raise SystemExit()"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _fix_install_dir_for_user_site(self):\n if not self.user or not site.ENABLE_USER_SITE:\n return\n\n self.create_home_path()\n if self.install_userbase is None:\n msg = \"User base directory is not specified\"\n raise DistutilsPlatformError(msg)\n self.install_base = self.install_platbase = self.install_userbase\n scheme_name = os.name.replace('posix', 'unix') + '_user'\n self.select_scheme(scheme_name)", "response": "Fix the install_dir for user site."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nchecking whether. pth files are supported in the current directory.", "response": "def check_pth_processing(self):\n \"\"\"Empirically verify whether .pth files are supported in inst. dir\"\"\"\n instdir = self.install_dir\n log.info(\"Checking .pth file support in %s\", instdir)\n pth_file = self.pseudo_tempname() + \".pth\"\n ok_file = pth_file + '.ok'\n ok_exists = os.path.exists(ok_file)\n tmpl = _one_liner(\"\"\"\n import os\n f = open({ok_file!r}, 'w')\n f.write('OK')\n f.close()\n \"\"\") + '\\n'\n try:\n if ok_exists:\n os.unlink(ok_file)\n dirname = os.path.dirname(ok_file)\n pkg_resources.py31compat.makedirs(dirname, exist_ok=True)\n f = open(pth_file, 'w')\n except (OSError, IOError):\n self.cant_write_to_target()\n else:\n try:\n f.write(tmpl.format(**locals()))\n f.close()\n f = None\n executable = sys.executable\n if os.name == 'nt':\n dirname, basename = os.path.split(executable)\n alt = os.path.join(dirname, 'pythonw.exe')\n use_alt = (\n basename.lower() == 'python.exe' and\n os.path.exists(alt)\n )\n if use_alt:\n # use pythonw.exe to avoid opening a console window\n executable = alt\n\n from distutils.spawn import spawn\n\n spawn([executable, '-E', '-c', 'pass'], 0)\n\n if os.path.exists(ok_file):\n log.info(\n \"TEST PASSED: %s appears to support .pth files\",\n instdir\n )\n return True\n finally:\n if f:\n f.close()\n if os.path.exists(ok_file):\n os.unlink(ok_file)\n if os.path.exists(pth_file):\n os.unlink(pth_file)\n if not self.multi_version:\n log.warn(\"TEST FAILED: %s does NOT support .pth files\", instdir)\n return False"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef install_egg_scripts(self, dist):\n if not self.exclude_scripts and dist.metadata_isdir('scripts'):\n for script_name in dist.metadata_listdir('scripts'):\n if dist.metadata_isdir('scripts/' + script_name):\n # The \"script\" is a directory, likely a Python 3\n # __pycache__ directory, so skip it.\n continue\n self.install_script(\n dist, script_name,\n dist.get_metadata('scripts/' + script_name)\n )\n self.install_wrapper_scripts(dist)", "response": "Write all the scripts for dist unless scripts are excluded"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef select_scheme(self, name):\n # it's the caller's problem if they supply a bad name!\n scheme = INSTALL_SCHEMES[name]\n for key in SCHEME_KEYS:\n attrname = 'install_' + key\n if getattr(self, attrname) is None:\n setattr(self, attrname, scheme[key])", "response": "Sets the install directories by applying the install schemes."} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nwrites an executable file to the scripts directory", "response": "def write_script(self, script_name, contents, mode=\"t\", blockers=()):\n \"\"\"Write an executable file to the scripts directory\"\"\"\n self.delete_blockers( # clean up old .py/.pyw w/o a script\n [os.path.join(self.script_dir, x) for x in blockers]\n )\n log.info(\"Installing %s script to %s\", script_name, self.script_dir)\n target = os.path.join(self.script_dir, script_name)\n self.add_output(target)\n\n if self.dry_run:\n return\n\n mask = current_umask()\n ensure_directory(target)\n if os.path.exists(target):\n os.unlink(target)\n with open(target, \"w\" + mode) as f:\n f.write(contents)\n chmod(target, 0o777 - mask)"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nextracting a bdist_wininst to the directories an egg would use", "response": "def exe_to_egg(self, dist_filename, egg_tmp):\n \"\"\"Extract a bdist_wininst to the directories an egg would use\"\"\"\n # Check for .pth file and set up prefix translations\n prefixes = get_exe_prefixes(dist_filename)\n to_compile = []\n native_libs = []\n top_level = {}\n\n def process(src, dst):\n s = src.lower()\n for old, new in prefixes:\n if s.startswith(old):\n src = new + src[len(old):]\n parts = src.split('/')\n dst = os.path.join(egg_tmp, *parts)\n dl = dst.lower()\n if dl.endswith('.pyd') or dl.endswith('.dll'):\n parts[-1] = bdist_egg.strip_module(parts[-1])\n top_level[os.path.splitext(parts[0])[0]] = 1\n native_libs.append(src)\n elif dl.endswith('.py') and old != 'SCRIPTS/':\n top_level[os.path.splitext(parts[0])[0]] = 1\n to_compile.append(dst)\n return dst\n if not src.endswith('.pth'):\n log.warn(\"WARNING: can't process %s\", src)\n return None\n\n # extract, tracking .pyd/.dll->native_libs and .py -> to_compile\n unpack_archive(dist_filename, egg_tmp, process)\n stubs = []\n for res in native_libs:\n if res.lower().endswith('.pyd'): # create stubs for .pyd's\n parts = res.split('/')\n resource = parts[-1]\n parts[-1] = bdist_egg.strip_module(parts[-1]) + '.py'\n pyfile = os.path.join(egg_tmp, *parts)\n to_compile.append(pyfile)\n stubs.append(pyfile)\n bdist_egg.write_stub(resource, pyfile)\n self.byte_compile(to_compile) # compile .py's\n bdist_egg.write_safety_flag(\n os.path.join(egg_tmp, 'EGG-INFO'),\n bdist_egg.analyze_egg(egg_tmp, stubs)) # write zip-safety flag\n\n for name in 'top_level', 'native_libs':\n if locals()[name]:\n txt = os.path.join(egg_tmp, 'EGG-INFO', name + '.txt')\n if not os.path.exists(txt):\n f = open(txt, 'w')\n f.write('\\n'.join(locals()[name]) + '\\n')\n f.close()"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef add(self, dist):\n new_path = (\n dist.location not in self.paths and (\n dist.location not in self.sitedirs or\n # account for '.' being in PYTHONPATH\n dist.location == os.getcwd()\n )\n )\n if new_path:\n self.paths.append(dist.location)\n self.dirty = True\n Environment.add(self, dist)", "response": "Add dist to the distribution map"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef remove(self, dist):\n while dist.location in self.paths:\n self.paths.remove(dist.location)\n self.dirty = True\n Environment.remove(self, dist)", "response": "Remove dist from the distribution map."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nconstructing a CommandSpec from a parameter.", "response": "def from_param(cls, param):\n \"\"\"\n Construct a CommandSpec from a parameter to build_scripts, which may\n be None.\n \"\"\"\n if isinstance(param, cls):\n return param\n if isinstance(param, list):\n return cls(param)\n if param is None:\n return cls.from_environment()\n # otherwise, assume it's a string.\n return cls.from_string(param)"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nconstruct a command spec from a simple string representing a command line parseable by shlex. split.", "response": "def from_string(cls, string):\n \"\"\"\n Construct a command spec from a simple string representing a command\n line parseable by shlex.split.\n \"\"\"\n items = shlex.split(string, **cls.split_args)\n return cls(items)"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _extract_options(orig_script):\n first = (orig_script + '\\n').splitlines()[0]\n match = _first_line_re().match(first)\n options = match.group(1) or '' if match else ''\n return options.strip()", "response": "Extract any options from the first line of the script."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nyielding a list of write_script argument tuples for a distribution s console_scripts and gui_scripts entry points.", "response": "def get_args(cls, dist, header=None):\n \"\"\"\n Yield write_script() argument tuples for a distribution's\n console_scripts and gui_scripts entry points.\n \"\"\"\n if header is None:\n header = cls.get_header()\n spec = str(dist.as_requirement())\n for type_ in 'console', 'gui':\n group = type_ + '_scripts'\n for name, ep in dist.get_entry_map(group).items():\n cls._ensure_safe_name(name)\n script_text = cls.template % locals()\n args = cls._get_script_args(type_, name, header, script_text)\n for res in args:\n yield res"} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nselect the best ScriptWriter for this environment.", "response": "def best(cls):\n \"\"\"\n Select the best ScriptWriter for this environment.\n \"\"\"\n if sys.platform == 'win32' or (os.name == 'java' and os._name == 'nt'):\n return WindowsScriptWriter.best()\n else:\n return cls"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_header(cls, script_text=\"\", executable=None):\n cmd = cls.command_spec_class.best().from_param(executable)\n cmd.install_options(script_text)\n return cmd.as_header()", "response": "Create a header line from script_text"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef best(cls):\n writer_lookup = dict(\n executable=WindowsExecutableLauncherWriter,\n natural=cls,\n )\n # for compatibility, use the executable launcher by default\n launcher = os.environ.get('SETUPTOOLS_LAUNCHER', 'executable')\n return writer_lookup[launcher]", "response": "Select the best ScriptWriter suitable for Windows\n "} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _get_script_args(cls, type_, name, header, script_text):\n \"For Windows, add a .py extension\"\n ext = dict(console='.pya', gui='.pyw')[type_]\n if ext not in os.environ['PATHEXT'].lower().split(';'):\n msg = (\n \"{ext} not listed in PATHEXT; scripts will not be \"\n \"recognized as executables.\"\n ).format(**locals())\n warnings.warn(msg, UserWarning)\n old = ['.pya', '.py', '-script.py', '.pyc', '.pyo', '.pyw', '.exe']\n old.remove(ext)\n header = cls._adjust_header(type_, header)\n blockers = [name + x for x in old]\n yield name + ext, header + script_text, 't', blockers", "response": "For Windows add a. py extension"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _adjust_header(cls, type_, orig_header):\n pattern = 'pythonw.exe'\n repl = 'python.exe'\n if type_ == 'gui':\n pattern, repl = repl, pattern\n pattern_ob = re.compile(re.escape(pattern), re.IGNORECASE)\n new_header = pattern_ob.sub(string=orig_header, repl=repl)\n return new_header if cls._use_header(new_header) else orig_header", "response": "Adjust the header for the current version of the log file."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _get_script_args(cls, type_, name, header, script_text):\n if type_ == 'gui':\n launcher_type = 'gui'\n ext = '-script.pyw'\n old = ['.pyw']\n else:\n launcher_type = 'cli'\n ext = '-script.py'\n old = ['.py', '.pyc', '.pyo']\n hdr = cls._adjust_header(type_, header)\n blockers = [name + x for x in old]\n yield (name + ext, hdr + script_text, 't', blockers)\n yield (\n name + '.exe', get_win_launcher(launcher_type),\n 'b' # write in binary mode\n )\n if not is_64bit():\n # install a manifest for the launcher to prevent Windows\n # from detecting it as an installer (which it will for\n # launchers like easy_install.exe). Consider only\n # adding a manifest for launchers detected as installers.\n # See Distribute #143 for details.\n m_name = name + '.exe.manifest'\n yield (m_name, load_launcher_manifest(name), 't')", "response": "Yields the command line arguments for the specified script."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _load_password_from_keyring(self):\n try:\n keyring = __import__('keyring')\n return keyring.get_password(self.repository, self.username)\n except Exception:\n pass", "response": "Attempt to load password from keyring. Suppress Exceptions."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _to_str(s):\n if sys.version_info[0] == 2 and not isinstance(s, str):\n # Assume it's Unicode, as that's what the PEP says\n # should be provided.\n return s.encode(sys.getfilesystemencoding())\n return s", "response": "Convert a filename to a string."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\npatch distutils. dist. Distribution with this class.", "response": "def patch(cls):\n \"\"\"\n Replace\n distutils.dist.Distribution with this class\n for the duration of this context.\n \"\"\"\n orig = distutils.core.Distribution\n distutils.core.Distribution = cls\n try:\n yield\n finally:\n distutils.core.Distribution = orig"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nrunning the script in sys. argv [ 1 ] as if it had been invoked naturally.", "response": "def run():\n \"\"\"\n Run the script in sys.argv[1] as if it had\n been invoked naturally.\n \"\"\"\n __builtins__\n script_name = sys.argv[1]\n namespace = dict(\n __file__=script_name,\n __name__='__main__',\n __doc__=None,\n )\n sys.argv[:] = sys.argv[1:]\n\n open_ = getattr(tokenize, 'open', open)\n script = open_(script_name).read()\n norm_script = script.replace('\\\\r\\\\n', '\\\\n')\n code = compile(norm_script, script_name, 'exec')\n exec(code, namespace)"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nbuilds modules packages and copy data files to build directory", "response": "def run(self):\n \"\"\"Build modules, packages, and copy data files to build directory\"\"\"\n if not self.py_modules and not self.packages:\n return\n\n if self.py_modules:\n self.build_modules()\n\n if self.packages:\n self.build_packages()\n self.build_package_data()\n\n self.run_2to3(self.__updated_files, False)\n self.run_2to3(self.__updated_files, True)\n self.run_2to3(self.__doctests_2to3, True)\n\n # Only compile actual .py files, using our base class' idea of what our\n # output files are.\n self.byte_compile(orig.build_py.get_outputs(self, include_bytecode=0))"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _get_data_files(self):\n self.analyze_manifest()\n return list(map(self._get_pkg_data_files, self.packages or ()))", "response": "Generate list of ( package src_dir build_dir filenames ) tuples"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns filenames for package s data files in src_dir", "response": "def find_data_files(self, package, src_dir):\n \"\"\"Return filenames for package's data files in 'src_dir'\"\"\"\n patterns = self._get_platform_patterns(\n self.package_data,\n package,\n src_dir,\n )\n globs_expanded = map(glob, patterns)\n # flatten the expanded globs into an iterable of matches\n globs_matches = itertools.chain.from_iterable(globs_expanded)\n glob_files = filter(os.path.isfile, globs_matches)\n files = itertools.chain(\n self.manifest_files.get(package, []),\n glob_files,\n )\n return self.exclude_data_files(package, src_dir, files)"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef build_package_data(self):\n for package, src_dir, build_dir, filenames in self.data_files:\n for filename in filenames:\n target = os.path.join(build_dir, filename)\n self.mkpath(os.path.dirname(target))\n srcfile = os.path.join(src_dir, filename)\n outf, copied = self.copy_file(srcfile, target)\n srcfile = os.path.abspath(srcfile)\n if (copied and\n srcfile in self.distribution.convert_2to3_doctests):\n self.__doctests_2to3.append(outf)", "response": "Copy data files into build directory"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef check_package(self, package, package_dir):\n try:\n return self.packages_checked[package]\n except KeyError:\n pass\n\n init_py = orig.build_py.check_package(self, package, package_dir)\n self.packages_checked[package] = init_py\n\n if not init_py or not self.distribution.namespace_packages:\n return init_py\n\n for pkg in self.distribution.namespace_packages:\n if pkg == package or pkg.startswith(package + '.'):\n break\n else:\n return init_py\n\n with io.open(init_py, 'rb') as f:\n contents = f.read()\n if b'declare_namespace' not in contents:\n raise distutils.errors.DistutilsError(\n \"Namespace package problem: %s is a namespace package, but \"\n \"its\\n__init__.py does not call declare_namespace()! Please \"\n 'fix it.\\n(See the setuptools manual under '\n '\"Namespace Packages\" for details.)\\n\"' % (package,)\n )\n return init_py", "response": "Check namespace packages __init__ for declare_namespace"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef exclude_data_files(self, package, src_dir, files):\n files = list(files)\n patterns = self._get_platform_patterns(\n self.exclude_package_data,\n package,\n src_dir,\n )\n match_groups = (\n fnmatch.filter(files, pattern)\n for pattern in patterns\n )\n # flatten the groups of matches into an iterable of matches\n matches = itertools.chain.from_iterable(match_groups)\n bad = set(matches)\n keepers = (\n fn\n for fn in files\n if fn not in bad\n )\n # ditch dupes\n return list(_unique_everseen(keepers))", "response": "Filter filenames for package s data files in src_dir"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nyields a list of platform - specific path patterns from a glob - based spec.", "response": "def _get_platform_patterns(spec, package, src_dir):\n \"\"\"\n yield platform-specific path patterns (suitable for glob\n or fn_match) from a glob-based spec (such as\n self.package_data or self.exclude_package_data)\n matching package in src_dir.\n \"\"\"\n raw_patterns = itertools.chain(\n spec.get('', []),\n spec.get(package, []),\n )\n return (\n # Each pattern has to be converted to a platform-specific path\n os.path.join(src_dir, convert_path(pattern))\n for pattern in raw_patterns\n )"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef require_metadata():\n \"Prevent improper installs without necessary metadata. See #659\"\n egg_info_dir = os.path.join(here, 'setuptools.egg-info')\n if not os.path.exists(egg_info_dir):\n msg = (\n \"Cannot build setuptools without metadata. \"\n \"Run `bootstrap.py`.\"\n )\n raise RuntimeError(msg)", "response": "Prevent improper installs without necessary metadata. See #659"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ngive the filename including md5 fragment construct the dependency link for PyPI.", "response": "def pypi_link(pkg_filename):\n \"\"\"\n Given the filename, including md5 fragment, construct the\n dependency link for PyPI.\n \"\"\"\n root = 'https://files.pythonhosted.org/packages/source'\n name, sep, rest = pkg_filename.partition('-')\n parts = root, name[0], name, pkg_filename\n return '/'.join(parts)"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef glibc_version_string():\n \"Returns glibc version string, or None if not using glibc.\"\n\n # ctypes.CDLL(None) internally calls dlopen(NULL), and as the dlopen\n # manpage says, \"If filename is NULL, then the returned handle is for the\n # main program\". This way we can let the linker do the work to figure out\n # which libc our process is actually using.\n process_namespace = ctypes.CDLL(None)\n try:\n gnu_get_libc_version = process_namespace.gnu_get_libc_version\n except AttributeError:\n # Symbol doesn't exist -> therefore, we are not linked to\n # glibc.\n return None\n\n # Call gnu_get_libc_version, which returns a string like \"2.5\"\n gnu_get_libc_version.restype = ctypes.c_char_p\n version_str = gnu_get_libc_version()\n # py2 / py3 compatibility:\n if not isinstance(version_str, str):\n version_str = version_str.decode(\"ascii\")\n\n return version_str", "response": "Returns glibc version string or None if not using glibc."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef unpack_archive(filename, extract_dir, progress_filter=default_filter,\n drivers=None):\n \"\"\"Unpack `filename` to `extract_dir`, or raise ``UnrecognizedFormat``\n\n `progress_filter` is a function taking two arguments: a source path\n internal to the archive ('/'-separated), and a filesystem path where it\n will be extracted. The callback must return the desired extract path\n (which may be the same as the one passed in), or else ``None`` to skip\n that file or directory. The callback can thus be used to report on the\n progress of the extraction, as well as to filter the items extracted or\n alter their extraction paths.\n\n `drivers`, if supplied, must be a non-empty sequence of functions with the\n same signature as this function (minus the `drivers` argument), that raise\n ``UnrecognizedFormat`` if they do not support extracting the designated\n archive type. The `drivers` are tried in sequence until one is found that\n does not raise an error, or until all are exhausted (in which case\n ``UnrecognizedFormat`` is raised). If you do not supply a sequence of\n drivers, the module's ``extraction_drivers`` constant will be used, which\n means that ``unpack_zipfile`` and ``unpack_tarfile`` will be tried, in that\n order.\n \"\"\"\n for driver in drivers or extraction_drivers:\n try:\n driver(filename, extract_dir, progress_filter)\n except UnrecognizedFormat:\n continue\n else:\n return\n else:\n raise UnrecognizedFormat(\n \"Not a recognized archive type: %s\" % filename\n )", "response": "Unpacks a file to extract_dir."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef unpack_zipfile(filename, extract_dir, progress_filter=default_filter):\n\n if not zipfile.is_zipfile(filename):\n raise UnrecognizedFormat(\"%s is not a zip file\" % (filename,))\n\n with zipfile.ZipFile(filename) as z:\n for info in z.infolist():\n name = info.filename\n\n # don't extract absolute paths or ones with .. in them\n if name.startswith('/') or '..' in name.split('/'):\n continue\n\n target = os.path.join(extract_dir, *name.split('/'))\n target = progress_filter(name, target)\n if not target:\n continue\n if name.endswith('/'):\n # directory\n ensure_directory(target)\n else:\n # file\n ensure_directory(target)\n data = z.read(info.filename)\n with open(target, 'wb') as f:\n f.write(data)\n unix_attributes = info.external_attr >> 16\n if unix_attributes:\n os.chmod(target, unix_attributes)", "response": "Unpacks a zip file to extract_dir."} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nunpack a tar file to extract_dir.", "response": "def unpack_tarfile(filename, extract_dir, progress_filter=default_filter):\n \"\"\"Unpack tar/tar.gz/tar.bz2 `filename` to `extract_dir`\n\n Raises ``UnrecognizedFormat`` if `filename` is not a tarfile (as determined\n by ``tarfile.open()``). See ``unpack_archive()`` for an explanation\n of the `progress_filter` argument.\n \"\"\"\n try:\n tarobj = tarfile.open(filename)\n except tarfile.TarError:\n raise UnrecognizedFormat(\n \"%s is not a compressed or uncompressed tar file\" % (filename,)\n )\n with contextlib.closing(tarobj):\n # don't do any chowning!\n tarobj.chown = lambda *args: None\n for member in tarobj:\n name = member.name\n # don't extract absolute paths or ones with .. in them\n if not name.startswith('/') and '..' not in name.split('/'):\n prelim_dst = os.path.join(extract_dir, *name.split('/'))\n\n # resolve any links and to extract the link targets as normal\n # files\n while member is not None and (member.islnk() or member.issym()):\n linkpath = member.linkname\n if member.issym():\n base = posixpath.dirname(member.name)\n linkpath = posixpath.join(base, linkpath)\n linkpath = posixpath.normpath(linkpath)\n member = tarobj._getmember(linkpath)\n\n if member is not None and (member.isfile() or member.isdir()):\n final_dst = progress_filter(name, prelim_dst)\n if final_dst:\n if final_dst.endswith(os.sep):\n final_dst = final_dst[:-1]\n try:\n # XXX Ugh\n tarobj._extract_member(member, final_dst)\n except tarfile.ExtractError:\n # chown/chmod/mkfifo/mknode/makedev failed\n pass\n return True"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\npatch distutils. msvc9compiler. find_vcvarsall to use the standalone compiler build for Python 2. 7.", "response": "def msvc9_find_vcvarsall(version):\n \"\"\"\n Patched \"distutils.msvc9compiler.find_vcvarsall\" to use the standalone\n compiler build for Python (VCForPython). Fall back to original behavior\n when the standalone compiler is not available.\n\n Redirect the path of \"vcvarsall.bat\".\n\n Known supported compilers\n -------------------------\n Microsoft Visual C++ 9.0:\n Microsoft Visual C++ Compiler for Python 2.7 (x86, amd64)\n\n Parameters\n ----------\n version: float\n Required Microsoft Visual C++ version.\n\n Return\n ------\n vcvarsall.bat path: str\n \"\"\"\n VC_BASE = r'Software\\%sMicrosoft\\DevDiv\\VCForPython\\%0.1f'\n key = VC_BASE % ('', version)\n try:\n # Per-user installs register the compiler path here\n productdir = Reg.get_value(key, \"installdir\")\n except KeyError:\n try:\n # All-user installs on a 64-bit system register here\n key = VC_BASE % ('Wow6432Node\\\\', version)\n productdir = Reg.get_value(key, \"installdir\")\n except KeyError:\n productdir = None\n\n if productdir:\n vcvarsall = os.path.os.path.join(productdir, \"vcvarsall.bat\")\n if os.path.isfile(vcvarsall):\n return vcvarsall\n\n return get_unpatched(msvc9_find_vcvarsall)(version)"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef msvc9_query_vcvarsall(ver, arch='x86', *args, **kwargs):\n # Try to get environement from vcvarsall.bat (Classical way)\n try:\n orig = get_unpatched(msvc9_query_vcvarsall)\n return orig(ver, arch, *args, **kwargs)\n except distutils.errors.DistutilsPlatformError:\n # Pass error if Vcvarsall.bat is missing\n pass\n except ValueError:\n # Pass error if environment not set after executing vcvarsall.bat\n pass\n\n # If error, try to set environment directly\n try:\n return EnvironmentInfo(arch, ver).return_env()\n except distutils.errors.DistutilsPlatformError as exc:\n _augment_exception(exc, ver, arch)\n raise", "response": "Get environment info from vcvarsall. bat."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef msvc14_get_vc_env(plat_spec):\n # Try to get environment from vcvarsall.bat (Classical way)\n try:\n return get_unpatched(msvc14_get_vc_env)(plat_spec)\n except distutils.errors.DistutilsPlatformError:\n # Pass error Vcvarsall.bat is missing\n pass\n\n # If error, try to set environment directly\n try:\n return EnvironmentInfo(plat_spec, vc_min_ver=14.0).return_env()\n except distutils.errors.DistutilsPlatformError as exc:\n _augment_exception(exc, 14.0)\n raise", "response": "Get environment from vcvarsall. bat."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\npatch distutils. ccompiler. gen_lib_options for Numpy < 1. 11. 2", "response": "def msvc14_gen_lib_options(*args, **kwargs):\n \"\"\"\n Patched \"distutils._msvccompiler.gen_lib_options\" for fix\n compatibility between \"numpy.distutils\" and \"distutils._msvccompiler\"\n (for Numpy < 1.11.2)\n \"\"\"\n if \"numpy.distutils\" in sys.modules:\n import numpy as np\n if LegacyVersion(np.__version__) < LegacyVersion('1.11.2'):\n return np.distutils.ccompiler.gen_lib_options(*args, **kwargs)\n return get_unpatched(msvc14_gen_lib_options)(*args, **kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\naugmenting the exception message with details to help guide the user to resolve it.", "response": "def _augment_exception(exc, version, arch=''):\n \"\"\"\n Add details to the exception message to help guide the user\n as to what action will resolve it.\n \"\"\"\n # Error if MSVC++ directory not found or environment not set\n message = exc.args[0]\n\n if \"vcvarsall\" in message.lower() or \"visual c\" in message.lower():\n # Special error message if MSVC++ not installed\n tmpl = 'Microsoft Visual C++ {version:0.1f} is required.'\n message = tmpl.format(**locals())\n msdownload = 'www.microsoft.com/download/details.aspx?id=%d'\n if version == 9.0:\n if arch.lower().find('ia64') > -1:\n # For VC++ 9.0, if IA64 support is needed, redirect user\n # to Windows SDK 7.0\n message += ' Get it with \"Microsoft Windows SDK 7.0\": '\n message += msdownload % 3138\n else:\n # For VC++ 9.0 redirect user to Vc++ for Python 2.7 :\n # This redirection link is maintained by Microsoft.\n # Contact vspython@microsoft.com if it needs updating.\n message += ' Get it from http://aka.ms/vcpython27'\n elif version == 10.0:\n # For VC++ 10.0 Redirect user to Windows SDK 7.1\n message += ' Get it with \"Microsoft Windows SDK 7.1\": '\n message += msdownload % 8279\n elif version >= 14.0:\n # For VC++ 14.0 Redirect user to Visual C++ Build Tools\n message += (' Get it with \"Microsoft Visual C++ Build Tools\": '\n r'https://visualstudio.microsoft.com/downloads/')\n\n exc.args = (message, )"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning the current directory of the current architectural system.", "response": "def current_dir(self, hidex86=False, x64=False):\n \"\"\"\n Current platform specific subfolder.\n\n Parameters\n ----------\n hidex86: bool\n return '' and not '\\x86' if architecture is x86.\n x64: bool\n return '\\x64' and not '\\amd64' if architecture is amd64.\n\n Return\n ------\n subfolder: str\n '\\target', or '' (see hidex86 parameter)\n \"\"\"\n return (\n '' if (self.current_cpu == 'x86' and hidex86) else\n r'\\x64' if (self.current_cpu == 'amd64' and x64) else\n r'\\%s' % self.current_cpu\n )"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef cross_dir(self, forcex86=False):\n current = 'x86' if forcex86 else self.current_cpu\n return (\n '' if self.target_cpu == current else\n self.target_dir().replace('\\\\', '\\\\%s_' % current)\n )", "response": "r Returns the cross platform specific subfolder."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning the path where look.", "response": "def microsoft(self, key, x86=False):\n \"\"\"\n Return key in Microsoft software registry.\n\n Parameters\n ----------\n key: str\n Registry key path where look.\n x86: str\n Force x86 software registry.\n\n Return\n ------\n str: value\n \"\"\"\n node64 = '' if self.pi.current_is_x86() or x86 else 'Wow6432Node'\n return os.path.join('Software', node64, 'Microsoft', key)"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef lookup(self, key, name):\n KEY_READ = winreg.KEY_READ\n openkey = winreg.OpenKey\n ms = self.microsoft\n for hkey in self.HKEYS:\n try:\n bkey = openkey(hkey, ms(key), 0, KEY_READ)\n except (OSError, IOError):\n if not self.pi.current_is_x86():\n try:\n bkey = openkey(hkey, ms(key, True), 0, KEY_READ)\n except (OSError, IOError):\n continue\n else:\n continue\n try:\n return winreg.QueryValueEx(bkey, name)[0]\n except (OSError, IOError):\n pass", "response": "Look for values in Microsoft software registry in Microsoft software registry."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef find_available_vc_vers(self):\n ms = self.ri.microsoft\n vckeys = (self.ri.vc, self.ri.vc_for_python, self.ri.vs)\n vc_vers = []\n for hkey in self.ri.HKEYS:\n for key in vckeys:\n try:\n bkey = winreg.OpenKey(hkey, ms(key), 0, winreg.KEY_READ)\n except (OSError, IOError):\n continue\n subkeys, values, _ = winreg.QueryInfoKey(bkey)\n for i in range(values):\n try:\n ver = float(winreg.EnumValue(bkey, i)[0])\n if ver not in vc_vers:\n vc_vers.append(ver)\n except ValueError:\n pass\n for i in range(subkeys):\n try:\n ver = float(winreg.EnumKey(bkey, i))\n if ver not in vc_vers:\n vc_vers.append(ver)\n except ValueError:\n pass\n return sorted(vc_vers)", "response": "Find all available Microsoft Visual C ++ versions."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _guess_vc(self):\n if self.vc_ver <= 14.0:\n return\n\n default = r'VC\\Tools\\MSVC'\n guess_vc = os.path.join(self.VSInstallDir, default)\n # Subdir with VC exact version as name\n try:\n vc_exact_ver = os.listdir(guess_vc)[-1]\n return os.path.join(guess_vc, vc_exact_ver)\n except (OSError, IOError, IndexError):\n pass", "response": "Find the VC name for 2017. 0 or later."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _guess_vc_legacy(self):\n default = r'Microsoft Visual Studio %0.1f\\VC' % self.vc_ver\n return os.path.join(self.ProgramFilesx86, default)", "response": "Try to guess Visual C from version 0. 1f."} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ngets Windows SDK directory.", "response": "def WindowsSdkDir(self):\n \"\"\"\n Microsoft Windows SDK directory.\n \"\"\"\n sdkdir = ''\n for ver in self.WindowsSdkVersion:\n # Try to get it from registry\n loc = os.path.join(self.ri.windows_sdk, 'v%s' % ver)\n sdkdir = self.ri.lookup(loc, 'installationfolder')\n if sdkdir:\n break\n if not sdkdir or not os.path.isdir(sdkdir):\n # Try to get \"VC++ for Python\" version from registry\n path = os.path.join(self.ri.vc_for_python, '%0.1f' % self.vc_ver)\n install_base = self.ri.lookup(path, 'installdir')\n if install_base:\n sdkdir = os.path.join(install_base, 'WinSDK')\n if not sdkdir or not os.path.isdir(sdkdir):\n # If fail, use default new path\n for ver in self.WindowsSdkVersion:\n intver = ver[:ver.rfind('.')]\n path = r'Microsoft SDKs\\Windows Kits\\%s' % (intver)\n d = os.path.join(self.ProgramFiles, path)\n if os.path.isdir(d):\n sdkdir = d\n if not sdkdir or not os.path.isdir(sdkdir):\n # If fail, use default old path\n for ver in self.WindowsSdkVersion:\n path = r'Microsoft SDKs\\Windows\\v%s' % ver\n d = os.path.join(self.ProgramFiles, path)\n if os.path.isdir(d):\n sdkdir = d\n if not sdkdir:\n # If fail, use Platform SDK\n sdkdir = os.path.join(self.VCInstallDir, 'PlatformSDK')\n return sdkdir"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn the Windows SDK executable directory path.", "response": "def WindowsSDKExecutablePath(self):\n \"\"\"\n Microsoft Windows SDK executable directory.\n \"\"\"\n # Find WinSDK NetFx Tools registry dir name\n if self.vc_ver <= 11.0:\n netfxver = 35\n arch = ''\n else:\n netfxver = 40\n hidex86 = True if self.vc_ver <= 12.0 else False\n arch = self.pi.current_dir(x64=True, hidex86=hidex86)\n fx = 'WinSDK-NetFx%dTools%s' % (netfxver, arch.replace('\\\\', '-'))\n\n # liste all possibles registry paths\n regpaths = []\n if self.vc_ver >= 14.0:\n for ver in self.NetFxSdkVersion:\n regpaths += [os.path.join(self.ri.netfx_sdk, ver, fx)]\n\n for ver in self.WindowsSdkVersion:\n regpaths += [os.path.join(self.ri.windows_sdk, 'v%sA' % ver, fx)]\n\n # Return installation folder from the more recent path\n for path in regpaths:\n execpath = self.ri.lookup(path, 'installationfolder')\n if execpath:\n break\n return execpath"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef FrameworkDir32(self):\n # Default path\n guess_fw = os.path.join(self.WinDir, r'Microsoft.NET\\Framework')\n\n # Try to get path from registry, if fail use default path\n return self.ri.lookup(self.ri.vc, 'frameworkdir32') or guess_fw", "response": "Microsoft. NET Framework 32bit directory."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef FrameworkDir64(self):\n # Default path\n guess_fw = os.path.join(self.WinDir, r'Microsoft.NET\\Framework64')\n\n # Try to get path from registry, if fail use default path\n return self.ri.lookup(self.ri.vc, 'frameworkdir64') or guess_fw", "response": "Microsoft. NET Framework 64bit directory."} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nfinds Microsoft. NET Framework versions.", "response": "def _find_dot_net_versions(self, bits):\n \"\"\"\n Find Microsoft .NET Framework versions.\n\n Parameters\n ----------\n bits: int\n Platform number of bits: 32 or 64.\n \"\"\"\n # Find actual .NET version in registry\n reg_ver = self.ri.lookup(self.ri.vc, 'frameworkver%d' % bits)\n dot_net_dir = getattr(self, 'FrameworkDir%d' % bits)\n ver = reg_ver or self._use_last_dir_name(dot_net_dir, 'v') or ''\n\n # Set .NET versions for specified MSVC++ version\n if self.vc_ver >= 12.0:\n frameworkver = (ver, 'v4.0')\n elif self.vc_ver >= 10.0:\n frameworkver = ('v4.0.30319' if ver.lower()[:2] != 'v4' else ver,\n 'v3.5')\n elif self.vc_ver == 9.0:\n frameworkver = ('v3.5', 'v2.0.50727')\n if self.vc_ver == 8.0:\n frameworkver = ('v3.0', 'v2.0.50727')\n return frameworkver"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _use_last_dir_name(self, path, prefix=''):\n matching_dirs = (\n dir_name\n for dir_name in reversed(os.listdir(path))\n if os.path.isdir(os.path.join(path, dir_name)) and\n dir_name.startswith(prefix)\n )\n return next(matching_dirs, None) or ''", "response": "Return the last dir name in path or an empty string if no dir found."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef VCTools(self):\n si = self.si\n tools = [os.path.join(si.VCInstallDir, 'VCPackages')]\n\n forcex86 = True if self.vc_ver <= 10.0 else False\n arch_subdir = self.pi.cross_dir(forcex86)\n if arch_subdir:\n tools += [os.path.join(si.VCInstallDir, 'Bin%s' % arch_subdir)]\n\n if self.vc_ver == 14.0:\n path = 'Bin%s' % self.pi.current_dir(hidex86=True)\n tools += [os.path.join(si.VCInstallDir, path)]\n\n elif self.vc_ver >= 15.0:\n host_dir = (r'bin\\HostX86%s' if self.pi.current_is_x86() else\n r'bin\\HostX64%s')\n tools += [os.path.join(\n si.VCInstallDir, host_dir % self.pi.target_dir(x64=True))]\n\n if self.pi.current_cpu != self.pi.target_cpu:\n tools += [os.path.join(\n si.VCInstallDir, host_dir % self.pi.current_dir(x64=True))]\n\n else:\n tools += [os.path.join(si.VCInstallDir, 'Bin')]\n\n return tools", "response": "Microsoft Visual C ++ Tools"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef OSLibraries(self):\n if self.vc_ver <= 10.0:\n arch_subdir = self.pi.target_dir(hidex86=True, x64=True)\n return [os.path.join(self.si.WindowsSdkDir, 'Lib%s' % arch_subdir)]\n\n else:\n arch_subdir = self.pi.target_dir(x64=True)\n lib = os.path.join(self.si.WindowsSdkDir, 'lib')\n libver = self._sdk_subdir\n return [os.path.join(lib, '%sum%s' % (libver , arch_subdir))]", "response": "Microsoft Windows SDK Libraries"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _sdk_tools(self):\n if self.vc_ver < 15.0:\n bin_dir = 'Bin' if self.vc_ver <= 11.0 else r'Bin\\x86'\n yield os.path.join(self.si.WindowsSdkDir, bin_dir)\n\n if not self.pi.current_is_x86():\n arch_subdir = self.pi.current_dir(x64=True)\n path = 'Bin%s' % arch_subdir\n yield os.path.join(self.si.WindowsSdkDir, path)\n\n if self.vc_ver == 10.0 or self.vc_ver == 11.0:\n if self.pi.target_is_x86():\n arch_subdir = ''\n else:\n arch_subdir = self.pi.current_dir(hidex86=True, x64=True)\n path = r'Bin\\NETFX 4.0 Tools%s' % arch_subdir\n yield os.path.join(self.si.WindowsSdkDir, path)\n\n elif self.vc_ver >= 15.0:\n path = os.path.join(self.si.WindowsSdkDir, 'Bin')\n arch_subdir = self.pi.current_dir(x64=True)\n sdkver = self.si.WindowsSdkLastVersion\n yield os.path.join(path, '%s%s' % (sdkver, arch_subdir))\n\n if self.si.WindowsSDKExecutablePath:\n yield self.si.WindowsSDKExecutablePath", "response": "Microsoft Windows SDK Tools paths generator"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef FxTools(self):\n pi = self.pi\n si = self.si\n\n if self.vc_ver <= 10.0:\n include32 = True\n include64 = not pi.target_is_x86() and not pi.current_is_x86()\n else:\n include32 = pi.target_is_x86() or pi.current_is_x86()\n include64 = pi.current_cpu == 'amd64' or pi.target_cpu == 'amd64'\n\n tools = []\n if include32:\n tools += [os.path.join(si.FrameworkDir32, ver)\n for ver in si.FrameworkVersion32]\n if include64:\n tools += [os.path.join(si.FrameworkDir64, ver)\n for ver in si.FrameworkVersion64]\n return tools", "response": "Microsoft. NET Framework Tools"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef NetFxSDKLibraries(self):\n if self.vc_ver < 14.0 or not self.si.NetFxSdkDir:\n return []\n\n arch_subdir = self.pi.target_dir(x64=True)\n return [os.path.join(self.si.NetFxSdkDir, r'lib\\um%s' % arch_subdir)]", "response": "Microsoft. Net Framework SDK Libraries"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef NetFxSDKIncludes(self):\n if self.vc_ver < 14.0 or not self.si.NetFxSdkDir:\n return []\n\n return [os.path.join(self.si.NetFxSdkDir, r'include\\um')]", "response": "Microsoft. Net Framework SDK Includes"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef HTMLHelpWorkshop(self):\n if self.vc_ver < 11.0:\n return []\n\n return [os.path.join(self.si.ProgramFilesx86, 'HTML Help Workshop')]", "response": "Microsoft HTML Help Workshop"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef UCRTIncludes(self):\n if self.vc_ver < 14.0:\n return []\n\n include = os.path.join(self.si.UniversalCRTSdkDir, 'include')\n return [os.path.join(include, '%sucrt' % self._ucrt_subdir)]", "response": "Microsoft Universal C Runtime SDK Include"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef VCRuntimeRedist(self):\n arch_subdir = self.pi.target_dir(x64=True)\n if self.vc_ver < 15:\n redist_path = self.si.VCInstallDir\n vcruntime = 'redist%s\\\\Microsoft.VC%d0.CRT\\\\vcruntime%d0.dll'\n else:\n redist_path = self.si.VCInstallDir.replace('\\\\Tools', '\\\\Redist')\n vcruntime = 'onecore%s\\\\Microsoft.VC%d0.CRT\\\\vcruntime%d0.dll'\n\n # Visual Studio 2017 is still Visual C++ 14.0\n dll_ver = 14.0 if self.vc_ver == 15 else self.vc_ver\n\n vcruntime = vcruntime % (arch_subdir, self.vc_ver, dll_ver)\n return os.path.join(redist_path, vcruntime)", "response": "Microsoft Visual C ++ runtime redistribuable dll\n "} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nbuilds and return a dict of environment variables.", "response": "def return_env(self, exists=True):\n \"\"\"\n Return environment dict.\n\n Parameters\n ----------\n exists: bool\n It True, only return existing paths.\n \"\"\"\n env = dict(\n include=self._build_paths('include',\n [self.VCIncludes,\n self.OSIncludes,\n self.UCRTIncludes,\n self.NetFxSDKIncludes],\n exists),\n lib=self._build_paths('lib',\n [self.VCLibraries,\n self.OSLibraries,\n self.FxTools,\n self.UCRTLibraries,\n self.NetFxSDKLibraries],\n exists),\n libpath=self._build_paths('libpath',\n [self.VCLibraries,\n self.FxTools,\n self.VCStoreRefs,\n self.OSLibpath],\n exists),\n path=self._build_paths('path',\n [self.VCTools,\n self.VSTools,\n self.VsTDb,\n self.SdkTools,\n self.SdkSetup,\n self.FxTools,\n self.MSBuild,\n self.HTMLHelpWorkshop,\n self.FSharp],\n exists),\n )\n if self.vc_ver >= 14 and os.path.isfile(self.VCRuntimeRedist):\n env['py_vcruntime_redist'] = self.VCRuntimeRedist\n return env"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ngiving an environment variable name and specified paths return a pathsep - separated string of paths containing unique extant directories from those paths and from the environment variable. Raise an error if no paths are found.", "response": "def _build_paths(self, name, spec_path_lists, exists):\n \"\"\"\n Given an environment variable name and specified paths,\n return a pathsep-separated string of paths containing\n unique, extant, directories from those paths and from\n the environment variable. Raise an error if no paths\n are resolved.\n \"\"\"\n # flatten spec_path_lists\n spec_paths = itertools.chain.from_iterable(spec_path_lists)\n env_paths = safe_env.get(name, '').split(os.pathsep)\n paths = itertools.chain(spec_paths, env_paths)\n extant_paths = list(filter(os.path.isdir, paths)) if exists else paths\n if not extant_paths:\n msg = \"%s environment variable is empty\" % name.upper()\n raise distutils.errors.DistutilsPlatformError(msg)\n unique_paths = self._unique_everseen(extant_paths)\n return os.pathsep.join(unique_paths)"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning the file extension for an abi3 - compliant Extension", "response": "def get_abi3_suffix():\n \"\"\"Return the file extension for an abi3-compliant Extension()\"\"\"\n for suffix, _, _ in (s for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION):\n if '.abi3' in suffix: # Unix\n return suffix\n elif suffix == '.pyd': # Windows\n return suffix"} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nruns build extensions in build directory then copy extensions to source directory.", "response": "def run(self):\n \"\"\"Build extensions in build directory, then copy if --inplace\"\"\"\n old_inplace, self.inplace = self.inplace, 0\n _build_ext.run(self)\n self.inplace = old_inplace\n if old_inplace:\n self.copy_extensions_to_source()"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef save_modules():\n saved = sys.modules.copy()\n with ExceptionSaver() as saved_exc:\n yield saved\n\n sys.modules.update(saved)\n # remove any modules imported since\n del_modules = (\n mod_name for mod_name in sys.modules\n if mod_name not in saved\n # exclude any encodings modules. See #285\n and not mod_name.startswith('encodings.')\n )\n _clear_modules(del_modules)\n\n saved_exc.resume()", "response": "Context in which imported modules are saved."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nchecking if the given module name is a valid Cython module.", "response": "def _needs_hiding(mod_name):\n \"\"\"\n >>> _needs_hiding('setuptools')\n True\n >>> _needs_hiding('pkg_resources')\n True\n >>> _needs_hiding('setuptools_plugin')\n False\n >>> _needs_hiding('setuptools.__init__')\n True\n >>> _needs_hiding('distutils')\n True\n >>> _needs_hiding('os')\n False\n >>> _needs_hiding('Cython')\n True\n \"\"\"\n pattern = re.compile(r'(setuptools|pkg_resources|distutils|Cython)(\\.|$)')\n return bool(pattern.match(mod_name))"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nrun a distutils setup script sandboxed in its directory", "response": "def run_setup(setup_script, args):\n \"\"\"Run a distutils setup script, sandboxed in its directory\"\"\"\n setup_dir = os.path.abspath(os.path.dirname(setup_script))\n with setup_context(setup_dir):\n try:\n sys.argv[:] = [setup_script] + list(args)\n sys.path.insert(0, setup_dir)\n # reset to include setup dir, w/clean callback list\n working_set.__init__()\n working_set.callbacks.append(lambda dist: dist.activate())\n\n # __file__ should be a byte string on Python 2 (#712)\n dunder_file = (\n setup_script\n if isinstance(setup_script, str) else\n setup_script.encode(sys.getfilesystemencoding())\n )\n\n with DirectorySandbox(setup_dir):\n ns = dict(__file__=dunder_file, __name__='__main__')\n _execfile(setup_script, ns)\n except SystemExit as v:\n if v.args and v.args[0]:\n raise"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef dump(type, exc):\n try:\n return pickle.dumps(type), pickle.dumps(exc)\n except Exception:\n # get UnpickleableException inside the sandbox\n from setuptools.sandbox import UnpickleableException as cls\n return cls.dump(cls, cls(repr(exc)))", "response": "Dump a type and exception into a string."} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nrestoring and re - raise any exception", "response": "def resume(self):\n \"restore and re-raise any exception\"\n\n if '_saved' not in vars(self):\n return\n\n type, exc = map(pickle.loads, self._saved)\n six.reraise(type, exc, self._tb)"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _remap_pair(self, operation, src, dst, *args, **kw):\n return (\n self._remap_input(operation + '-from', src, *args, **kw),\n self._remap_input(operation + '-to', dst, *args, **kw)\n )", "response": "Called for path pairs like rename link symlink and rename operations"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _remap_input(self, operation, path, *args, **kw):\n if operation in self.write_ops and not self._ok(path):\n self._violation(operation, os.path.realpath(path), *args, **kw)\n return path", "response": "Called for path inputs"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncalling for path pairs like rename link and symlink operations.", "response": "def _remap_pair(self, operation, src, dst, *args, **kw):\n \"\"\"Called for path pairs like rename, link, and symlink operations\"\"\"\n if not self._ok(src) or not self._ok(dst):\n self._violation(operation, src, dst, *args, **kw)\n return (src, dst)"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn the bases classes for cls sorted by the MRO.", "response": "def _get_mro(cls):\n \"\"\"\n Returns the bases classes for cls sorted by the MRO.\n\n Works around an issue on Jython where inspect.getmro will not return all\n base classes if multiple classes share the same name. Instead, this\n function will return a tuple containing the class itself, and the contents\n of cls.__bases__. See https://github.com/pypa/setuptools/issues/1024.\n \"\"\"\n if platform.python_implementation() == \"Jython\":\n return (cls,) + cls.__bases__\n return inspect.getmro(cls)"} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\npatches the DistributionMetadata class to include the version of the metadata.", "response": "def _patch_distribution_metadata():\n \"\"\"Patch write_pkg_file and read_pkg_file for higher metadata standards\"\"\"\n for attr in ('write_pkg_file', 'read_pkg_file', 'get_metadata_version'):\n new_val = getattr(setuptools.dist, attr)\n setattr(distutils.dist.DistributionMetadata, attr, new_val)"} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\npatch a function in a module with a replacement function.", "response": "def patch_func(replacement, target_mod, func_name):\n \"\"\"\n Patch func_name in target_mod with replacement\n\n Important - original must be resolved by name to avoid\n patching an already patched function.\n \"\"\"\n original = getattr(target_mod, func_name)\n\n # set the 'unpatched' attribute on the replacement to\n # point to the original.\n vars(replacement).setdefault('unpatched', original)\n\n # replace the function in the original module\n setattr(target_mod, func_name, replacement)"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef patch_for_msvc_specialized_compiler():\n # import late to avoid circular imports on Python < 3.5\n msvc = import_module('setuptools.msvc')\n\n if platform.system() != 'Windows':\n # Compilers only availables on Microsoft Windows\n return\n\n def patch_params(mod_name, func_name):\n \"\"\"\n Prepare the parameters for patch_func to patch indicated function.\n \"\"\"\n repl_prefix = 'msvc9_' if 'msvc9' in mod_name else 'msvc14_'\n repl_name = repl_prefix + func_name.lstrip('_')\n repl = getattr(msvc, repl_name)\n mod = import_module(mod_name)\n if not hasattr(mod, func_name):\n raise ImportError(func_name)\n return repl, mod, func_name\n\n # Python 2.7 to 3.4\n msvc9 = functools.partial(patch_params, 'distutils.msvc9compiler')\n\n # Python 3.5+\n msvc14 = functools.partial(patch_params, 'distutils._msvccompiler')\n\n try:\n # Patch distutils.msvc9compiler\n patch_func(*msvc9('find_vcvarsall'))\n patch_func(*msvc9('query_vcvarsall'))\n except ImportError:\n pass\n\n try:\n # Patch distutils._msvccompiler._get_vc_env\n patch_func(*msvc14('_get_vc_env'))\n except ImportError:\n pass\n\n try:\n # Patch distutils._msvccompiler.gen_lib_options for Numpy\n patch_func(*msvc14('gen_lib_options'))\n except ImportError:\n pass", "response": "Patch functions in distutils to use standalone Microsoft Visual C ++ compilers."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nescaping all special characters in a pathname.", "response": "def escape(pathname):\n \"\"\"Escape all special characters.\n \"\"\"\n # Escaping is done by wrapping any of \"*?[\" between square brackets.\n # Metacharacters do not work in the drive part and shouldn't be escaped.\n drive, pathname = os.path.splitdrive(pathname)\n if isinstance(pathname, bytes):\n pathname = magic_check_bytes.sub(br'[\\1]', pathname)\n else:\n pathname = magic_check.sub(r'[\\1]', pathname)\n return drive + pathname"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _build_multipart(cls, data):\n boundary = b'--------------GHSKFJDLGDS7543FJKLFHRE75642756743254'\n sep_boundary = b'\\n--' + boundary\n end_boundary = sep_boundary + b'--'\n end_items = end_boundary, b\"\\n\",\n builder = functools.partial(\n cls._build_part,\n sep_boundary=sep_boundary,\n )\n part_groups = map(builder, data.items())\n parts = itertools.chain.from_iterable(part_groups)\n body_items = itertools.chain(parts, end_items)\n content_type = 'multipart/form-data; boundary=%s' % boundary.decode('ascii')\n return b''.join(body_items), content_type", "response": "Builds the MIME payload for the POST data."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nunpacking all files under src_dir to dst_dir and deletes the former.", "response": "def unpack(src_dir, dst_dir):\n '''Move everything under `src_dir` to `dst_dir`, and delete the former.'''\n for dirpath, dirnames, filenames in os.walk(src_dir):\n subdir = os.path.relpath(dirpath, src_dir)\n for f in filenames:\n src = os.path.join(dirpath, f)\n dst = os.path.join(dst_dir, subdir, f)\n os.renames(src, dst)\n for n, d in reversed(list(enumerate(dirnames))):\n src = os.path.join(dirpath, d)\n dst = os.path.join(dst_dir, subdir, d)\n if not os.path.exists(dst):\n # Directory does not exist in destination,\n # rename it and prune it from os.walk list.\n os.renames(src, dst)\n del dirnames[n]\n # Cleanup.\n for dirpath, dirnames, filenames in os.walk(src_dir, topdown=True):\n assert not filenames\n os.rmdir(dirpath)"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef tags(self):\n '''List tags (py_version, abi, platform) supported by this wheel.'''\n return itertools.product(\n self.py_version.split('.'),\n self.abi.split('.'),\n self.platform.split('.'),\n )", "response": "List tags supported by this wheel."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef is_compatible(self):\n '''Is the wheel is compatible with the current platform?'''\n supported_tags = pep425tags.get_supported()\n return next((True for t in self.tags() if t in supported_tags), False)", "response": "Is the wheel is compatible with the current platform?"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef install_as_egg(self, destination_eggdir):\n '''Install wheel as an egg directory.'''\n with zipfile.ZipFile(self.filename) as zf:\n self._install_as_egg(destination_eggdir, zf)", "response": "Install wheel as an egg directory."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _move_data_entries(destination_eggdir, dist_data):\n dist_data = os.path.join(destination_eggdir, dist_data)\n dist_data_scripts = os.path.join(dist_data, 'scripts')\n if os.path.exists(dist_data_scripts):\n egg_info_scripts = os.path.join(\n destination_eggdir, 'EGG-INFO', 'scripts')\n os.mkdir(egg_info_scripts)\n for entry in os.listdir(dist_data_scripts):\n # Remove bytecode, as it's not properly handled\n # during easy_install scripts install phase.\n if entry.endswith('.pyc'):\n os.unlink(os.path.join(dist_data_scripts, entry))\n else:\n os.rename(\n os.path.join(dist_data_scripts, entry),\n os.path.join(egg_info_scripts, entry),\n )\n os.rmdir(dist_data_scripts)\n for subdir in filter(os.path.exists, (\n os.path.join(dist_data, d)\n for d in ('data', 'headers', 'purelib', 'platlib')\n )):\n unpack(subdir, destination_eggdir)\n if os.path.exists(dist_data):\n os.rmdir(dist_data)", "response": "Move data entries to their correct location."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nread given configuration file and returns options as a dict.", "response": "def read_configuration(\n filepath, find_others=False, ignore_option_errors=False):\n \"\"\"Read given configuration file and returns options from it as a dict.\n\n :param str|unicode filepath: Path to configuration file\n to get options from.\n\n :param bool find_others: Whether to search for other configuration files\n which could be on in various places.\n\n :param bool ignore_option_errors: Whether to silently ignore\n options, values of which could not be resolved (e.g. due to exceptions\n in directives such as file:, attr:, etc.).\n If False exceptions are propagated as expected.\n\n :rtype: dict\n \"\"\"\n from setuptools.dist import Distribution, _Distribution\n\n filepath = os.path.abspath(filepath)\n\n if not os.path.isfile(filepath):\n raise DistutilsFileError(\n 'Configuration file %s does not exist.' % filepath)\n\n current_directory = os.getcwd()\n os.chdir(os.path.dirname(filepath))\n\n try:\n dist = Distribution()\n\n filenames = dist.find_config_files() if find_others else []\n if filepath not in filenames:\n filenames.append(filepath)\n\n _Distribution.parse_config_files(dist, filenames=filenames)\n\n handlers = parse_configuration(\n dist, dist.command_options,\n ignore_option_errors=ignore_option_errors)\n\n finally:\n os.chdir(current_directory)\n\n return configuration_to_dict(handlers)"} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ngets an option from a target object.", "response": "def _get_option(target_obj, key):\n \"\"\"\n Given a target object and option key, get that option from\n the target object, either through a get_{key} method or\n from an attribute directly.\n \"\"\"\n getter_name = 'get_{key}'.format(**locals())\n by_attribute = functools.partial(getattr, target_obj, key)\n getter = getattr(target_obj, getter_name, by_attribute)\n return getter()"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn the configuration data gathered by given handlers as a dict.", "response": "def configuration_to_dict(handlers):\n \"\"\"Returns configuration data gathered by given handlers as a dict.\n\n :param list[ConfigHandler] handlers: Handlers list,\n usually from parse_configuration()\n\n :rtype: dict\n \"\"\"\n config_dict = defaultdict(dict)\n\n for handler in handlers:\n for option in handler.set_options:\n value = _get_option(handler.target_obj, option)\n config_dict[handler.section_prefix][option] = value\n\n return config_dict"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef parse_configuration(\n distribution, command_options, ignore_option_errors=False):\n \"\"\"Performs additional parsing of configuration options\n for a distribution.\n\n Returns a list of used option handlers.\n\n :param Distribution distribution:\n :param dict command_options:\n :param bool ignore_option_errors: Whether to silently ignore\n options, values of which could not be resolved (e.g. due to exceptions\n in directives such as file:, attr:, etc.).\n If False exceptions are propagated as expected.\n :rtype: list\n \"\"\"\n options = ConfigOptionsHandler(\n distribution, command_options, ignore_option_errors)\n options.parse()\n\n meta = ConfigMetadataHandler(\n distribution.metadata, command_options, ignore_option_errors,\n distribution.package_dir)\n meta.parse()\n\n return meta, options", "response": "Performs additional parsing of configuration options\n for a distribution."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nrepresenting value as a list.", "response": "def _parse_list(cls, value, separator=','):\n \"\"\"Represents value as a list.\n\n Value is split either by separator (defaults to comma) or by lines.\n\n :param value:\n :param separator: List items separator character.\n :rtype: list\n \"\"\"\n if isinstance(value, list): # _get_parser_compound case\n return value\n\n if '\\n' in value:\n value = value.splitlines()\n else:\n value = value.split(separator)\n\n return [chunk.strip() for chunk in value if chunk.strip()]"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nrepresent value as a dict.", "response": "def _parse_dict(cls, value):\n \"\"\"Represents value as a dict.\n\n :param value:\n :rtype: dict\n \"\"\"\n separator = '='\n result = {}\n for line in cls._parse_list(value):\n key, sep, val = line.partition(separator)\n if sep != separator:\n raise DistutilsOptionError(\n 'Unable to parse option value to dict: %s' % value)\n result[key.strip()] = val.strip()\n\n return result"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _exclude_files_parser(cls, key):\n def parser(value):\n exclude_directive = 'file:'\n if value.startswith(exclude_directive):\n raise ValueError(\n 'Only strings are accepted for the {0} field, '\n 'files are not accepted'.format(key))\n return value\n return parser", "response": "Returns a parser function to make sure field inputs\n are not files."} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nrepresent value as a string allowing including text from nearest files using file : directive.", "response": "def _parse_file(cls, value):\n \"\"\"Represents value as a string, allowing including text\n from nearest files using `file:` directive.\n\n Directive is sandboxed and won't reach anything outside\n directory with setup.py.\n\n Examples:\n file: README.rst, CHANGELOG.md, src/file.txt\n\n :param str value:\n :rtype: str\n \"\"\"\n include_directive = 'file:'\n\n if not isinstance(value, string_types):\n return value\n\n if not value.startswith(include_directive):\n return value\n\n spec = value[len(include_directive):]\n filepaths = (os.path.abspath(path.strip()) for path in spec.split(','))\n return '\\n'.join(\n cls._read_file(path)\n for path in filepaths\n if (cls._assert_local(path) or True)\n and os.path.isfile(path)\n )"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _parse_attr(cls, value, package_dir=None):\n attr_directive = 'attr:'\n if not value.startswith(attr_directive):\n return value\n\n attrs_path = value.replace(attr_directive, '').strip().split('.')\n attr_name = attrs_path.pop()\n\n module_name = '.'.join(attrs_path)\n module_name = module_name or '__init__'\n\n parent_path = os.getcwd()\n if package_dir:\n if attrs_path[0] in package_dir:\n # A custom path was specified for the module we want to import\n custom_path = package_dir[attrs_path[0]]\n parts = custom_path.rsplit('/', 1)\n if len(parts) > 1:\n parent_path = os.path.join(os.getcwd(), parts[0])\n module_name = parts[1]\n else:\n module_name = custom_path\n elif '' in package_dir:\n # A custom parent directory was specified for all root modules\n parent_path = os.path.join(os.getcwd(), package_dir[''])\n sys.path.insert(0, parent_path)\n try:\n module = import_module(module_name)\n value = getattr(module, attr_name)\n\n finally:\n sys.path = sys.path[1:]\n\n return value", "response": "Represents value as a module attribute."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a parser function to represents a value as a list.", "response": "def _get_parser_compound(cls, *parse_methods):\n \"\"\"Returns parser function to represents value as a list.\n\n Parses a value applying given methods one after another.\n\n :param parse_methods:\n :rtype: callable\n \"\"\"\n def parse(value):\n parsed = value\n\n for method in parse_methods:\n parsed = method(parsed)\n\n return parsed\n\n return parse"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _parse_section_to_dict(cls, section_options, values_parser=None):\n value = {}\n values_parser = values_parser or (lambda val: val)\n for key, (_, val) in section_options.items():\n value[key] = values_parser(val)\n return value", "response": "Parses section options into a dictionary."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nparse the configuration file section.", "response": "def parse_section(self, section_options):\n \"\"\"Parses configuration file section.\n\n :param dict section_options:\n \"\"\"\n for (name, (_, value)) in section_options.items():\n try:\n self[name] = value\n\n except KeyError:\n pass"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning a dictionary of metadata item name to parser function mapping.", "response": "def parsers(self):\n \"\"\"Metadata item name to parser function mapping.\"\"\"\n parse_list = self._parse_list\n parse_file = self._parse_file\n parse_dict = self._parse_dict\n exclude_files_parser = self._exclude_files_parser\n\n return {\n 'platforms': parse_list,\n 'keywords': parse_list,\n 'provides': parse_list,\n 'requires': self._deprecated_config_handler(\n parse_list,\n \"The requires parameter is deprecated, please use \"\n \"install_requires for runtime dependencies.\",\n DeprecationWarning),\n 'obsoletes': parse_list,\n 'classifiers': self._get_parser_compound(parse_file, parse_list),\n 'license': exclude_files_parser('license'),\n 'description': parse_file,\n 'long_description': parse_file,\n 'version': self._parse_version,\n 'project_urls': parse_dict,\n }"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _parse_version(self, value):\n version = self._parse_file(value)\n\n if version != value:\n version = version.strip()\n # Be strict about versions loaded from file because it's easy to\n # accidentally include newlines and other unintended content\n if isinstance(parse(version), LegacyVersion):\n tmpl = (\n 'Version loaded from {value} does not '\n 'comply with PEP 440: {version}'\n )\n raise DistutilsOptionError(tmpl.format(**locals()))\n\n return version\n\n version = self._parse_attr(value, self.package_dir)\n\n if callable(version):\n version = version()\n\n if not isinstance(version, string_types):\n if hasattr(version, '__iter__'):\n version = '.'.join(map(str, version))\n else:\n version = '%s' % version\n\n return version", "response": "Parses version option value."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning a dictionary of metadata item name to parser function mapping.", "response": "def parsers(self):\n \"\"\"Metadata item name to parser function mapping.\"\"\"\n parse_list = self._parse_list\n parse_list_semicolon = partial(self._parse_list, separator=';')\n parse_bool = self._parse_bool\n parse_dict = self._parse_dict\n\n return {\n 'zip_safe': parse_bool,\n 'use_2to3': parse_bool,\n 'include_package_data': parse_bool,\n 'package_dir': parse_dict,\n 'use_2to3_fixers': parse_list,\n 'use_2to3_exclude_fixers': parse_list,\n 'convert_2to3_doctests': parse_list,\n 'scripts': parse_list,\n 'eager_resources': parse_list,\n 'dependency_links': parse_list,\n 'namespace_packages': parse_list,\n 'install_requires': parse_list_semicolon,\n 'setup_requires': parse_list_semicolon,\n 'tests_require': parse_list_semicolon,\n 'packages': self._parse_packages,\n 'entry_points': self._parse_file,\n 'py_modules': parse_list,\n }"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nparsing the packages option value.", "response": "def _parse_packages(self, value):\n \"\"\"Parses `packages` option value.\n\n :param value:\n :rtype: list\n \"\"\"\n find_directives = ['find:', 'find_namespace:']\n trimmed_value = value.strip()\n\n if trimmed_value not in find_directives:\n return self._parse_list(value)\n\n findns = trimmed_value == find_directives[1]\n if findns and not PY3:\n raise DistutilsOptionError(\n 'find_namespace: directive is unsupported on Python < 3.3')\n\n # Read function arguments from a dedicated section.\n find_kwargs = self.parse_section_packages__find(\n self.sections.get('packages.find', {}))\n\n if findns:\n from setuptools import find_namespace_packages as find_packages\n else:\n from setuptools import find_packages\n\n return find_packages(**find_kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef parse_section_packages__find(self, section_options):\n section_data = self._parse_section_to_dict(\n section_options, self._parse_list)\n\n valid_keys = ['where', 'include', 'exclude']\n\n find_kwargs = dict(\n [(k, v) for k, v in section_data.items() if k in valid_keys and v])\n\n where = find_kwargs.get('where')\n if where is not None:\n find_kwargs['where'] = where[0] # cast list to single val\n\n return find_kwargs", "response": "Parses packages. find section."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nparse entry_points configuration file section.", "response": "def parse_section_entry_points(self, section_options):\n \"\"\"Parses `entry_points` configuration file section.\n\n :param dict section_options:\n \"\"\"\n parsed = self._parse_section_to_dict(section_options, self._parse_list)\n self['entry_points'] = parsed"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nparse the extras_require section.", "response": "def parse_section_extras_require(self, section_options):\n \"\"\"Parses `extras_require` configuration file section.\n\n :param dict section_options:\n \"\"\"\n parse_list = partial(self._parse_list, separator=';')\n self['extras_require'] = self._parse_section_to_dict(\n section_options, parse_list)"} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nparses the data_files section.", "response": "def parse_section_data_files(self, section_options):\n \"\"\"Parses `data_files` configuration file section.\n\n :param dict section_options:\n \"\"\"\n parsed = self._parse_section_to_dict(section_options, self._parse_list)\n self['data_files'] = [(k, v) for k, v in parsed.items()]"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef read_pkg_file(self, file):\n msg = message_from_file(file)\n\n def _read_field(name):\n value = msg[name]\n if value == 'UNKNOWN':\n return None\n return value\n\n def _read_list(name):\n values = msg.get_all(name, None)\n if values == []:\n return None\n return values\n\n self.metadata_version = StrictVersion(msg['metadata-version'])\n self.name = _read_field('name')\n self.version = _read_field('version')\n self.description = _read_field('summary')\n # we are filling author only.\n self.author = _read_field('author')\n self.maintainer = None\n self.author_email = _read_field('author-email')\n self.maintainer_email = None\n self.url = _read_field('home-page')\n self.license = _read_field('license')\n\n if 'download-url' in msg:\n self.download_url = _read_field('download-url')\n else:\n self.download_url = None\n\n self.long_description = _read_field('description')\n self.description = _read_field('summary')\n\n if 'keywords' in msg:\n self.keywords = _read_field('keywords').split(',')\n\n self.platforms = _read_list('platform')\n self.classifiers = _read_list('classifier')\n\n # PEP 314 - these fields only exist in 1.1\n if self.metadata_version == StrictVersion('1.1'):\n self.requires = _read_list('requires')\n self.provides = _read_list('provides')\n self.obsoletes = _read_list('obsoletes')\n else:\n self.requires = None\n self.provides = None\n self.obsoletes = None", "response": "Reads the metadata values from a file object."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nwrite the PKG - INFO format data to a file object.", "response": "def write_pkg_file(self, file):\n \"\"\"Write the PKG-INFO format data to a file object.\n \"\"\"\n version = self.get_metadata_version()\n\n if six.PY2:\n def write_field(key, value):\n file.write(\"%s: %s\\n\" % (key, self._encode_field(value)))\n else:\n def write_field(key, value):\n file.write(\"%s: %s\\n\" % (key, value))\n\n write_field('Metadata-Version', str(version))\n write_field('Name', self.get_name())\n write_field('Version', self.get_version())\n write_field('Summary', self.get_description())\n write_field('Home-page', self.get_url())\n\n if version < StrictVersion('1.2'):\n write_field('Author', self.get_contact())\n write_field('Author-email', self.get_contact_email())\n else:\n optional_fields = (\n ('Author', 'author'),\n ('Author-email', 'author_email'),\n ('Maintainer', 'maintainer'),\n ('Maintainer-email', 'maintainer_email'),\n )\n\n for field, attr in optional_fields:\n attr_val = getattr(self, attr)\n\n if attr_val is not None:\n write_field(field, attr_val)\n\n write_field('License', self.get_license())\n if self.download_url:\n write_field('Download-URL', self.download_url)\n for project_url in self.project_urls.items():\n write_field('Project-URL', '%s, %s' % project_url)\n\n long_desc = rfc822_escape(self.get_long_description())\n write_field('Description', long_desc)\n\n keywords = ','.join(self.get_keywords())\n if keywords:\n write_field('Keywords', keywords)\n\n if version >= StrictVersion('1.2'):\n for platform in self.get_platforms():\n write_field('Platform', platform)\n else:\n self._write_list(file, 'Platform', self.get_platforms())\n\n self._write_list(file, 'Classifier', self.get_classifiers())\n\n # PEP 314\n self._write_list(file, 'Requires', self.get_requires())\n self._write_list(file, 'Provides', self.get_provides())\n self._write_list(file, 'Obsoletes', self.get_obsoletes())\n\n # Setuptools specific for PEP 345\n if hasattr(self, 'python_requires'):\n write_field('Requires-Python', self.python_requires)\n\n # PEP 566\n if self.long_description_content_type:\n write_field(\n 'Description-Content-Type',\n self.long_description_content_type\n )\n if self.provides_extras:\n for extra in self.provides_extras:\n write_field('Provides-Extra', extra)"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nverifying that value is a list of strings or None", "response": "def assert_string_list(dist, attr, value):\n \"\"\"Verify that value is a string list or None\"\"\"\n try:\n assert ''.join(value) != value\n except (TypeError, ValueError, AttributeError, AssertionError):\n raise DistutilsSetupError(\n \"%r must be a list of strings (got %r)\" % (attr, value)\n )"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nverifying that namespace packages are valid.", "response": "def check_nsp(dist, attr, value):\n \"\"\"Verify that namespace packages are valid\"\"\"\n ns_packages = value\n assert_string_list(dist, attr, ns_packages)\n for nsp in ns_packages:\n if not dist.has_contents_for(nsp):\n raise DistutilsSetupError(\n \"Distribution contains no modules or packages for \" +\n \"namespace package %r\" % nsp\n )\n parent, sep, child = nsp.rpartition('.')\n if parent and parent not in ns_packages:\n distutils.log.warn(\n \"WARNING: %r is declared as a package namespace, but %r\"\n \" is not: please correct this in setup.py\", nsp, parent\n )"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef check_extras(dist, attr, value):\n try:\n list(itertools.starmap(_check_extra, value.items()))\n except (TypeError, ValueError, AttributeError):\n raise DistutilsSetupError(\n \"'extras_require' must be a dictionary whose values are \"\n \"strings or lists of strings containing valid project/version \"\n \"requirement specifiers.\"\n )", "response": "Verify that extras_require mapping is valid."} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nverifies that value is True False 0 or 1", "response": "def assert_bool(dist, attr, value):\n \"\"\"Verify that value is True, False, 0, or 1\"\"\"\n if bool(value) != value:\n tmpl = \"{attr!r} must be a boolean value (got {value!r})\"\n raise DistutilsSetupError(tmpl.format(attr=attr, value=value))"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef check_specifier(dist, attr, value):\n try:\n packaging.specifiers.SpecifierSet(value)\n except packaging.specifiers.InvalidSpecifier as error:\n tmpl = (\n \"{attr!r} must be a string \"\n \"containing valid version specifiers; {error}\"\n )\n raise DistutilsSetupError(tmpl.format(attr=attr, error=error))", "response": "Verify that value is a valid version specifier"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef check_package_data(dist, attr, value):\n if isinstance(value, dict):\n for k, v in value.items():\n if not isinstance(k, str):\n break\n try:\n iter(v)\n except TypeError:\n break\n else:\n return\n raise DistutilsSetupError(\n attr + \" must be a dictionary mapping package names to lists of \"\n \"wildcard patterns\"\n )", "response": "Verify that value is a dictionary of package names to glob lists"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _finalize_requires(self):\n if getattr(self, 'python_requires', None):\n self.metadata.python_requires = self.python_requires\n\n if getattr(self, 'extras_require', None):\n for extra in self.extras_require.keys():\n # Since this gets called multiple times at points where the\n # keys have become 'converted' extras, ensure that we are only\n # truly adding extras we haven't seen before here.\n extra = extra.split(':')[0]\n if extra:\n self.metadata.provides_extras.add(extra)\n\n self._convert_extras_requirements()\n self._move_install_requirements_markers()", "response": "Set metadata. python_requires and fix environment markers\n in install_requires and extras_require."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _convert_extras_requirements(self):\n spec_ext_reqs = getattr(self, 'extras_require', None) or {}\n self._tmp_extras_require = defaultdict(list)\n for section, v in spec_ext_reqs.items():\n # Do not strip empty sections.\n self._tmp_extras_require[section]\n for r in pkg_resources.parse_requirements(v):\n suffix = self._suffix_for(r)\n self._tmp_extras_require[section + suffix].append(r)", "response": "Convert requirements in extras_require of the form\n extra = barbazquux ; marker = barbazquux ; marker = barbazquux ; marker = barbazquux ; marker = barbazquux ; marker = barbazquux ; marker = barbazquux ; marker = barbazquux ; marker = barbazquux ; marker = barbazquux ; marker = barbazquux ;"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nmoving requirements in install_requires that are using environment markers extras_require.", "response": "def _move_install_requirements_markers(self):\n \"\"\"\n Move requirements in `install_requires` that are using environment\n markers `extras_require`.\n \"\"\"\n\n # divide the install_requires into two sets, simple ones still\n # handled by install_requires and more complex ones handled\n # by extras_require.\n\n def is_simple_req(req):\n return not req.marker\n\n spec_inst_reqs = getattr(self, 'install_requires', None) or ()\n inst_reqs = list(pkg_resources.parse_requirements(spec_inst_reqs))\n simple_reqs = filter(is_simple_req, inst_reqs)\n complex_reqs = filterfalse(is_simple_req, inst_reqs)\n self.install_requires = list(map(str, simple_reqs))\n\n for r in complex_reqs:\n self._tmp_extras_require[':' + str(r.marker)].append(r)\n self.extras_require = dict(\n (k, [str(r) for r in map(self._clean_req, v)])\n for k, v in self._tmp_extras_require.items()\n )"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nparse the config files and returns a set of options.", "response": "def _parse_config_files(self, filenames=None):\n \"\"\"\n Adapted from distutils.dist.Distribution.parse_config_files,\n this method provides the same functionality in subtly-improved\n ways.\n \"\"\"\n from setuptools.extern.six.moves.configparser import ConfigParser\n\n # Ignore install directory options if we have a venv\n if six.PY3 and sys.prefix != sys.base_prefix:\n ignore_options = [\n 'install-base', 'install-platbase', 'install-lib',\n 'install-platlib', 'install-purelib', 'install-headers',\n 'install-scripts', 'install-data', 'prefix', 'exec-prefix',\n 'home', 'user', 'root']\n else:\n ignore_options = []\n\n ignore_options = frozenset(ignore_options)\n\n if filenames is None:\n filenames = self.find_config_files()\n\n if DEBUG:\n self.announce(\"Distribution.parse_config_files():\")\n\n parser = ConfigParser()\n for filename in filenames:\n with io.open(filename, encoding='utf-8') as reader:\n if DEBUG:\n self.announce(\" reading {filename}\".format(**locals()))\n (parser.read_file if six.PY3 else parser.readfp)(reader)\n for section in parser.sections():\n options = parser.options(section)\n opt_dict = self.get_option_dict(section)\n\n for opt in options:\n if opt != '__name__' and opt not in ignore_options:\n val = self._try_str(parser.get(section, opt))\n opt = opt.replace('-', '_')\n opt_dict[opt] = (filename, val)\n\n # Make the ConfigParser forget everything (so we retain\n # the original filenames that options come from)\n parser.__init__()\n\n # If there was a \"global\" section in the config file, use it\n # to set Distribution options.\n\n if 'global' in self.command_options:\n for (opt, (src, val)) in self.command_options['global'].items():\n alias = self.negative_opt.get(opt)\n try:\n if alias:\n setattr(self, alias, not strtobool(val))\n elif opt in ('verbose', 'dry_run'): # ugh!\n setattr(self, opt, strtobool(val))\n else:\n setattr(self, opt, val)\n except ValueError as msg:\n raise DistutilsOptionError(msg)"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _set_command_options(self, command_obj, option_dict=None):\n command_name = command_obj.get_command_name()\n if option_dict is None:\n option_dict = self.get_option_dict(command_name)\n\n if DEBUG:\n self.announce(\" setting options for '%s' command:\" % command_name)\n for (option, (source, value)) in option_dict.items():\n if DEBUG:\n self.announce(\" %s = %s (from %s)\" % (option, value,\n source))\n try:\n bool_opts = [translate_longopt(o)\n for o in command_obj.boolean_options]\n except AttributeError:\n bool_opts = []\n try:\n neg_opt = command_obj.negative_opt\n except AttributeError:\n neg_opt = {}\n\n try:\n is_string = isinstance(value, six.string_types)\n if option in neg_opt and is_string:\n setattr(command_obj, neg_opt[option], not strtobool(value))\n elif option in bool_opts and is_string:\n setattr(command_obj, option, strtobool(value))\n elif hasattr(command_obj, option):\n setattr(command_obj, option, value)\n else:\n raise DistutilsOptionError(\n \"error in %s: command '%s' has no such option '%s'\"\n % (source, command_name, option))\n except ValueError as msg:\n raise DistutilsOptionError(msg)", "response": "Set the options for a command instance from a dictionary."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef parse_config_files(self, filenames=None, ignore_option_errors=False):\n self._parse_config_files(filenames=filenames)\n\n parse_configuration(self, self.command_options,\n ignore_option_errors=ignore_option_errors)\n self._finalize_requires()", "response": "Parses configuration files from various levels\n and loads configuration."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef parse_command_line(self):\n result = _Distribution.parse_command_line(self)\n if self.features:\n self._finalize_features()\n return result", "response": "Process features after parsing command line options"} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nresolves pre - setup requirements and return a list of distributions that are needed for building the build.", "response": "def fetch_build_eggs(self, requires):\n \"\"\"Resolve pre-setup requirements\"\"\"\n resolved_dists = pkg_resources.working_set.resolve(\n pkg_resources.parse_requirements(requires),\n installer=self.fetch_build_egg,\n replace_conflicting=True,\n )\n for dist in resolved_dists:\n pkg_resources.working_set.add(dist, replace=True)\n return resolved_dists"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef fetch_build_egg(self, req):\n from setuptools.command.easy_install import easy_install\n dist = self.__class__({'script_args': ['easy_install']})\n opts = dist.get_option_dict('easy_install')\n opts.clear()\n opts.update(\n (k, v)\n for k, v in self.get_option_dict('easy_install').items()\n if k in (\n # don't use any other settings\n 'find_links', 'site_dirs', 'index_url',\n 'optimize', 'site_dirs', 'allow_hosts',\n ))\n if self.dependency_links:\n links = self.dependency_links[:]\n if 'find_links' in opts:\n links = opts['find_links'][1] + links\n opts['find_links'] = ('setup', links)\n install_dir = self.get_egg_cache_dir()\n cmd = easy_install(\n dist, args=[\"x\"], install_dir=install_dir,\n exclude_scripts=True,\n always_copy=False, build_directory=None, editable=False,\n upgrade=False, multi_version=True, no_report=True, user=False\n )\n cmd.ensure_finalized()\n return cmd.easy_install(req)", "response": "Fetch an egg needed for building"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nadding global options based on optional features", "response": "def _set_global_opts_from_features(self):\n \"\"\"Add --with-X/--without-X options based on optional features\"\"\"\n\n go = []\n no = self.negative_opt.copy()\n\n for name, feature in self.features.items():\n self._set_feature(name, None)\n feature.validate(self)\n\n if feature.optional:\n descr = feature.description\n incdef = ' (default)'\n excdef = ''\n if not feature.include_by_default():\n excdef, incdef = incdef, excdef\n\n new = (\n ('with-' + name, None, 'include ' + descr + incdef),\n ('without-' + name, None, 'exclude ' + descr + excdef),\n )\n go.extend(new)\n no['without-' + name] = 'with-' + name\n\n self.global_options = self.feature_options = go + self.global_options\n self.negative_opt = self.feature_negopt = no"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _finalize_features(self):\n\n # First, flag all the enabled items (and thus their dependencies)\n for name, feature in self.features.items():\n enabled = self.feature_is_included(name)\n if enabled or (enabled is None and feature.include_by_default()):\n feature.include_in(self)\n self._set_feature(name, 1)\n\n # Then disable the rest, so that off-by-default features don't\n # get flagged as errors when they're required by an enabled feature\n for name, feature in self.features.items():\n if not self.feature_is_included(name):\n feature.exclude_from(self)\n self._set_feature(name, 0)", "response": "Add or remove features and resolve dependencies between them"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nsets the feature s inclusion status", "response": "def _set_feature(self, name, status):\n \"\"\"Set feature's inclusion status\"\"\"\n setattr(self, self._feature_attrname(name), status)"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef include_feature(self, name):\n\n if self.feature_is_included(name) == 0:\n descr = self.features[name].description\n raise DistutilsOptionError(\n descr + \" is required, but was excluded or is not available\"\n )\n self.features[name].include_in(self)\n self._set_feature(name, 1)", "response": "Request inclusion of a feature named name"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef include(self, **attrs):\n for k, v in attrs.items():\n include = getattr(self, '_include_' + k, None)\n if include:\n include(v)\n else:\n self._include_misc(k, v)", "response": "Add items to the distribution that are named in keyword arguments\n "} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nremove packages modules and extensions from a named package.", "response": "def exclude_package(self, package):\n \"\"\"Remove packages, modules, and extensions in named package\"\"\"\n\n pfx = package + '.'\n if self.packages:\n self.packages = [\n p for p in self.packages\n if p != package and not p.startswith(pfx)\n ]\n\n if self.py_modules:\n self.py_modules = [\n p for p in self.py_modules\n if p != package and not p.startswith(pfx)\n ]\n\n if self.ext_modules:\n self.ext_modules = [\n p for p in self.ext_modules\n if p.name != package and not p.name.startswith(pfx)\n ]"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef has_contents_for(self, package):\n\n pfx = package + '.'\n\n for p in self.iter_distribution_names():\n if p == package or p.startswith(pfx):\n return True", "response": "Return true if exclude_package ( package ) would do something"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef exclude(self, **attrs):\n for k, v in attrs.items():\n exclude = getattr(self, '_exclude_' + k, None)\n if exclude:\n exclude(v)\n else:\n self._exclude_misc(k, v)", "response": "Remove items from distribution that are named in keyword arguments\n will be excluded."} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns a dictionary of all command - line options and values.", "response": "def get_cmdline_options(self):\n \"\"\"Return a '{cmd: {opt:val}}' map of all command-line options\n\n Option names are all long, but do not include the leading '--', and\n contain dashes rather than underscores. If the option doesn't take\n an argument (e.g. '--quiet'), the 'val' is 'None'.\n\n Note that options provided by config files are intentionally excluded.\n \"\"\"\n\n d = {}\n\n for cmd, opts in self.command_options.items():\n\n for opt, (src, val) in opts.items():\n\n if src != \"command line\":\n continue\n\n opt = opt.replace('_', '-')\n\n if val == 0:\n cmdobj = self.get_command_obj(cmd)\n neg_opt = self.negative_opt.copy()\n neg_opt.update(getattr(cmdobj, 'negative_opt', {}))\n for neg, pos in neg_opt.items():\n if pos == opt:\n opt = neg\n val = None\n break\n else:\n raise AssertionError(\"Shouldn't be able to get here\")\n\n elif val == 1:\n val = None\n\n d.setdefault(cmd, {})[opt] = val\n\n return d"} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nyields all packages modules and extension names in distribution", "response": "def iter_distribution_names(self):\n \"\"\"Yield all packages, modules, and extension names in distribution\"\"\"\n\n for pkg in self.packages or ():\n yield pkg\n\n for module in self.py_modules or ():\n yield module\n\n for ext in self.ext_modules or ():\n if isinstance(ext, tuple):\n name, buildinfo = ext\n else:\n name = ext.name\n if name.endswith('module'):\n name = name[:-6]\n yield name"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nensure that the feature and its requirements are included in the distribution.", "response": "def include_in(self, dist):\n \"\"\"Ensure feature and its requirements are included in distribution\n\n You may override this in a subclass to perform additional operations on\n the distribution. Note that this method may be called more than once\n per feature, and so should be idempotent.\n\n \"\"\"\n\n if not self.available:\n raise DistutilsPlatformError(\n self.description + \" is required, \"\n \"but is not available on this platform\"\n )\n\n dist.include(**self.extras)\n\n for f in self.require_features:\n dist.include_feature(f)"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nensuring feature is excluded from distribution", "response": "def exclude_from(self, dist):\n \"\"\"Ensure feature is excluded from distribution\n\n You may override this in a subclass to perform additional operations on\n the distribution. This method will be called at most once per\n feature, and only after all included features have been asked to\n include themselves.\n \"\"\"\n\n dist.exclude(**self.extras)\n\n if self.remove:\n for item in self.remove:\n dist.exclude_package(item)"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef validate(self, dist):\n\n for item in self.remove:\n if not dist.has_contents_for(item):\n raise DistutilsSetupError(\n \"%s wants to be able to remove %s, but the distribution\"\n \" doesn't contain any packages or modules under %s\"\n % (self.description, item, item)\n )", "response": "Verify that the feature makes sense in context of distribution"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nvalidates the resource path according to the docs.", "response": "def _validate_resource_path(path):\n \"\"\"\n Validate the resource paths according to the docs.\n https://setuptools.readthedocs.io/en/latest/pkg_resources.html#basic-resource-access\n\n >>> warned = getfixture('recwarn')\n >>> warnings.simplefilter('always')\n >>> vrp = NullProvider._validate_resource_path\n >>> vrp('foo/bar.txt')\n >>> bool(warned)\n False\n >>> vrp('../foo/bar.txt')\n >>> bool(warned)\n True\n >>> warned.clear()\n >>> vrp('/foo/bar.txt')\n >>> bool(warned)\n True\n >>> vrp('foo/../../bar.txt')\n >>> bool(warned)\n True\n >>> warned.clear()\n >>> vrp('foo/f../bar.txt')\n >>> bool(warned)\n False\n\n Windows path separators are straight-up disallowed.\n >>> vrp(r'\\\\foo/bar.txt')\n Traceback (most recent call last):\n ...\n ValueError: Use of .. or absolute path in a resource path \\\nis not allowed.\n\n >>> vrp(r'C:\\\\foo/bar.txt')\n Traceback (most recent call last):\n ...\n ValueError: Use of .. or absolute path in a resource path \\\nis not allowed.\n\n Blank values are allowed\n\n >>> vrp('')\n >>> bool(warned)\n False\n\n Non-string values are not.\n\n >>> vrp(None)\n Traceback (most recent call last):\n ...\n AttributeError: ...\n \"\"\"\n invalid = (\n os.path.pardir in path.split(posixpath.sep) or\n posixpath.isabs(path) or\n ntpath.isabs(path)\n )\n if not invalid:\n return\n\n msg = \"Use of .. or absolute path in a resource path is not allowed.\"\n\n # Aggressively disallow Windows absolute paths\n if ntpath.isabs(path) and not posixpath.isabs(path):\n raise ValueError(msg)\n\n # for compatibility, warn; in future\n # raise ValueError(msg)\n warnings.warn(\n msg[:-1] + \" and will raise exceptions in a future release.\",\n DeprecationWarning,\n stacklevel=4,\n )"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _get_metadata_path_for_display(self, name):\n try:\n # We need to access _get_metadata_path() on the provider object\n # directly rather than through this class's __getattr__()\n # since _get_metadata_path() is marked private.\n path = self._provider._get_metadata_path(name)\n\n # Handle exceptions e.g. in case the distribution's metadata\n # provider doesn't support _get_metadata_path().\n except Exception:\n return '[could not detect]'\n\n return path", "response": "Return the path to the given metadata file if available."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreloads the version number of the current version of the object.", "response": "def _reload_version(self):\n \"\"\"\n Packages installed by distutils (e.g. numpy or scipy),\n which uses an old safe_version, and so\n their version numbers can get mangled when\n converted to filenames (e.g., 1.11.0.dev0+2329eae to\n 1.11.0.dev0_2329eae). These distributions will not be\n parsed properly\n downstream by Distribution and safe_version, so\n take an extra step and try to get the version number from\n the metadata file itself instead of the filename.\n \"\"\"\n md_version = self._get_version()\n if md_version:\n self._version = md_version\n return self"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef parse_bdist_wininst(name):\n\n lower = name.lower()\n base, py_ver, plat = None, None, None\n\n if lower.endswith('.exe'):\n if lower.endswith('.win32.exe'):\n base = name[:-10]\n plat = 'win32'\n elif lower.startswith('.win32-py', -16):\n py_ver = name[-7:-4]\n base = name[:-16]\n plat = 'win32'\n elif lower.endswith('.win-amd64.exe'):\n base = name[:-14]\n plat = 'win-amd64'\n elif lower.startswith('.win-amd64-py', -20):\n py_ver = name[-7:-4]\n base = name[:-20]\n plat = 'win-amd64'\n return base, py_ver, plat", "response": "Return base pyversion plat for possible. exe name"} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nyields all egg or source distribution objects that might be found at a given URL.", "response": "def distros_for_url(url, metadata=None):\n \"\"\"Yield egg or source distribution objects that might be found at a URL\"\"\"\n base, fragment = egg_info_for_url(url)\n for dist in distros_for_location(url, base, metadata):\n yield dist\n if fragment:\n match = EGG_FRAGMENT.match(fragment)\n if match:\n for dist in interpret_distro_name(\n url, match.group(1), metadata, precedence=CHECKOUT_DIST\n ):\n yield dist"} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nyields a list of distributions for a given location.", "response": "def distros_for_location(location, basename, metadata=None):\n \"\"\"Yield egg or source distribution objects based on basename\"\"\"\n if basename.endswith('.egg.zip'):\n basename = basename[:-4] # strip the .zip\n if basename.endswith('.egg') and '-' in basename:\n # only one, unambiguous interpretation\n return [Distribution.from_location(location, basename, metadata)]\n if basename.endswith('.whl') and '-' in basename:\n wheel = Wheel(basename)\n if not wheel.is_compatible():\n return []\n return [Distribution(\n location=location,\n project_name=wheel.project_name,\n version=wheel.version,\n # Increase priority over eggs.\n precedence=EGG_DIST + 1,\n )]\n if basename.endswith('.exe'):\n win_base, py_ver, platform = parse_bdist_wininst(basename)\n if win_base is not None:\n return interpret_distro_name(\n location, win_base, metadata, py_ver, BINARY_DIST, platform\n )\n # Try source distro extensions (.zip, .tgz, etc.)\n #\n for ext in EXTENSIONS:\n if basename.endswith(ext):\n basename = basename[:-len(ext)]\n return interpret_distro_name(location, basename, metadata)\n return []"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nyielding possible egg or source distribution objects based on a filename.", "response": "def distros_for_filename(filename, metadata=None):\n \"\"\"Yield possible egg or source distribution objects based on a filename\"\"\"\n return distros_for_location(\n normalize_path(filename), os.path.basename(filename), metadata\n )"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef interpret_distro_name(\n location, basename, metadata, py_version=None, precedence=SOURCE_DIST,\n platform=None\n):\n \"\"\"Generate alternative interpretations of a source distro name\n\n Note: if `location` is a filesystem filename, you should call\n ``pkg_resources.normalize_path()`` on it before passing it to this\n routine!\n \"\"\"\n # Generate alternative interpretations of a source distro name\n # Because some packages are ambiguous as to name/versions split\n # e.g. \"adns-python-1.1.0\", \"egenix-mx-commercial\", etc.\n # So, we generate each possible interepretation (e.g. \"adns, python-1.1.0\"\n # \"adns-python, 1.1.0\", and \"adns-python-1.1.0, no version\"). In practice,\n # the spurious interpretations should be ignored, because in the event\n # there's also an \"adns\" package, the spurious \"python-1.1.0\" version will\n # compare lower than any numeric version number, and is therefore unlikely\n # to match a request for it. It's still a potential problem, though, and\n # in the long run PyPI and the distutils should go for \"safe\" names and\n # versions in distribution archive names (sdist and bdist).\n\n parts = basename.split('-')\n if not py_version and any(re.match(r'py\\d\\.\\d$', p) for p in parts[2:]):\n # it is a bdist_dumb, not an sdist -- bail out\n return\n\n for p in range(1, len(parts) + 1):\n yield Distribution(\n location, metadata, '-'.join(parts[:p]), '-'.join(parts[p:]),\n py_version=py_version, precedence=precedence,\n platform=platform\n )", "response": "Generate an alternative interpretation of a source distro name."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef unique_values(func):\n\n @wraps(func)\n def wrapper(*args, **kwargs):\n return unique_everseen(func(*args, **kwargs))\n\n return wrapper", "response": "A decorator that returns an iterable that yields unique items."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nencoding auth string to base64.", "response": "def _encode_auth(auth):\n \"\"\"\n A function compatible with Python 2.3-3.3 that will encode\n auth from a URL suitable for an HTTP header.\n >>> str(_encode_auth('username%3Apassword'))\n 'dXNlcm5hbWU6cGFzc3dvcmQ='\n\n Long auth strings should not cause a newline to be inserted.\n >>> long_auth = 'username:' + 'password'*10\n >>> chr(10) in str(_encode_auth(long_auth))\n False\n \"\"\"\n auth_s = urllib.parse.unquote(auth)\n # convert to bytes\n auth_bytes = auth_s.encode()\n encoded_bytes = base64.b64encode(auth_bytes)\n # convert back to a string\n encoded = encoded_bytes.decode()\n # strip the trailing carriage return\n return encoded.replace('\\n', '')"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _splituser(host):\n user, delim, host = host.rpartition('@')\n return (user if delim else None), host", "response": "Split host into user and host parts."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nopening a local path with special support for files and directories", "response": "def local_open(url):\n \"\"\"Read a local path, with special support for directories\"\"\"\n scheme, server, path, param, query, frag = urllib.parse.urlparse(url)\n filename = urllib.request.url2pathname(path)\n if os.path.isfile(filename):\n return urllib.request.urlopen(url)\n elif path.endswith('/') and os.path.isdir(filename):\n files = []\n for f in os.listdir(filename):\n filepath = os.path.join(filename, f)\n if f == 'index.html':\n with open(filepath, 'r') as fp:\n body = fp.read()\n break\n elif os.path.isdir(filepath):\n f += '/'\n files.append('{name}'.format(name=f))\n else:\n tmpl = (\n \"{url}\"\n \"{files}\")\n body = tmpl.format(url=url, files='\\n'.join(files))\n status, message = 200, \"OK\"\n else:\n status, message, body = 404, \"Path not found\", \"Not found\"\n\n headers = {'content-type': 'text/html'}\n body_stream = six.StringIO(body)\n return urllib.error.HTTPError(url, status, message, headers, body_stream)"} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nprocess the contents of a PyPI page into the index page.", "response": "def process_index(self, url, page):\n \"\"\"Process the contents of a PyPI page\"\"\"\n\n def scan(link):\n # Process a URL to see if it's for a package page\n if link.startswith(self.index_url):\n parts = list(map(\n urllib.parse.unquote, link[len(self.index_url):].split('/')\n ))\n if len(parts) == 2 and '#' not in parts[1]:\n # it's a package page, sanitize and index it\n pkg = safe_name(parts[0])\n ver = safe_version(parts[1])\n self.package_pages.setdefault(pkg.lower(), {})[link] = True\n return to_filename(pkg), to_filename(ver)\n return None, None\n\n # process an index page into the package-page index\n for match in HREF.finditer(page):\n try:\n scan(urllib.parse.urljoin(url, htmldecode(match.group(1))))\n except ValueError:\n pass\n\n pkg, ver = scan(url) # ensure this page is in the page index\n if pkg:\n # process individual package page\n for new_url in find_external_links(url, page):\n # Process the found URL\n base, frag = egg_info_for_url(new_url)\n if base.endswith('.py') and not frag:\n if ver:\n new_url += '#egg=%s-%s' % (pkg, ver)\n else:\n self.need_version_info(url)\n self.scan_url(new_url)\n\n return PYPI_MD5.sub(\n lambda m: '%s' % m.group(1, 3, 2), page\n )\n else:\n return \"\""} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef check_hash(self, checker, filename, tfp):\n checker.report(\n self.debug,\n \"Validating %%s checksum for %s\" % filename)\n if not checker.is_valid():\n tfp.close()\n os.unlink(filename)\n raise DistutilsError(\n \"%s validation failed for %s; \"\n \"possible download problem?\"\n % (checker.hash.name, os.path.basename(filename))\n )", "response": "Check the checksum of a file."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nadd urls to the list that will be prescanned for searches", "response": "def add_find_links(self, urls):\n \"\"\"Add `urls` to the list that will be prescanned for searches\"\"\"\n for url in urls:\n if (\n self.to_scan is None # if we have already \"gone online\"\n or not URL_SCHEME(url) # or it's a local file/directory\n or url.startswith('file:')\n or list(distros_for_url(url)) # or a direct package link\n ):\n # then go ahead and process it now\n self.scan_url(url)\n else:\n # otherwise, defer retrieval till later\n self.to_scan.append(url)"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nremoving any links that are not in the list of to_scan.", "response": "def prescan(self):\n \"\"\"Scan urls scheduled for prescanning (e.g. --find-links)\"\"\"\n if self.to_scan:\n list(map(self.scan_url, self.to_scan))\n self.to_scan = None"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef download(self, spec, tmpdir):\n if not isinstance(spec, Requirement):\n scheme = URL_SCHEME(spec)\n if scheme:\n # It's a url, download it to tmpdir\n found = self._download_url(scheme.group(1), spec, tmpdir)\n base, fragment = egg_info_for_url(spec)\n if base.endswith('.py'):\n found = self.gen_setup(found, fragment, tmpdir)\n return found\n elif os.path.exists(spec):\n # Existing file or directory, just return it\n return spec\n else:\n spec = parse_requirement_arg(spec)\n return getattr(self.fetch_distribution(spec, tmpdir), 'location', None)", "response": "Locate and or download a specific version of a specific distribution from the specified directory."} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nobtain a file suitable for fulfilling requirement.", "response": "def fetch(self, requirement, tmpdir, force_scan=False, source=False):\n \"\"\"Obtain a file suitable for fulfilling `requirement`\n\n DEPRECATED; use the ``fetch_distribution()`` method now instead. For\n backward compatibility, this routine is identical but returns the\n ``location`` of the downloaded distribution instead of a distribution\n object.\n \"\"\"\n dist = self.fetch_distribution(requirement, tmpdir, force_scan, source)\n if dist is not None:\n return dist.location\n return None"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef find_credential(self, url):\n for repository, cred in self.creds_by_repository.items():\n if url.startswith(repository):\n return cred", "response": "Find the credential for the given URL."} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ngets the filename of the distutils local global or per - user config", "response": "def config_file(kind=\"local\"):\n \"\"\"Get the filename of the distutils, local, global, or per-user config\n\n `kind` must be one of \"local\", \"global\", or \"user\"\n \"\"\"\n if kind == 'local':\n return 'setup.cfg'\n if kind == 'global':\n return os.path.join(\n os.path.dirname(distutils.__file__), 'distutils.cfg'\n )\n if kind == 'user':\n dot = os.name == 'posix' and '.' or ''\n return os.path.expanduser(convert_path(\"~/%spydistutils.cfg\" % dot))\n raise ValueError(\n \"config_file() type must be 'local', 'global', or 'user'\", kind\n )"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef edit_config(filename, settings, dry_run=False):\n log.debug(\"Reading configuration from %s\", filename)\n opts = configparser.RawConfigParser()\n opts.read([filename])\n for section, options in settings.items():\n if options is None:\n log.info(\"Deleting section [%s] from %s\", section, filename)\n opts.remove_section(section)\n else:\n if not opts.has_section(section):\n log.debug(\"Adding new section [%s] to %s\", section, filename)\n opts.add_section(section)\n for option, value in options.items():\n if value is None:\n log.debug(\n \"Deleting %s.%s from %s\",\n section, option, filename\n )\n opts.remove_option(section, option)\n if not opts.options(section):\n log.info(\"Deleting empty [%s] section from %s\",\n section, filename)\n opts.remove_section(section)\n else:\n log.debug(\n \"Setting %s.%s to %r in %s\",\n section, option, value, filename\n )\n opts.set(section, option, value)\n\n log.info(\"Writing %s\", filename)\n if not dry_run:\n with open(filename, 'w') as f:\n opts.write(f)", "response": "Edit a configuration file to include settings."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef hide_file(path):\n __import__('ctypes.wintypes')\n SetFileAttributes = ctypes.windll.kernel32.SetFileAttributesW\n SetFileAttributes.argtypes = ctypes.wintypes.LPWSTR, ctypes.wintypes.DWORD\n SetFileAttributes.restype = ctypes.wintypes.BOOL\n\n FILE_ATTRIBUTE_HIDDEN = 0x02\n\n ret = SetFileAttributes(path, FILE_ATTRIBUTE_HIDDEN)\n if not ret:\n raise ctypes.WinError()", "response": "Sets the hidden attribute on a file or directory."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nfind the module that is vendored by this importer.", "response": "def find_module(self, fullname, path=None):\n \"\"\"\n Return self when fullname starts with root_name and the\n target module is one vendored through this importer.\n \"\"\"\n root, base, target = fullname.partition(self.root_name + '.')\n if root:\n return\n if not any(map(target.startswith, self.vendored_names)):\n return\n return self"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef load_module(self, fullname):\n root, base, target = fullname.partition(self.root_name + '.')\n for prefix in self.search_path:\n try:\n extant = prefix + target\n __import__(extant)\n mod = sys.modules[extant]\n sys.modules[fullname] = mod\n # mysterious hack:\n # Remove the reference to the extant package/module\n # on later Python versions to cause relative imports\n # in the vendor package to resolve the same modules\n # as those going through this importer.\n if prefix and sys.version_info > (3, 3):\n del sys.modules[extant]\n return mod\n except ImportError:\n pass\n else:\n raise ImportError(\n \"The '{target}' package is required; \"\n \"normally this is bundled with this package so if you get \"\n \"this warning, consult the packager of your \"\n \"distribution.\".format(**locals())\n )", "response": "Load the module with the given name."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nadding all the default files to the list.", "response": "def add_defaults(self):\n \"\"\"Add all the default files to self.filelist:\n - README or README.txt\n - setup.py\n - test/test*.py\n - all pure Python modules mentioned in setup script\n - all files pointed by package_data (build_py)\n - all files defined in data_files.\n - all files defined as scripts.\n - all C sources listed as part of extensions or C libraries\n in the setup script (doesn't catch C headers!)\n Warns if (README or README.txt) or setup.py are missing; everything\n else is optional.\n \"\"\"\n self._add_defaults_standards()\n self._add_defaults_optional()\n self._add_defaults_python()\n self._add_defaults_data_files()\n self._add_defaults_ext()\n self._add_defaults_c_libs()\n self._add_defaults_scripts()"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ncheck if a path exists in the current directory.", "response": "def _cs_path_exists(fspath):\n \"\"\"\n Case-sensitive path existence check\n\n >>> sdist_add_defaults._cs_path_exists(__file__)\n True\n >>> sdist_add_defaults._cs_path_exists(__file__.upper())\n False\n \"\"\"\n if not os.path.exists(fspath):\n return False\n # make absolute so we always have a directory\n abspath = os.path.abspath(fspath)\n directory, filename = os.path.split(abspath)\n return filename in os.listdir(directory)"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nensuring that the given path is decoded", "response": "def filesys_decode(path):\n \"\"\"\n Ensure that the given path is decoded,\n NONE when no expected encoding works\n \"\"\"\n\n if isinstance(path, six.text_type):\n return path\n\n fs_enc = sys.getfilesystemencoding() or 'utf-8'\n candidates = fs_enc, 'utf-8'\n\n for enc in candidates:\n try:\n return path.decode(enc)\n except UnicodeDecodeError:\n continue"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_flag(var, fallback, expected=True, warn=True):\n val = get_config_var(var)\n if val is None:\n if warn:\n log.debug(\"Config variable '%s' is unset, Python ABI tag may \"\n \"be incorrect\", var)\n return fallback()\n return val == expected", "response": "Get the SOABI flag for a given config variable."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn a list of supported arches for a given major minor and machine.", "response": "def get_darwin_arches(major, minor, machine):\n \"\"\"Return a list of supported arches (including group arches) for\n the given major, minor and machine architecture of a macOS machine.\n \"\"\"\n arches = []\n\n def _supports_arch(major, minor, arch):\n # Looking at the application support for macOS versions in the chart\n # provided by https://en.wikipedia.org/wiki/OS_X#Versions it appears\n # our timeline looks roughly like:\n #\n # 10.0 - Introduces ppc support.\n # 10.4 - Introduces ppc64, i386, and x86_64 support, however the ppc64\n # and x86_64 support is CLI only, and cannot be used for GUI\n # applications.\n # 10.5 - Extends ppc64 and x86_64 support to cover GUI applications.\n # 10.6 - Drops support for ppc64\n # 10.7 - Drops support for ppc\n #\n # Given that we do not know if we're installing a CLI or a GUI\n # application, we must be conservative and assume it might be a GUI\n # application and behave as if ppc64 and x86_64 support did not occur\n # until 10.5.\n #\n # Note: The above information is taken from the \"Application support\"\n # column in the chart not the \"Processor support\" since I believe\n # that we care about what instruction sets an application can use\n # not which processors the OS supports.\n if arch == 'ppc':\n return (major, minor) <= (10, 5)\n if arch == 'ppc64':\n return (major, minor) == (10, 5)\n if arch == 'i386':\n return (major, minor) >= (10, 4)\n if arch == 'x86_64':\n return (major, minor) >= (10, 5)\n if arch in groups:\n for garch in groups[arch]:\n if _supports_arch(major, minor, garch):\n return True\n return False\n\n groups = OrderedDict([\n (\"fat\", (\"i386\", \"ppc\")),\n (\"intel\", (\"x86_64\", \"i386\")),\n (\"fat64\", (\"x86_64\", \"ppc64\")),\n (\"fat32\", (\"x86_64\", \"i386\", \"ppc\")),\n ])\n\n if _supports_arch(major, minor, machine):\n arches.append(machine)\n\n for garch in groups:\n if machine in groups[garch] and _supports_arch(major, minor, garch):\n arches.append(garch)\n\n arches.append('universal')\n\n return arches"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_supported(versions=None, noarch=False, platform=None,\n impl=None, abi=None):\n \"\"\"Return a list of supported tags for each version specified in\n `versions`.\n\n :param versions: a list of string versions, of the form [\"33\", \"32\"],\n or None. The first version will be assumed to support our ABI.\n :param platform: specify the exact platform you want valid\n tags for, or None. If None, use the local system platform.\n :param impl: specify the exact implementation you want valid\n tags for, or None. If None, use the local interpreter impl.\n :param abi: specify the exact abi you want valid\n tags for, or None. If None, use the local interpreter abi.\n \"\"\"\n supported = []\n\n # Versions must be given with respect to the preference\n if versions is None:\n versions = []\n version_info = get_impl_version_info()\n major = version_info[:-1]\n # Support all previous minor Python versions.\n for minor in range(version_info[-1], -1, -1):\n versions.append(''.join(map(str, major + (minor,))))\n\n impl = impl or get_abbr_impl()\n\n abis = []\n\n abi = abi or get_abi_tag()\n if abi:\n abis[0:0] = [abi]\n\n abi3s = set()\n import imp\n for suffix in imp.get_suffixes():\n if suffix[0].startswith('.abi'):\n abi3s.add(suffix[0].split('.', 2)[1])\n\n abis.extend(sorted(list(abi3s)))\n\n abis.append('none')\n\n if not noarch:\n arch = platform or get_platform()\n if arch.startswith('macosx'):\n # support macosx-10.6-intel on macosx-10.9-x86_64\n match = _osx_arch_pat.match(arch)\n if match:\n name, major, minor, actual_arch = match.groups()\n tpl = '{}_{}_%i_%s'.format(name, major)\n arches = []\n for m in reversed(range(int(minor) + 1)):\n for a in get_darwin_arches(int(major), m, actual_arch):\n arches.append(tpl % (m, a))\n else:\n # arch pattern didn't match (?!)\n arches = [arch]\n elif platform is None and is_manylinux1_compatible():\n arches = [arch.replace('linux', 'manylinux1'), arch]\n else:\n arches = [arch]\n\n # Current version, current API (built specifically for our Python):\n for abi in abis:\n for arch in arches:\n supported.append(('%s%s' % (impl, versions[0]), abi, arch))\n\n # abi3 modules compatible with older version of Python\n for version in versions[1:]:\n # abi3 was introduced in Python 3.2\n if version in {'31', '30'}:\n break\n for abi in abi3s: # empty set if not Python 3\n for arch in arches:\n supported.append((\"%s%s\" % (impl, version), abi, arch))\n\n # Has binaries, does not use the Python API:\n for arch in arches:\n supported.append(('py%s' % (versions[0][0]), 'none', arch))\n\n # No abi / arch, but requires our implementation:\n supported.append(('%s%s' % (impl, versions[0]), 'none', 'any'))\n # Tagged specifically as being cross-version compatible\n # (with just the major version specified)\n supported.append(('%s%s' % (impl, versions[0][0]), 'none', 'any'))\n\n # No abi / arch, generic Python\n for i, version in enumerate(versions):\n supported.append(('py%s' % (version,), 'none', 'any'))\n if i == 0:\n supported.append(('py%s' % (version[0]), 'none', 'any'))\n\n return supported", "response": "Return a list of tags that are supported by the current version of the current language."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _find_all_simple(path):\n results = (\n os.path.join(base, file)\n for base, dirs, files in os.walk(path, followlinks=True)\n for file in files\n )\n return filter(os.path.isfile, results)", "response": "Find all files under path and return a list of all the files under path."} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nfinds all files under dir and return the list of full filenames.", "response": "def findall(dir=os.curdir):\n \"\"\"\n Find all files under 'dir' and return the list of full filenames.\n Unless dir is '.', return full filenames with dir prepended.\n \"\"\"\n files = _find_all_simple(dir)\n if dir == os.curdir:\n make_rel = functools.partial(os.path.relpath, start=dir)\n files = map(make_rel, files)\n return list(files)"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef find(cls, where='.', exclude=(), include=('*',)):\n\n return list(cls._find_packages_iter(\n convert_path(where),\n cls._build_filter('ez_setup', '*__pycache__', *exclude),\n cls._build_filter(*include)))", "response": "Return a list of Python packages found within a directory where."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _find_packages_iter(cls, where, exclude, include):\n for root, dirs, files in os.walk(where, followlinks=True):\n # Copy dirs to iterate over it, then empty dirs.\n all_dirs = dirs[:]\n dirs[:] = []\n\n for dir in all_dirs:\n full_path = os.path.join(root, dir)\n rel_path = os.path.relpath(full_path, where)\n package = rel_path.replace(os.path.sep, '.')\n\n # Skip directory trees that are not valid packages\n if ('.' in dir or not cls._looks_like_package(full_path)):\n continue\n\n # Should this package be included?\n if include(package) and not exclude(package):\n yield package\n\n # Keep searching subdirectories, as there may be more packages\n # down there, even if the parent was excluded.\n dirs.append(dir)", "response": "Iterate over all the packages in where and yield them."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nbuild a filter that will be true only if the input matches at least one of the patterns.", "response": "def _build_filter(*patterns):\n \"\"\"\n Given a list of patterns, return a callable that will be true only if\n the input matches at least one of the patterns.\n \"\"\"\n return lambda name: any(fnmatchcase(name, pat=pat) for pat in patterns)"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef ensure_string_list(self, option):\n val = getattr(self, option)\n if val is None:\n return\n elif isinstance(val, string_types):\n setattr(self, option, re.split(r',\\s*|\\s+', val))\n else:\n if isinstance(val, list):\n ok = all(isinstance(v, string_types) for v in val)\n else:\n ok = False\n if not ok:\n raise DistutilsOptionError(\n \"'%s' must be a list of strings (got %r)\"\n % (option, val))", "response": "r Ensure that option is a list of strings."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ndetermining whether run was called from setup or by another base command.", "response": "def _called_from_setup(run_frame):\n \"\"\"\n Attempt to detect whether run() was called from setup() or by another\n command. If called by setup(), the parent caller will be the\n 'run_command' method in 'distutils.dist', and *its* caller will be\n the 'run_commands' method. If called any other way, the\n immediate caller *might* be 'run_command', but it won't have been\n called by 'run_commands'. Return True in that case or if a call stack\n is unavailable. Return False otherwise.\n \"\"\"\n if run_frame is None:\n msg = \"Call stack not available. bdist_* commands may fail.\"\n warnings.warn(msg)\n if platform.python_implementation() == 'IronPython':\n msg = \"For best results, pass -X:Frames to enable call stack.\"\n warnings.warn(msg)\n return True\n res = inspect.getouterframes(run_frame)[2]\n caller, = res[:1]\n info = inspect.getframeinfo(caller)\n caller_module = caller.f_globals.get('__name__', '')\n return (\n caller_module == 'distutils.dist'\n and info.function == 'run_commands'\n )"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef newer_pairwise_group(sources_groups, targets):\n if len(sources_groups) != len(targets):\n raise ValueError(\"'sources_group' and 'targets' must be the same length\")\n\n # build a pair of lists (sources_groups, targets) where source is newer\n n_sources = []\n n_targets = []\n for i in range(len(sources_groups)):\n if newer_group(sources_groups[i], targets[i]):\n n_sources.append(sources_groups[i])\n n_targets.append(targets[i])\n\n return n_sources, n_targets", "response": "Walk both arguments in parallel testing if each source group is newer than its corresponding target. Returns a pair of lists where sources is newer than target."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef sorted_walk(dir):\n for base, dirs, files in os.walk(dir):\n dirs.sort()\n files.sort()\n yield base, dirs, files", "response": "A generator that iterates over a directory and returns a list of all the files in that directory."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef walk_egg(egg_dir):\n walker = sorted_walk(egg_dir)\n base, dirs, files = next(walker)\n if 'EGG-INFO' in dirs:\n dirs.remove('EGG-INFO')\n yield base, dirs, files\n for bdf in walker:\n yield bdf", "response": "Walk an unpacked egg s contents skipping the metadata directory"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nyielding names and strings used by code and its nested code objects", "response": "def iter_symbols(code):\n \"\"\"Yield names and strings used by `code` and its nested code objects\"\"\"\n for name in code.co_names:\n yield name\n for const in code.co_consts:\n if isinstance(const, six.string_types):\n yield const\n elif isinstance(const, CodeType):\n for name in iter_symbols(const):\n yield name"} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncreate a zip file from all the files under base_dir.", "response": "def make_zipfile(zip_filename, base_dir, verbose=0, dry_run=0, compress=True,\n mode='w'):\n \"\"\"Create a zip file from all the files under 'base_dir'. The output\n zip file will be named 'base_dir' + \".zip\". Uses either the \"zipfile\"\n Python module (if available) or the InfoZIP \"zip\" utility (if installed\n and found on the default search path). If neither tool is available,\n raises DistutilsExecError. Returns the name of the output zip file.\n \"\"\"\n import zipfile\n\n mkpath(os.path.dirname(zip_filename), dry_run=dry_run)\n log.info(\"creating '%s' and adding '%s' to it\", zip_filename, base_dir)\n\n def visit(z, dirname, names):\n for name in names:\n path = os.path.normpath(os.path.join(dirname, name))\n if os.path.isfile(path):\n p = path[len(base_dir) + 1:]\n if not dry_run:\n z.write(path, p)\n log.debug(\"adding '%s'\", p)\n\n compression = zipfile.ZIP_DEFLATED if compress else zipfile.ZIP_STORED\n if not dry_run:\n z = zipfile.ZipFile(zip_filename, mode, compression=compression)\n for dirname, dirs, files in sorted_walk(base_dir):\n visit(z, dirname, files)\n z.close()\n else:\n for dirname, dirs, files in sorted_walk(base_dir):\n visit(None, dirname, files)\n return zip_filename"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ninvokes a command with keyword args", "response": "def call_command(self, cmdname, **kw):\n \"\"\"Invoke reinitialized command `cmdname` with keyword args\"\"\"\n for dirname in INSTALL_DIRECTORY_ATTRS:\n kw.setdefault(dirname, self.bdist_dir)\n kw.setdefault('skip_build', self.skip_build)\n kw.setdefault('dry_run', self.dry_run)\n cmd = self.reinitialize_command(cmdname, **kw)\n self.run_command(cmdname)\n return cmd"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef copy_metadata_to(self, target_dir):\n \"Copy metadata (egg info) to the target_dir\"\n # normalize the path (so that a forward-slash in egg_info will\n # match using startswith below)\n norm_egg_info = os.path.normpath(self.egg_info)\n prefix = os.path.join(norm_egg_info, '')\n for path in self.ei_cmd.filelist.files:\n if path.startswith(prefix):\n target = os.path.join(target_dir, path[len(prefix):])\n ensure_directory(target)\n self.copy_file(path, target)", "response": "Copy metadata ( egg info ) to the target_dir"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_ext_outputs(self):\n\n all_outputs = []\n ext_outputs = []\n\n paths = {self.bdist_dir: ''}\n for base, dirs, files in sorted_walk(self.bdist_dir):\n for filename in files:\n if os.path.splitext(filename)[1].lower() in NATIVE_EXTENSIONS:\n all_outputs.append(paths[base] + filename)\n for filename in dirs:\n paths[os.path.join(base, filename)] = (paths[base] +\n filename + '/')\n\n if self.distribution.has_ext_modules():\n build_cmd = self.get_finalized_command('build_ext')\n for ext in build_cmd.extensions:\n if isinstance(ext, Library):\n continue\n fullname = build_cmd.get_ext_fullname(ext.name)\n filename = build_cmd.get_ext_filename(fullname)\n if not os.path.basename(filename).startswith('dl-'):\n if os.path.exists(os.path.join(self.bdist_dir, filename)):\n ext_outputs.append(filename)\n\n return all_outputs, ext_outputs", "response": "Get a list of relative paths to C extensions in the output distro"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef translate_pattern(glob):\n pat = ''\n\n # This will split on '/' within [character classes]. This is deliberate.\n chunks = glob.split(os.path.sep)\n\n sep = re.escape(os.sep)\n valid_char = '[^%s]' % (sep,)\n\n for c, chunk in enumerate(chunks):\n last_chunk = c == len(chunks) - 1\n\n # Chunks that are a literal ** are globstars. They match anything.\n if chunk == '**':\n if last_chunk:\n # Match anything if this is the last component\n pat += '.*'\n else:\n # Match '(name/)*'\n pat += '(?:%s+%s)*' % (valid_char, sep)\n continue # Break here as the whole path component has been handled\n\n # Find any special characters in the remainder\n i = 0\n chunk_len = len(chunk)\n while i < chunk_len:\n char = chunk[i]\n if char == '*':\n # Match any number of name characters\n pat += valid_char + '*'\n elif char == '?':\n # Match a name character\n pat += valid_char\n elif char == '[':\n # Character class\n inner_i = i + 1\n # Skip initial !/] chars\n if inner_i < chunk_len and chunk[inner_i] == '!':\n inner_i = inner_i + 1\n if inner_i < chunk_len and chunk[inner_i] == ']':\n inner_i = inner_i + 1\n\n # Loop till the closing ] is found\n while inner_i < chunk_len and chunk[inner_i] != ']':\n inner_i = inner_i + 1\n\n if inner_i >= chunk_len:\n # Got to the end of the string without finding a closing ]\n # Do not treat this as a matching group, but as a literal [\n pat += re.escape(char)\n else:\n # Grab the insides of the [brackets]\n inner = chunk[i + 1:inner_i]\n char_class = ''\n\n # Class negation\n if inner[0] == '!':\n char_class = '^'\n inner = inner[1:]\n\n char_class += re.escape(inner)\n pat += '[%s]' % (char_class,)\n\n # Skip to the end ]\n i = inner_i\n else:\n pat += re.escape(char)\n i += 1\n\n # Join each chunk with the dir separator\n if not last_chunk:\n pat += sep\n\n pat += r'\\Z'\n return re.compile(pat, flags=re.MULTILINE|re.DOTALL)", "response": "Translate a glob like. txt in to a regular expression."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef write_file(filename, contents):\n contents = \"\\n\".join(contents)\n\n # assuming the contents has been vetted for utf-8 encoding\n contents = contents.encode(\"utf-8\")\n\n with open(filename, \"wb\") as f: # always write POSIX-style manifest\n f.write(contents)", "response": "Write the contents to a file."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get_pkg_info_revision():\n warnings.warn(\"get_pkg_info_revision is deprecated.\", EggInfoDeprecationWarning)\n if os.path.exists('PKG-INFO'):\n with io.open('PKG-INFO') as f:\n for line in f:\n match = re.match(r\"Version:.*-r(\\d+)\\s*$\", line)\n if match:\n return int(match.group(1))\n return 0", "response": "Get a - r### off of PKG - INFO Version in case this is an sdist of\n a subversion revision."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef save_version_info(self, filename):\n egg_info = collections.OrderedDict()\n # follow the order these keys would have been added\n # when PYTHONHASHSEED=0\n egg_info['tag_build'] = self.tags()\n egg_info['tag_date'] = 0\n edit_config(filename, dict(egg_info=egg_info))", "response": "Save the build tag and date into the version info file."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nwrite data to filename or delete if filename exists.", "response": "def write_or_delete_file(self, what, filename, data, force=False):\n \"\"\"Write `data` to `filename` or delete if empty\n\n If `data` is non-empty, this routine is the same as ``write_file()``.\n If `data` is empty but not ``None``, this is the same as calling\n ``delete_file(filename)`. If `data` is ``None``, then this is a no-op\n unless `filename` exists, in which case a warning is issued about the\n orphaned file (if `force` is false), or deleted (if `force` is true).\n \"\"\"\n if data:\n self.write_file(what, filename, data)\n elif os.path.exists(filename):\n if data is None and not force:\n log.warn(\n \"%s not set in setup(), but %s exists\", what, filename\n )\n return\n else:\n self.delete_file(filename)"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef delete_file(self, filename):\n log.info(\"deleting %s\", filename)\n if not self.dry_run:\n os.unlink(filename)", "response": "Delete filename after announcing it"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ngenerating SOURCES. txt manifest file", "response": "def find_sources(self):\n \"\"\"Generate SOURCES.txt manifest file\"\"\"\n manifest_filename = os.path.join(self.egg_info, \"SOURCES.txt\")\n mm = manifest_maker(self.distribution)\n mm.manifest = manifest_filename\n mm.run()\n self.filelist = mm.filelist"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _remove_files(self, predicate):\n found = False\n for i in range(len(self.files) - 1, -1, -1):\n if predicate(self.files[i]):\n self.debug_print(\" removing \" + self.files[i])\n del self.files[i]\n found = True\n return found", "response": "Remove all files that match the predicate. Return True if any matching files were removed."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ninclude files that match pattern.", "response": "def include(self, pattern):\n \"\"\"Include files that match 'pattern'.\"\"\"\n found = [f for f in glob(pattern) if not os.path.isdir(f)]\n self.extend(found)\n return bool(found)"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef exclude(self, pattern):\n match = translate_pattern(pattern)\n return self._remove_files(match.match)", "response": "Exclude files that match pattern."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nincludes all files anywhere in dir that match the pattern.", "response": "def recursive_include(self, dir, pattern):\n \"\"\"\n Include all files anywhere in 'dir/' that match the pattern.\n \"\"\"\n full_pattern = os.path.join(dir, '**', pattern)\n found = [f for f in glob(full_pattern, recursive=True)\n if not os.path.isdir(f)]\n self.extend(found)\n return bool(found)"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef recursive_exclude(self, dir, pattern):\n match = translate_pattern(os.path.join(dir, '**', pattern))\n return self._remove_files(match.match)", "response": "Recursively exclude any files in dir that match the pattern."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef graft(self, dir):\n found = [\n item\n for match_dir in glob(dir)\n for item in distutils.filelist.findall(match_dir)\n ]\n self.extend(found)\n return bool(found)", "response": "Include all files from dir"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef prune(self, dir):\n match = translate_pattern(os.path.join(dir, '**'))\n return self._remove_files(match.match)", "response": "Remove all files from dir."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef global_include(self, pattern):\n if self.allfiles is None:\n self.findall()\n match = translate_pattern(os.path.join('**', pattern))\n found = [f for f in self.allfiles if match.match(f)]\n self.extend(found)\n return bool(found)", "response": "Include all files that match the pattern."} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nexcludes all files anywhere that match the pattern.", "response": "def global_exclude(self, pattern):\n \"\"\"\n Exclude all files anywhere that match the pattern.\n \"\"\"\n match = translate_pattern(os.path.join('**', pattern))\n return self._remove_files(match.match)"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _repair(self):\n self.files = list(filter(self._safe_path, self.files))", "response": "Replace self. files with only safe paths"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef write_manifest(self):\n self.filelist._repair()\n\n # Now _repairs should encodability, but not unicode\n files = [self._manifest_normalize(f) for f in self.filelist.files]\n msg = \"writing manifest file '%s'\" % self.manifest\n self.execute(write_file, (self.manifest, files), msg)", "response": "Write the file list in self. filelist to the manifest file named by self. manifest."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nwalk the revision control directory and yield all files under it", "response": "def walk_revctrl(dirname=''):\n \"\"\"Find all files under revision control\"\"\"\n for ep in pkg_resources.iter_entry_points('setuptools.file_finders'):\n for item in ep.load()(dirname):\n yield item"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nremoves and restore os. link if it exists", "response": "def _remove_os_link():\n \"\"\"\n In a context, remove and restore os.link if it exists\n \"\"\"\n\n class NoValue:\n pass\n\n orig_val = getattr(os, 'link', NoValue)\n try:\n del os.link\n except Exception:\n pass\n try:\n yield\n finally:\n if orig_val is not NoValue:\n setattr(os, 'link', orig_val)"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _add_defaults_python(self):\n if self.distribution.has_pure_modules():\n build_py = self.get_finalized_command('build_py')\n self.filelist.extend(build_py.get_source_files())\n # This functionality is incompatible with include_package_data, and\n # will in fact create an infinite recursion if include_package_data\n # is True. Use of include_package_data will imply that\n # distutils-style automatic handling of package_data is disabled\n if not self.distribution.include_package_data:\n for _, src_dir, _, filenames in build_py.data_files:\n self.filelist.extend([os.path.join(src_dir, filename)\n for filename in filenames])", "response": "add python files to the list of files to be built"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nread the manifest file and add it to the list of files to include in the source distribution.", "response": "def read_manifest(self):\n \"\"\"Read the manifest file (named by 'self.manifest') and use it to\n fill in 'self.filelist', the list of files to include in the source\n distribution.\n \"\"\"\n log.info(\"reading manifest file '%s'\", self.manifest)\n manifest = open(self.manifest, 'rb')\n for line in manifest:\n # The manifest must contain UTF-8. See #303.\n if six.PY3:\n try:\n line = line.decode('UTF-8')\n except UnicodeDecodeError:\n log.warn(\"%r not UTF-8 decodable -- skipping\" % line)\n continue\n # ignore comments and blank lines\n line = line.strip()\n if line.startswith('#') or not line:\n continue\n self.filelist.append(line)\n manifest.close()"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef check_license(self):\n\n opts = self.distribution.get_option_dict('metadata')\n\n # ignore the source of the value\n _, license_file = opts.get('license_file', (None, None))\n\n if license_file is None:\n log.debug(\"'license_file' option was not specified\")\n return\n\n if not os.path.exists(license_file):\n log.warn(\"warning: Failed to find the configured license file '%s'\",\n license_file)\n return\n\n self.filelist.append(license_file)", "response": "Checks if license_file is configured and adds it to self. filelist"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get_exclusions(self):\n all_packages = (\n pkg\n for ns_pkg in self._get_SVEM_NSPs()\n for pkg in self._all_packages(ns_pkg)\n )\n\n excl_specs = product(all_packages, self._gen_exclusion_paths())\n return set(starmap(self._exclude_pkg_path, excl_specs))", "response": "Return a set of paths to be\n excluded for single_version_externally_managed installations."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _exclude_pkg_path(self, pkg, exclusion_path):\n parts = pkg.split('.') + [exclusion_path]\n return os.path.join(self.install_dir, *parts)", "response": "Given a package name and exclusion path compute the full exclusion path."} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ngets namespace packages but only for single_version_externally_managed installations and empty otherwise.", "response": "def _get_SVEM_NSPs(self):\n \"\"\"\n Get namespace packages (list) but only for\n single_version_externally_managed installations and empty otherwise.\n \"\"\"\n # TODO: is it necessary to short-circuit here? i.e. what's the cost\n # if get_finalized_command is called even when namespace_packages is\n # False?\n if not self.distribution.namespace_packages:\n return []\n\n install_cmd = self.get_finalized_command('install')\n svem = install_cmd.single_version_externally_managed\n\n return self.distribution.namespace_packages if svem else []"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ngenerate file paths to be excluded for namespace packages.", "response": "def _gen_exclusion_paths():\n \"\"\"\n Generate file paths to be excluded for namespace packages (bytecode\n cache files).\n \"\"\"\n # always exclude the package module itself\n yield '__init__.py'\n\n yield '__init__.pyc'\n yield '__init__.pyo'\n\n if not hasattr(imp, 'get_tag'):\n return\n\n base = os.path.join('__pycache__', '__init__.' + imp.get_tag())\n yield base + '.pyc'\n yield base + '.pyo'\n yield base + '.opt-1.pyc'\n yield base + '.opt-2.pyc'"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nresolves the path to the setup script from the egg base to the egg path.", "response": "def _resolve_setup_path(egg_base, install_dir, egg_path):\n \"\"\"\n Generate a path from egg_base back to '.' where the\n setup script resides and ensure that path points to the\n setup path from $install_dir/$egg_path.\n \"\"\"\n path_to_setup = egg_base.replace(os.sep, '/').rstrip('/')\n if path_to_setup != os.curdir:\n path_to_setup = '../' * (path_to_setup.count('/') + 1)\n resolved = pkg_resources.normalize_path(\n os.path.join(install_dir, egg_path, path_to_setup)\n )\n if resolved != pkg_resources.normalize_path(os.curdir):\n raise DistutilsOptionError(\n \"Can't get a consistent path to setup script from\"\n \" installation directory\", resolved,\n pkg_resources.normalize_path(os.curdir))\n return path_to_setup"} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nfinds a module in a sequence of modules.", "response": "def find_module(module, paths=None):\n \"\"\"Just like 'imp.find_module()', but with package support\"\"\"\n\n parts = module.split('.')\n\n while parts:\n part = parts.pop(0)\n f, path, (suffix, mode, kind) = info = imp.find_module(part, paths)\n\n if kind == PKG_DIRECTORY:\n parts = parts or ['__init__']\n paths = [path]\n\n elif parts:\n raise ImportError(\"Can't find %r in %s\" % (parts, module))\n\n return info"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef extract_constant(code, symbol, default=-1):\n if symbol not in code.co_names:\n # name's not there, can't possibly be an assignment\n return None\n\n name_idx = list(code.co_names).index(symbol)\n\n STORE_NAME = 90\n STORE_GLOBAL = 97\n LOAD_CONST = 100\n\n const = default\n\n for byte_code in Bytecode(code):\n op = byte_code.opcode\n arg = byte_code.arg\n\n if op == LOAD_CONST:\n const = code.co_consts[arg]\n elif arg == name_idx and (op == STORE_NAME or op == STORE_GLOBAL):\n return const\n else:\n const = default", "response": "Extracts the constant value of symbol from the code block."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\npatch the globals to remove objects not available on some platforms.", "response": "def _update_globals():\n \"\"\"\n Patch the globals to remove the objects not available on some platforms.\n\n XXX it'd be better to test assertions about bytecode instead.\n \"\"\"\n\n if not sys.platform.startswith('java') and sys.platform != 'cli':\n return\n incompatible = 'extract_constant', 'get_module_constant'\n for name in incompatible:\n del globals()[name]\n __all__.remove(name)"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef full_name(self):\n if self.requested_version is not None:\n return '%s-%s' % (self.name, self.requested_version)\n return self.name", "response": "Return full package name w / version"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nbeing version sufficiently up - to - date?", "response": "def version_ok(self, version):\n \"\"\"Is 'version' sufficiently up-to-date?\"\"\"\n return self.attribute is None or self.format is None or \\\n str(version) != \"unknown\" and version >= self.requested_version"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_version(self, paths=None, default=\"unknown\"):\n\n if self.attribute is None:\n try:\n f, p, i = find_module(self.module, paths)\n if f:\n f.close()\n return default\n except ImportError:\n return None\n\n v = get_module_constant(self.module, self.attribute, default, paths)\n\n if v is not None and v is not default and self.format is not None:\n return self.format(v)\n\n return v", "response": "Get version number of installed module or default."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef is_current(self, paths=None):\n version = self.get_version(paths)\n if version is None:\n return False\n return self.version_ok(version)", "response": "Return True if dependency is present and up - to - date on paths."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _get_all_ns_packages(self):\n pkgs = self.distribution.namespace_packages or []\n return sorted(flatten(map(self._pkg_names, pkgs)))", "response": "Return sorted list of all package namespaces"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nyields the names of the components of that namespace package.", "response": "def _pkg_names(pkg):\n \"\"\"\n Given a namespace package, yield the components of that\n package.\n\n >>> names = Installer._pkg_names('a.b.c')\n >>> set(names) == set(['a', 'a.b', 'a.b.c'])\n True\n \"\"\"\n parts = pkg.split('.')\n while parts:\n yield '.'.join(parts)\n parts.pop()"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef find_ca_bundle():\n extant_cert_paths = filter(os.path.isfile, cert_paths)\n return (\n get_win_certfile()\n or next(extant_cert_paths, None)\n or _certifi_where()\n )", "response": "Return an existing CA bundle path or None"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef rewrite_packaging(pkg_files, new_root):\n for file in pkg_files.glob('*.py'):\n text = file.text()\n text = re.sub(r' (pyparsing|six)', rf' {new_root}.\\1', text)\n file.write_text(text)", "response": "Rewrite all. py files in packaging to a new root."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef coarse_tag_str(pos_seq):\n\tglobal tag2coarse\n\ttags = [tag2coarse.get(tag, 'O') for tag in pos_seq]\n\treturn ''.join(tags)", "response": "Convert POS sequence to our coarse system formatted as a string."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef extract_finditer(pos_seq, regex=SimpleNP):\n\tss = coarse_tag_str(pos_seq)\n\n\tdef gen():\n\t\tfor m in re.finditer(regex, ss):\n\t\t\tyield (m.start(), m.end())\n\n\treturn list(gen())", "response": "The GreedyFSA method in Handler et al. 2016.\nReturns token position spans of valid ngrams."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_phrases(text=None, tokens=None, postags=None, tagger='nltk', grammar='SimpleNP', regex=None, minlen=2, maxlen=8,\n output='counts'):\n\t\"\"\"Give a text (or POS tag sequence), return the phrases matching the given\n\tgrammar. Works on documents or sentences.\n\tReturns a dict with one or more keys with the phrase information.\n\n\ttext: the text of the document. If supplied, we will try to POS tag it.\n\n\tYou can also do your own tokenzation and/or tagging and supply them as\n\t'tokens' and/or 'postags', which are lists of strings (of the same length).\n\t - Must supply both to get phrase counts back.\n\t - With only postags, can get phrase token spans back.\n\t - With only tokens, we will try to POS-tag them if possible.\n\n\toutput: a string, or list of strings, of information to return. Options include:\n\t - counts: a Counter with phrase frequencies. (default)\n\t - token_spans: a list of the token spans of each matched phrase. This is\n\t\t a list of (start,end) pairs of integers, which refer to token positions.\n\t - pos, tokens can be returned too.\n\n\ttagger: if you're passing in raw text, can supply your own tagger, from one\n\tof the get_*_tagger() functions. If this is not supplied, we will try to load one.\n\n\tgrammar: the grammar to use. Only one option right now...\n\n\tregex: a custom regex to use, instead of a premade grammar. Currently,\n\tthis must work on the 5-tag system described near the top of this file.\n\n\t\"\"\"\n\tglobal SimpleNP\n\n\t## try to get values for both 'postags' and 'tokens', parallel lists of strings\n\tif postags is None:\n\t\ttry:\n\t\t\ttagger = TAGGER_NAMES[tagger]()\n\t\texcept:\n\t\t\traise Exception(\"We don't support tagger %s\" % tagger)\n\t\t# otherwise, assume it's one of our wrapper *Tagger objects\n\n\t\td = None\n\t\tif tokens is not None:\n\t\t\td = tagger.tag_tokens(tokens)\n\t\telif text is not None:\n\t\t\td = tagger.tag_text(text)\n\t\telse:\n\t\t\traise Exception(\"Need to supply text or tokens.\")\n\t\tpostags = d['pos']\n\t\ttokens = d['tokens']\n\n\tif regex is None:\n\t\tif grammar == 'SimpleNP':\n\t\t\tregex = SimpleNP\n\t\telse:\n\t\t\tassert False, \"Don't know grammar %s\" % grammar\n\n\tphrase_tokspans = extract_ngram_filter(postags, minlen=minlen, maxlen=maxlen)\n\n\t## Handle multiple possible return info outputs\n\n\tif isinstance(output, str):\n\t\toutput = [output]\n\n\tour_options = set()\n\n\tdef retopt(x):\n\t\tour_options.add(x)\n\t\treturn x in output\n\n\tret = {}\n\tret['num_tokens'] = len(postags)\n\tif retopt('token_spans'):\n\t\tret['token_spans'] = phrase_tokspans\n\tif retopt('counts'):\n\t\tcounts = Counter()\n\t\tfor (start, end) in phrase_tokspans:\n\t\t\tphrase = safejoin([tokens[i] for i in xrange(start, end)])\n\t\t\tphrase = phrase.lower()\n\t\t\tcounts[phrase] += 1\n\t\tret['counts'] = counts\n\tif retopt('pos'):\n\t\tret['pos'] = postags\n\tif retopt('tokens'):\n\t\tret['tokens'] = tokens\n\n\txx = set(output) - our_options\n\tif xx:\n\t\traise Exception(\"Don't know how to handle output options: %s\" % list(xx))\n\treturn ret", "response": "Given a text or POS tag sequence return the phrases matching the given grammar."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef tag_text(self, text):\n\t\t'''take input text and return tokens w/ part of speech tags using NLTK'''\n\t\t# putting import here instead of top of file b.c. not all will have nltk installed\n\n\t\tsents = self.sent_detector.tokenize(text) # TODO: this will fail on some unicode chars. I think assumes ascii\n\t\tword_pos_pairs = []\n\n\t\tall_tokens = []\n\t\tfor sent in sents:\n\t\t\ttokens = self.tokenize(sent)\n\t\t\tall_tokens = all_tokens + tokens\n\t\t\tword_pos_pairs = word_pos_pairs + self.tagger.tag(tokens)\n\t\treturn {'tokens': all_tokens, 'pos': [tag for (w, tag) in word_pos_pairs]}", "response": "take input text and return tokens w / part of speech tags using NLTK"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef build(self):\n\t\t'''Constructs the term doc matrix.\n\n\t\tReturns\n\t\t-------\n\t\tscattertext.ParsedCorpus.ParsedCorpus\n\t\t'''\n\t\tself._y = self._get_y_and_populate_category_idx_store()\n\t\tself._df.apply(self._add_to_x_factory, axis=1)\n\t\tself._X = self._X_factory.set_last_row_idx(len(self._y) - 1).get_csr_matrix()\n\t\tself._mX = self._mX_factory.set_last_row_idx(len(self._y) - 1).get_csr_matrix()\n\t\tif self._parsed_col is not None and self._parsed_col in self._df:\n\t\t\treturn ParsedCorpus(self._df,\n\t\t\t self._X,\n\t\t\t self._mX,\n\t\t\t self._y,\n\t\t\t self._term_idx_store,\n\t\t\t self._category_idx_store,\n\t\t\t self._metadata_idx_store,\n\t\t\t self._parsed_col,\n\t\t\t self._category_col)\n\t\telse:\n\t\t\treturn CorpusDF(self._df,\n\t\t\t self._X,\n\t\t\t self._mX,\n\t\t\t self._y,\n\t\t\t self._text_col,\n\t\t\t self._term_idx_store,\n\t\t\t self._category_idx_store,\n\t\t\t self._metadata_idx_store)", "response": "Constructs the term doc matrix."} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns a sparse matrix that contains the terms that are in the same domain as the term_doc_matrix.", "response": "def get_new_term_doc_mat(self, doc_domains):\n\t\t'''\n\t\tCombines documents together that are in the same domain\n\n\t\tParameters\n\t\t----------\n\t\tdoc_domains : array-like\n\n\t\tReturns\n\t\t-------\n\t\tscipy.sparse.csr_matrix\n\n\t\t'''\n\t\tassert len(doc_domains) == self.term_doc_matrix.get_num_docs()\n\t\tdoc_domain_set = set(doc_domains)\n\t\tnum_terms = self.term_doc_matrix.get_num_terms()\n\t\tnum_domains = len(doc_domain_set)\n\t\tdomain_mat = lil_matrix((num_domains, num_terms), dtype=int)\n\t\tX = self.term_doc_matrix.get_term_doc_mat()\n\t\tfor i, domain in enumerate(doc_domain_set):\n\t\t\tdomain_mat[i, :] = X[np.array(doc_domains == domain)].sum(axis=0)\n\t\treturn domain_mat.tocsr()"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef build(self):\n\t\t'''Constructs the term doc matrix.\n\n\t\tReturns\n\t\t-------\n\t\tscattertext.ParsedCorpus.ParsedCorpus\n\t\t'''\n\t\tself._y = self._get_y_and_populate_category_idx_store()\n\t\tself._df.apply(self._add_to_x_factory, axis=1)\n\t\tself._X = self._X_factory.set_last_row_idx(len(self._y)-1).get_csr_matrix()\n\t\tself._mX = self._mX_factory.set_last_row_idx(len(self._y)-1).get_csr_matrix()\n\t\treturn ParsedCorpus(self._df,\n\t\t self._X,\n\t\t self._mX,\n\t\t self._y,\n\t\t self._term_idx_store,\n\t\t self._category_idx_store,\n\t\t self._metadata_idx_store,\n\t\t self._parsed_col,\n\t\t self._category_col)", "response": "Constructs the term doc matrix."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning a dataframe consisting of unigram term counts of words occurring in the TermDocumentMatrix and their corresponding background corpusCounts.", "response": "def get_term_and_background_counts(self):\n\t\t'''\n\t\tReturns\n\t\t-------\n\t\tA pd.DataFrame consisting of unigram term counts of words occurring\n\t\t in the TermDocumentMatrix and their corresponding background corpus\n\t\t counts. The dataframe has two columns, corpus and background.\n\n\t\t>>> corpus.get_unigram_corpus.get_term_and_background_counts()\n\t\t corpus background\n\t\tobama 702.0 565739.0\n\t\tromney 570.0 695398.0\n\t\tbarack 248.0 227861.0\n\t\t...\n\t\t'''\n\t\tbackground_df = self._get_background_unigram_frequencies()\n\t\tcorpus_freq_df = pd.DataFrame({'corpus': self.term_category_freq_df.sum(axis=1)})\n\t\tcorpus_unigram_freq = corpus_freq_df.loc[[w for w in corpus_freq_df.index if ' ' not in w]]\n\t\tdf = corpus_unigram_freq.join(background_df, how='outer').fillna(0)\n\t\tdel df.index.name\n\t\treturn df"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn a pandas. ndarray containing the document indices in the categories that are in the categories.", "response": "def get_doc_indices(self):\n\t\t'''\n\t\tReturns\n\t\t-------\n\t\tnp.array\n\n\t\tInteger document indices\n\t\t'''\n\t\tif self._document_category_df is None:\n\t\t\treturn pd.np.array([])\n\t\tcategories_d = {d: i for i, d in enumerate(self.get_categories())}\n\t\treturn self._document_category_df.category.apply(categories_d.get).values"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn ------- np.array Texts", "response": "def get_texts(self):\n\t\t'''\n\t\tReturns\n\t\t-------\n\t\tnp.array\n\n\t\tTexts\n\t\t'''\n\t\tif self._document_category_df is None:\n\t\t\treturn pd.np.array([])\n\t\treturn self._document_category_df.text.values"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef get_term_category_frequencies(self, scatterchartdata):\n\t\t'''\n\t\tParameters\n\t\t----------\n\t\tscatterchartdata : ScatterChartData\n\n\t\tReturns\n\t\t-------\n\t\tpd.DataFrame\n\t\t'''\n\n\t\tdf = self.term_category_freq_df.rename(\n\t\t\tcolumns={c: c + ' freq' for c in self.term_category_freq_df}\n\t\t)\n\t\tdf.index.name = 'term'\n\t\treturn df", "response": "Returns a DataFrame with the term category frequencies."} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nfilters the term doc matrix by term frequency", "response": "def filter(self, term_doc_matrix):\n\t\t'''\n\t\tParameters\n\t\t----------\n\t\tterm_doc_matrix : TermDocMatrix\n\n\t\tReturns\n\t\t-------\n\t\tTermDocMatrix pmi-filterd term doc matrix\n\t\t'''\n\t\tdf = term_doc_matrix.get_term_freq_df()\n\t\tif len(df) == 0:\n\t\t\treturn term_doc_matrix\n\t\tlow_pmi_bigrams = get_low_pmi_bigrams(self._threshold_coef, df).index\n\t\tinfrequent_terms = df[df.sum(axis=1) < self._min_freq].index\n\t\tfiltered_term_doc_mat = term_doc_matrix.remove_terms(set(low_pmi_bigrams | infrequent_terms))\n\t\ttry:\n\t\t\tfiltered_term_doc_mat.get_term_freq_df()\n\t\texcept ValueError:\n\t\t\traise AtLeastOneCategoryHasNoTermsException()\n\t\treturn filtered_term_doc_mat"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef produce_scattertext_explorer(corpus,\n category,\n category_name=None,\n not_category_name=None,\n protocol='https',\n pmi_threshold_coefficient=DEFAULT_MINIMUM_TERM_FREQUENCY,\n minimum_term_frequency=DEFAULT_PMI_THRESHOLD_COEFFICIENT,\n minimum_not_category_term_frequency=0,\n max_terms=None,\n filter_unigrams=False,\n height_in_pixels=None,\n width_in_pixels=None,\n max_snippets=None,\n max_docs_per_category=None,\n metadata=None,\n scores=None,\n x_coords=None,\n y_coords=None,\n original_x=None,\n original_y=None,\n rescale_x=None,\n rescale_y=None,\n singleScoreMode=False,\n sort_by_dist=False,\n reverse_sort_scores_for_not_category=True,\n use_full_doc=False,\n transform=percentile_alphabetical,\n jitter=0,\n gray_zero_scores=False,\n term_ranker=None,\n asian_mode=False,\n use_non_text_features=False,\n show_top_terms=True,\n show_characteristic=True,\n word_vec_use_p_vals=False,\n max_p_val=0.1,\n p_value_colors=False,\n term_significance=None,\n save_svg_button=False,\n x_label=None,\n y_label=None,\n d3_url=None,\n d3_scale_chromatic_url=None,\n pmi_filter_thresold=None,\n alternative_text_field=None,\n terms_to_include=None,\n semiotic_square=None,\n num_terms_semiotic_square=None,\n not_categories=None,\n neutral_categories=[],\n extra_categories=[],\n show_neutral=False,\n neutral_category_name=None,\n get_tooltip_content=None,\n x_axis_values=None,\n y_axis_values=None,\n color_func=None,\n term_scorer=None,\n show_axes=True,\n horizontal_line_y_position=None,\n vertical_line_x_position=None,\n show_cross_axes=True,\n show_extra=False,\n extra_category_name=None,\n censor_points=True,\n center_label_over_points=False,\n x_axis_labels=None,\n y_axis_labels=None,\n topic_model_term_lists=None,\n topic_model_preview_size=10,\n metadata_descriptions=None,\n vertical_lines=None,\n characteristic_scorer=None,\n term_colors=None,\n unified_context=False,\n show_category_headings=True,\n include_term_category_counts=False,\n div_name=None,\n alternative_term_func=None,\n term_metadata=None,\n return_data=False):\n '''Returns html code of visualization.\n\n Parameters\n ----------\n corpus : Corpus\n Corpus to use.\n category : str\n Name of category column as it appears in original data frame.\n category_name : str\n Name of category to use. E.g., \"5-star reviews.\"\n Optional, defaults to category name.\n not_category_name : str\n Name of everything that isn't in category. E.g., \"Below 5-star reviews\".\n Optional defaults to \"N(n)ot \" + category_name, with the case of the 'n' dependent\n on the case of the first letter in category_name.\n protocol : str, optional\n Protocol to use. Either http or https. Default is https.\n pmi_threshold_coefficient : int, optional\n Filter out bigrams with a PMI of < 2 * pmi_threshold_coefficient. Default is 6\n minimum_term_frequency : int, optional\n Minimum number of times word needs to appear to make it into visualization.\n minimum_not_category_term_frequency : int, optional\n If an n-gram does not occur in the category, minimum times it\n must been seen to be included. Default is 0.\n max_terms : int, optional\n Maximum number of terms to include in visualization.\n filter_unigrams : bool, optional\n Default False, do we filter out unigrams that only occur in one bigram\n width_in_pixels : int, optional\n Width of viz in pixels, if None, default to JS's choice\n height_in_pixels : int, optional\n Height of viz in pixels, if None, default to JS's choice\n max_snippets : int, optional\n Maximum number of snippets to show when term is clicked. If None, all are shown.\n max_docs_per_category: int, optional\n Maximum number of documents to store per category. If None, by default, all are stored.\n metadata : list, optional\n list of meta data strings that will be included for each document\n scores : np.array, optional\n Array of term scores or None.\n x_coords : np.array, optional\n Array of term x-axis positions or None. Must be in [0,1].\n If present, y_coords must also be present.\n y_coords : np.array, optional\n Array of term y-axis positions or None. Must be in [0,1].\n If present, x_coords must also be present.\n original_x : array-like\n Original, unscaled x-values. Defaults to x_coords\n original_y : array-like\n Original, unscaled y-values. Defaults to y_coords\n rescale_x : lambda list[0,1]: list[0,1], optional\n Array of term x-axis positions or None. Must be in [0,1].\n Rescales x-axis after filtering\n rescale_y : lambda list[0,1]: list[0,1], optional\n Array of term y-axis positions or None. Must be in [0,1].\n Rescales y-axis after filtering\n singleScoreMode : bool, optional\n Label terms based on score vs distance from corner. Good for topic scores. Show only one color.\n sort_by_dist: bool, optional\n Label terms based distance from corner. True by default. Negated by singleScoreMode.\n reverse_sort_scores_for_not_category: bool, optional\n If using a custom score, score the not-category class by\n lowest-score-as-most-predictive. Turn this off for word vector\n or topic similarity. Default True.\n use_full_doc : bool, optional\n Use the full document in snippets. False by default.\n transform : function, optional\n not recommended for editing. change the way terms are ranked. default is st.Scalers.percentile_ordinal\n jitter : float, optional\n percentage of axis to jitter each point. default is 0.\n gray_zero_scores : bool, optional\n If True, color points with zero-scores a light shade of grey. False by default.\n term_ranker : TermRanker, optional\n TermRanker class for determining term frequency ranks.\n asian_mode : bool, optional\n Use a special Javascript regular expression that's specific to chinese or japanese\n use_non_text_features : bool, optional\n Show non-bag-of-words features (e.g., Empath) instead of text. False by default.\n show_top_terms : bool, default True\n Show top terms on the left-hand side of the visualization\n show_characteristic: bool, default True\n Show characteristic terms on the far left-hand side of the visualization\n word_vec_use_p_vals: bool, default False\n Sort by harmonic mean of score and distance.\n max_p_val : float, default 0.1\n If word_vec_use_p_vals, the minimum p val to use.\n p_value_colors : bool, default False\n Color points differently if p val is above 1-max_p_val, below max_p_val, or\n in between.\n term_significance : TermSignificance instance or None\n Way of getting signfiance scores. If None, p values will not be added.\n save_svg_button : bool, default False\n Add a save as SVG button to the page.\n x_label : str, default None\n Custom x-axis label\n y_label : str, default None\n Custom y-axis label\n d3_url, str, None by default. The url (or path) of d3.\n URL of d3, to be inserted into '\n extra_libs = ''\n html_content = (html_content\n .replace('', extra_libs, 1)\n .replace('http://', protocol + '://'))\n return html_content", "response": "Returns the HTML representation of this object."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef search(self, ngram):\n\t\t'''\n\t\tParameters\n\t\t----------\n\t\tngram str or unicode, string to search for\n\n\t\tReturns\n\t\t-------\n\t\tpd.DataFrame, {'texts': , 'categories': }\n\n\t\t'''\n\t\tmask = self._document_index_mask(ngram)\n\t\treturn pd.DataFrame({\n\t\t\t'text': self.get_texts()[mask],\n\t\t\t'category': [self._category_idx_store.getval(idx)\n\t\t\t for idx in self._y[mask]]\n\t\t})", "response": "Search for a given ngram in the index store."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns a pd. DataFrame indexed on terms with columns giving frequencies for each term.", "response": "def get_term_freq_df(self, label_append=' freq'):\n '''\n Parameters\n -------\n label_append : str\n\n Returns\n -------\n pd.DataFrame indexed on terms, with columns giving frequencies for each\n '''\n\n '''\n row = self._row_category_ids()\n newX = csr_matrix((self._X.data, (row, self._X.indices)))\n return self._term_freq_df_from_matrix(newX)\n '''\n mat = self.get_term_freq_mat()\n return pd.DataFrame(mat,\n index=pd.Series(self.get_terms(), name='term'),\n columns=[c + label_append for c in self.get_categories()])"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_term_freq_mat(self):\n '''\n Returns\n -------\n np.array with columns as categories and rows as terms\n '''\n freq_mat = np.zeros(shape=(self.get_num_terms(), self.get_num_categories()), dtype=int)\n for cat_i in range(self.get_num_categories()):\n freq_mat[:, cat_i] = self._X[self._y == cat_i, :].sum(axis=0)\n return freq_mat", "response": "Returns ------- np. array with columns as categories and rows as terms\n "} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning ------- np. array with columns as categories and rows as terms", "response": "def get_term_count_mat(self):\n '''\n Returns\n -------\n np.array with columns as categories and rows as terms\n '''\n freq_mat = np.zeros(shape=(self.get_num_terms(), self.get_num_categories()), dtype=int)\n for cat_i in range(self.get_num_categories()):\n X = (self._X[self._y == cat_i, :] > 0).astype(int)\n freq_mat[:, cat_i] = X.sum(axis=0)\n return freq_mat"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get_metadata_count_mat(self):\n '''\n Returns\n -------\n np.array with columns as categories and rows as terms\n '''\n freq_mat = np.zeros(shape=(self.get_num_metadata(), self.get_num_categories()), dtype=int)\n for cat_i in range(self.get_num_categories()):\n mX = (self._mX[self._y == cat_i, :] > 0).astype(int)\n freq_mat[:, cat_i] = mX.sum(axis=0)\n return freq_mat", "response": "Returns ------- np. array with columns as categories and rows as terms\n "} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn a pd. DataFrame indexed on metadata with columns giving frequencies for each category.", "response": "def get_metadata_freq_df(self, label_append=' freq'):\n '''\n Parameters\n -------\n label_append : str\n\n Returns\n -------\n pd.DataFrame indexed on metadata, with columns giving frequencies for each category\n '''\n row = self._row_category_ids_for_meta()\n newX = csr_matrix((self._mX.data, (row, self._mX.indices)))\n return self._metadata_freq_df_from_matrix(newX, label_append)"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef keep_only_these_categories(self, categories, ignore_absences=False):\n '''\n Non destructive category removal.\n\n Parameters\n ----------\n categories : list\n list of categories to keep\n ignore_absences : bool, False by default\n if categories does not appear, don't raise an error, just move on.\n\n Returns\n -------\n TermDocMatrix, new object with categories removed.\n '''\n if not ignore_absences:\n assert set(self.get_categories()) & set(categories) == set(categories)\n categories_to_remove = [c for c in self.get_categories() if c not in categories]\n return self.remove_categories(categories_to_remove)", "response": "Removes only the specified categories from the TermDocMatrix."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef remove_categories(self, categories, ignore_absences=False):\n '''\n Non destructive category removal.\n\n Parameters\n ----------\n categories : list\n list of categories to remove\n ignore_absences : bool, False by default\n if categories does not appear, don't raise an error, just move on.\n\n Returns\n -------\n TermDocMatrix, new object with categories removed.\n '''\n idx_to_delete_list = []\n existing_categories = set(self.get_categories())\n for category in categories:\n if category not in existing_categories:\n if not ignore_absences:\n raise KeyError('Category %s not found' % (category))\n continue\n idx_to_delete_list.append(self._category_idx_store.getidx(category))\n new_category_idx_store = self._category_idx_store.batch_delete_idx(idx_to_delete_list)\n\n columns_to_delete = np.nonzero(np.isin(self._y, idx_to_delete_list))\n new_X = delete_columns(self._X.T, columns_to_delete).T\n new_mX = delete_columns(self._mX.T, columns_to_delete).T\n intermediate_y = self._y[~np.isin(self._y, idx_to_delete_list)]\n old_y_to_new_y = [self._category_idx_store.getidx(x)\n for x in new_category_idx_store._i2val]\n new_y = np.array([old_y_to_new_y.index(i) if i in old_y_to_new_y else None\n for i in range(intermediate_y.max() + 1)])[intermediate_y]\n\n new_metadata_idx_store = self._metadata_idx_store\n\n if self.metadata_in_use():\n meta_idx_to_delete = np.nonzero(new_mX.sum(axis=0).A1 == 0)[0]\n new_metadata_idx_store = self._metadata_idx_store.batch_delete_idx(meta_idx_to_delete)\n\n term_idx_to_delete = np.nonzero(new_X.sum(axis=0).A1 == 0)[0]\n new_term_idx_store = self._term_idx_store.batch_delete_idx(term_idx_to_delete)\n new_X = delete_columns(new_X, term_idx_to_delete)\n\n term_doc_mat_to_ret = self._make_new_term_doc_matrix(new_X, new_mX, new_y, new_term_idx_store,\n new_category_idx_store, new_metadata_idx_store,\n ~np.isin(self._y, idx_to_delete_list))\n return term_doc_mat_to_ret", "response": "Remove the specified categories from the term doc matrix."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nremoves terms from the term doc matrix by indices.", "response": "def remove_terms_by_indices(self, idx_to_delete_list):\n '''\n Parameters\n ----------\n idx_to_delete_list, list\n\n Returns\n -------\n TermDocMatrix\n '''\n new_X, new_term_idx_store = self._get_X_after_delete_terms(idx_to_delete_list)\n return self._make_new_term_doc_matrix(new_X, self._mX, self._y, new_term_idx_store, self._category_idx_store,\n self._metadata_idx_store, self._y == self._y)"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get_rudder_scores(self, category):\n ''' Computes Rudder score.\n Parameters\n ----------\n category : str\n category name to score\n\n Returns\n -------\n np.array\n '''\n category_percentiles = self._get_term_percentiles_in_category(category)\n not_category_percentiles = self._get_term_percentiles_not_in_category(category)\n rudder_scores = self._get_rudder_scores_for_percentile_pair(category_percentiles,\n not_category_percentiles)\n return rudder_scores", "response": "Computes the rudder scores for the specified category."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ncomputing the l2 - penalized logistic regression coefficient for a given category.", "response": "def get_logistic_regression_coefs_l2(self, category,\n clf=RidgeClassifierCV()):\n ''' Computes l2-penalized logistic regression score.\n Parameters\n ----------\n category : str\n category name to score\n\n category : str\n category name to score\n Returns\n -------\n (coefficient array, accuracy, majority class baseline accuracy)\n '''\n try:\n from sklearn.cross_validation import cross_val_predict\n except:\n from sklearn.model_selection import cross_val_predict\n y = self._get_mask_from_category(category)\n X = TfidfTransformer().fit_transform(self._X)\n clf.fit(X, y)\n y_hat = cross_val_predict(clf, X, y)\n acc, baseline = self._get_accuracy_and_baseline_accuracy(y, y_hat)\n return clf.coef_[0], acc, baseline"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncompute the logistic regression coefficient for a given category.", "response": "def get_logistic_regression_coefs_l1(self, category,\n clf=LassoCV(alphas=[0.1, 0.001],\n max_iter=10000,\n n_jobs=-1)):\n ''' Computes l1-penalized logistic regression score.\n Parameters\n ----------\n category : str\n category name to score\n\n Returns\n -------\n (coefficient array, accuracy, majority class baseline accuracy)\n '''\n try:\n from sklearn.cross_validation import cross_val_predict\n except:\n from sklearn.model_selection import cross_val_predict\n y = self._get_mask_from_category(category)\n y_continuous = self._get_continuous_version_boolean_y(y)\n # X = TfidfTransformer().fit_transform(self._X)\n X = self._X\n\n clf.fit(X, y_continuous)\n y_hat = (cross_val_predict(clf, X, y_continuous) > 0)\n acc, baseline = self._get_accuracy_and_baseline_accuracy(y, y_hat)\n clf.fit(X, y_continuous)\n return clf.coef_, acc, baseline"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncompute the regression coefficient of the tdfidf transformed features for the specified category.", "response": "def get_regression_coefs(self, category, clf=ElasticNet()):\n ''' Computes regression score of tdfidf transformed features\n Parameters\n ----------\n category : str\n category name to score\n clf : sklearn regressor\n\n Returns\n -------\n coefficient array\n '''\n self._fit_tfidf_model(category, clf)\n return clf.coef_"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_logreg_coefs(self, category, clf=LogisticRegression()):\n ''' Computes regression score of tdfidf transformed features\n Parameters\n ----------\n category : str\n category name to score\n clf : sklearn regressor\n\n Returns\n -------\n coefficient array\n '''\n self._fit_tfidf_model(category, clf)\n return clf.coef_[0]", "response": "Computes the logistic regression coefficient of the tdfidf transformed features."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_scaled_f_scores(self,\n category,\n scaler_algo=DEFAULT_SCALER_ALGO,\n beta=DEFAULT_BETA):\n ''' Computes scaled-fscores\n Parameters\n ----------\n category : str\n category name to score\n scaler_algo : str\n Function that scales an array to a range \\in [0 and 1]. Use 'percentile', 'normcdf'. Default.\n beta : float\n Beta in (1+B^2) * (Scale(P(w|c)) * Scale(P(c|w)))/(B^2*Scale(P(w|c)) + Scale(P(c|w))). Default.\n Returns\n -------\n np.array of harmonic means of scaled P(word|category) and scaled P(category|word)\n '''\n\n assert beta > 0\n cat_word_counts, not_cat_word_counts = self._get_catetgory_and_non_category_word_counts(category)\n scores = self._get_scaled_f_score_from_counts(cat_word_counts, not_cat_word_counts, scaler_algo, beta)\n return np.array(scores)", "response": "Returns a numpy array of scaled F scores for the given category and non - category."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a pd. DataFrame of fisher scores vs background.", "response": "def get_fisher_scores_vs_background(self):\n '''\n Returns\n -------\n pd.DataFrame of fisher scores vs background\n '''\n df = self.get_term_and_background_counts()\n odds_ratio, p_values = self._get_fisher_scores_from_counts(\n df['corpus'], df['background'])\n df['Odds ratio'] = odds_ratio\n df['Bonferroni-corrected p-values'] = p_values * len(df)\n df.sort_values(by=['Bonferroni-corrected p-values', 'Odds ratio'],\n ascending=[True, False])\n return df"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_posterior_mean_ratio_scores_vs_background(self):\n '''\n Returns\n -------\n pd.DataFrame of posterior mean scores vs background\n '''\n df = self.get_term_and_background_counts()\n df['Log Posterior Mean Ratio'] = self._get_posterior_mean_ratio_from_counts(df['corpus'],\n df['background'])\n return df.sort_values('Log Posterior Mean Ratio', ascending=False)", "response": "Returns a pd. DataFrame of posterior mean ratio scores vs background."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get_rudder_scores_vs_background(self):\n '''\n Returns\n -------\n pd.DataFrame of rudder scores vs background\n '''\n df = self.get_term_and_background_counts()\n corpus_percentiles = self._get_percentiles_from_freqs(df['corpus'])\n background_percentiles = self._get_percentiles_from_freqs(df['background'])\n df['Rudder'] = (self._get_rudder_scores_for_percentile_pair(corpus_percentiles,\n background_percentiles))\n df = df.sort_values(by='Rudder', ascending=True)\n return df", "response": "Returns a pd. DataFrame of rudder scores vs background."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_term_category_frequencies(self, scatterchartdata):\n '''\n Applies the ranker in scatterchartdata to term-category frequencies.\n\n Parameters\n ----------\n scatterchartdata : ScatterChartData\n\n Returns\n -------\n pd.DataFrame\n '''\n term_ranker = scatterchartdata.term_ranker(self)\n if scatterchartdata.use_non_text_features:\n term_ranker.use_non_text_features()\n return term_ranker.get_ranks()", "response": "Applies the ranker in scatterchartdata to term - category frequencies. Returns a Pandas DataFrame containing the term - category frequencies."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a TermDocMatrix which is identical to self except the metadata values are now identical to the categories present.", "response": "def use_categories_as_metadata(self):\n '''\n Returns a TermDocMatrix which is identical to self except the metadata values are now identical to the\n categories present.\n\n :return: TermDocMatrix\n '''\n new_metadata_factory = CSRMatrixFactory()\n for i, category_idx in enumerate(self.get_category_ids()):\n new_metadata_factory[i, category_idx] = 1\n new_metadata = new_metadata_factory.get_csr_matrix()\n new_tdm = self._make_new_term_doc_matrix(self._X,\n new_metadata,\n self._y,\n self._term_idx_store,\n self._category_idx_store,\n copy(self._category_idx_store),\n self._y == self._y)\n return new_tdm"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef use_categories_as_metadata_and_replace_terms(self):\n '''\n Returns a TermDocMatrix which is identical to self except the metadata values are now identical to the\n categories present and term-doc-matrix is now the metadata matrix.\n\n :return: TermDocMatrix\n '''\n new_metadata_factory = CSRMatrixFactory()\n for i, category_idx in enumerate(self.get_category_ids()):\n new_metadata_factory[i, category_idx] = 1\n new_metadata = new_metadata_factory.get_csr_matrix()\n new_tdm = self._make_new_term_doc_matrix(self._mX,\n new_metadata,\n self._y,\n self._metadata_idx_store,\n self._category_idx_store,\n copy(self._category_idx_store),\n self._y == self._y)\n return new_tdm", "response": "Returns a TermDocMatrix which is identical to self except the metadata values are now identical to the categories present."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_data():\n\t\t'''\n\t\tReturns\n\t\t-------\n\t\tpd.DataFrame\n\n\t\tI.e.,\n\t\t>>> convention_df.iloc[0]\n\t\tcategory plot\n\t\tfilename subjectivity_html/obj/2002/Abandon.html\n\t\ttext A senior at an elite college (Katie Holmes), a...\n\t\tmovie_name abandon\n\t\t'''\n\t\ttry:\n\t\t\tdata_stream = pkgutil.get_data('scattertext', 'data/rotten_tomatoes_corpus.csv.bz2')\n\t\texcept:\n\t\t\turl = ROTTEN_TOMATOES_DATA_URL\n\t\t\tdata_stream = urlopen(url).read()\n\t\treturn pd.read_csv(io.BytesIO(bz2.decompress(data_stream)))", "response": "Get data from ROTTEN Tomatoes corpus."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef use_general_term_frequencies(self):\n\t\t'''\n\t\tReturns\n\t\t-------\n\t\tPriorFactory\n\t\t'''\n\t\ttdf = self._get_relevant_term_freq()\n\t\tbg_df = self.term_doc_mat.get_term_and_background_counts()[['background']]\n\t\tbg_df = pd.merge(tdf,\n\t\t bg_df,\n\t\t left_index=True,\n\t\t right_index=True,\n\t\t how='left').fillna(0.)\n\t\tself._store_priors_from_background_dataframe(bg_df)\n\t\treturn self", "response": "This method is used to set the term frequencies for the current term and background."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nadd custom term frequencies to the internal list of priors.", "response": "def use_custom_term_frequencies(self, custom_term_frequencies):\n\t\t'''\n\t\tParameters\n\t\t----------\n\t\tpd.Series\n\t\tterm -> frequency\n\t\tReturns\n\t\t-------\n\t\tPriorFactory\n\t\t'''\n\t\tself.priors += custom_term_frequencies.reindex(self.priors.index).fillna(0)\n\t\treturn self"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nusing all categories in the term ranker.", "response": "def use_all_categories(self):\n\t\t'''\n\t\tReturns\n\t\t-------\n\t\tPriorFactory\n\t\t'''\n\t\tterm_df = self.term_ranker.get_ranks()\n\t\tself.priors += term_df.sum(axis=1).fillna(0.)\n\t\treturn self"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef use_neutral_categories(self):\n\t\t'''\n\t\tReturns\n\t\t-------\n\t\tPriorFactory\n\t\t'''\n\t\tterm_df = self.term_ranker.get_ranks()\n\t\tself.priors += term_df[[c + ' freq' for c in self._get_neutral_categories()]].sum(axis=1)\n\t\treturn self", "response": "Adds the neutral categories to the priority table."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef drop_neutral_categories_from_corpus(self):\n\t\t'''\n\t\tReturns\n\t\t-------\n\t\tPriorFactory\n\t\t'''\n\t\tneutral_categories = self._get_neutral_categories()\n\t\tself.term_doc_mat = self.term_doc_mat.remove_categories(neutral_categories)\n\t\tself._reindex_priors()\n\t\treturn self", "response": "Drop neutral categories from the corpus."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ndrop unused terms from the term doc mat.", "response": "def drop_unused_terms(self):\n\t\t'''\n\t\tReturns\n\t\t-------\n\t\tPriorFactory\n\t\t'''\n\t\tself.term_doc_mat = self.term_doc_mat.remove_terms(\n\t\t\tset(self.term_doc_mat.get_terms()) - set(self.priors.index)\n\t\t)\n\t\tself._reindex_priors()\n\t\treturn self"} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ndrops zero priors from the term doc mat.", "response": "def drop_zero_priors(self):\n\t\t'''\n\t\tReturns\n\t\t-------\n\t\tPriorFactory\n\t\t'''\n\t\tself.term_doc_mat = self.term_doc_mat.remove_terms(\n\t\t\tself.priors[self.priors == 0].index\n\t\t)\n\t\tself._reindex_priors()\n\t\treturn self"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef align_to_target(self, target_term_doc_mat):\n\t\t'''\n\t\tParameters\n\t\t----------\n\t\ttarget_term_doc_mat : TermDocMatrix\n\n\t\tReturns\n\t\t-------\n\t\tPriorFactory\n\t\t'''\n\t\tself.priors = self.priors[target_term_doc_mat.get_terms()].fillna(0)\n\t\treturn self", "response": "Aligns the priority matrix to the target term docmat."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_priors(self):\n\t\t'''\n\t\tReturns\n\t\t-------\n\t\tpd.Series\n\t\t'''\n\t\tpriors = self.priors\n\t\tpriors[~np.isfinite(priors)] = 0\n\t\tpriors += self.starting_count\n\t\treturn priors", "response": "Returns the number of priors in the series."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_zeta_i_j_given_separate_counts(self, y_i, y_j):\n\t\t'''\n\t\tParameters\n\t\t----------\n\t\ty_i, np.array(int)\n\t\t\tArrays of word counts of words occurring in positive class\n\t\ty_j, np.array(int)\n\n\t\tReturns\n\t\t-------\n\t\tnp.array of z-scores\n\t\t'''\n\t\typ_i = y_i + self.alpha_w\n\t\typ_j = y_j + self.alpha_w\n\t\tnp_i = np.sum(yp_i)\n\t\tnp_j = np.sum(yp_j)\n\n\t\tdelta_i_j = np.log(yp_i / (np_i - yp_i)) - np.log(yp_j / (np_j - yp_j))\n\t\tvar_delta_i_j = (1. / (yp_i)\n\t\t + 1. / (np_i - yp_i)\n\t\t + 1. / (yp_j)\n\t\t + 1. / (np_j - yp_j))\n\t\tzeta_i_j = delta_i_j / np.sqrt(var_delta_i_j)\n\t\treturn zeta_i_j", "response": "This function calculates the zeta_i_j and zeta_i_j given the separate counts of words occurring in positive class y_i and y_j."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_zeta_i_j(self, X):\n\t\t'''\n\t\tParameters\n\t\t----------\n\t\tX : np.array\n\t\t\tArray of word counts, shape (N, 2) where N is the vocab size. X[:,0] is the\n\t\t\tpositive class, while X[:,1] is the negative class. None by default\n\n\t\tReturns\n\t\t-------\n\t\tnp.array of z-scores\n\t\t'''\n\t\ty_i, y_j = X.T[0], X.T[1]\n\t\treturn self.get_zeta_i_j_given_separate_counts(y_i, y_j)", "response": "This method returns the zeta - i - j given separate counts."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nget scores for each class in the class", "response": "def get_scores(self, y_i, y_j):\n\t\t'''\n\t\tSame function as get_zeta_i_j_given_separate_counts\n\n\t\tParameters\n\t\t----------\n\t\ty_i, np.array(int)\n\t\t\tArrays of word counts of words occurring in positive class\n\t\ty_j, np.array(int)\n\n\t\tReturns\n\t\t-------\n\t\tnp.array of z-scores\n\t\t'''\n\t\tz_scores = self.get_zeta_i_j_given_separate_counts(y_i, y_j)\n\t\t# scaled_scores = scale_neg_1_to_1_with_zero_mean_abs_max(z_scores)\n\t\treturn z_scores"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef compact(self, term_doc_matrix):\n\t\t'''\n\t\tParameters\n\t\t-------\n\t\tterm_doc_matrix : TermDocMatrix\n\n\t\tReturns\n\t\t-------\n\n\t\tNew term doc matrix\n\t\t'''\n\t\tcount_df = self._get_statistics_dataframe(term_doc_matrix)\n\n\t\treturn term_doc_matrix.remove_terms(\n\t\t\tcount_df[count_df['pmi'] < self.minimum_pmi].index\n\t\t)", "response": "Compact the term doc matrix."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nchecks that the term dict is of type str or list of strings.", "response": "def check_topic_model_string_format(term_dict):\n '''\n Parameters\n ----------\n term_dict: dict {metadataname: [term1, term2, ....], ...}\n\n Returns\n -------\n None\n '''\n if type(term_dict) != dict:\n raise TypeError(\"Argument for term_dict must be a dict, keyed on strings, and contain a list of strings.\")\n for k, v in term_dict.items():\n if type(v) != list:\n raise TypeError(\"Values in term dict must only be lists.\")\n\n if sys.version_info[0] == 2:\n if type(k) != str and type(k) != unicode:\n raise TypeError(\"Keys in term dict must be of type str or unicode.\")\n for e in v:\n if type(k) != str and type(k) != unicode:\n raise TypeError(\"Values in term lists must be str or unicode.\")\n\n if sys.version_info[0] == 3:\n if type(k) != str:\n raise TypeError(\"Keys in term dict must be of type str.\")\n for e in v:\n if type(e) != str:\n raise TypeError(\"Values in term lists must be str.\")"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef inject_metadata_term_lists(self, term_dict):\n '''\n Inserts dictionary of meta data terms into object.\n\n Parameters\n ----------\n term_dict: dict {metadataname: [term1, term2, ....], ...}\n\n Returns\n -------\n self: ScatterChart\n '''\n check_topic_model_string_format(term_dict)\n\n if not self.term_doc_matrix.metadata_in_use():\n raise TermDocMatrixHasNoMetadataException(\"No metadata is present in the term document matrix\")\n\n self.metadata_term_lists = term_dict\n return self", "response": "Inserts dictionary of meta data terms into object."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ninserts a set of descriptions of meta data terms.", "response": "def inject_metadata_descriptions(self, term_dict):\n '''\n Inserts a set of descriptions of meta data terms. These will be displayed\n below the scatter plot when a meta data term is clicked. All keys in the term dict\n must occur as meta data.\n\n Parameters\n ----------\n term_dict: dict {metadataname: str: 'explanation to insert', ...}\n\n Returns\n -------\n self: ScatterChart\n '''\n assert type(term_dict) == dict\n if not self.term_doc_matrix.metadata_in_use():\n raise TermDocMatrixHasNoMetadataException(\"No metadata is present in the term document matrix\")\n # This doesn't seem necessary. If a definition's not in the corpus, it just won't be shown.\n # if set(term_dict.keys()) - set(self.term_doc_matrix.get_metadata()) != set():\n # raise Exception('The following meta data terms are not present: '\n # + ', '.join(list(set(term_dict.keys()) - set(self.term_doc_matrix.get_metadata()))))\n\n if sys.version_info[0] == 2:\n assert set([type(v) for v in term_dict.values()]) - set([str, unicode]) == set()\n else:\n assert set([type(v) for v in term_dict.values()]) - set([str]) == set()\n\n self.metadata_descriptions = term_dict\n return self"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ninjecting custom x and y coordinates for each term into chart.", "response": "def inject_coordinates(self,\n x_coords,\n y_coords,\n rescale_x=None,\n rescale_y=None,\n original_x=None,\n original_y=None):\n '''\n Inject custom x and y coordinates for each term into chart.\n\n Parameters\n ----------\n x_coords: array-like\n positions on x-axis \\in [0,1]\n y_coords: array-like\n positions on y-axis \\in [0,1]\n rescale_x: lambda list[0,1]: list[0,1], default identity\n Rescales x-axis after filtering\n rescale_y: lambda list[0,1]: list[0,1], default identity\n Rescales y-axis after filtering\n original_x : array-like, optional\n Original, unscaled x-values. Defaults to x_coords\n original_y : array-like, optional\n Original, unscaled y-values. Defaults to y_coords\n Returns\n -------\n self: ScatterChart\n\n '''\n self._verify_coordinates(x_coords, 'x')\n self._verify_coordinates(y_coords, 'y')\n self.x_coords = x_coords\n self.y_coords = y_coords\n self._rescale_x = rescale_x\n self._rescale_y = rescale_y\n self.original_x = x_coords if original_x is None else original_x\n self.original_y = y_coords if original_y is None else original_y"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nadds jitter to the array if needed", "response": "def _add_jitter(self, vec):\n \"\"\"\n :param vec: array to jitter\n :return: array, jittered version of arrays\n \"\"\"\n if self.scatterchartdata.jitter == 0 or self.scatterchartdata.jitter is None:\n return vec\n return vec + np.random.rand(1, len(vec))[0] * self.scatterchartdata.jitter"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef draw(self,\n category,\n num_top_words_to_annotate=4,\n words_to_annotate=[],\n scores=None,\n transform=percentile_alphabetical):\n '''Outdated. MPLD3 drawing.\n\n Parameters\n ----------\n category\n num_top_words_to_annotate\n words_to_annotate\n scores\n transform\n\n Returns\n -------\n pd.DataFrame, html of fgure\n '''\n try:\n import matplotlib.pyplot as plt\n except:\n raise Exception(\"matplotlib and mpld3 need to be installed to use this function.\")\n try:\n from mpld3 import plugins, fig_to_html\n except:\n raise Exception(\"mpld3 need to be installed to use this function.\")\n all_categories, other_categories = self._get_category_names(category)\n df = self._term_rank_score_and_frequency_df(all_categories, category, other_categories, scores)\n if self.x_coords is None:\n df['x'], df['y'] = self._get_coordinates_from_transform_and_jitter_frequencies \\\n (category, df, other_categories, transform)\n df_to_annotate = df[(df['not category score rank'] <= num_top_words_to_annotate)\n | (df['category score rank'] <= num_top_words_to_annotate)\n | df['term'].isin(words_to_annotate)]\n words = list(df['term'])\n\n font = {'family': 'sans-serif',\n 'color': 'black',\n 'weight': 'normal',\n 'size': 'large'}\n\n fig, ax = plt.subplots()\n plt.figure(figsize=(10, 10))\n plt.gcf().subplots_adjust(bottom=0.2)\n plt.gcf().subplots_adjust(right=0.2)\n\n points = ax.scatter(self.x_coords,\n self.y_coords,\n c=-df['color_scores'],\n cmap='seismic',\n s=10,\n edgecolors='none',\n alpha=0.9)\n tooltip = plugins.PointHTMLTooltip(points,\n ['%s' % w for w in words],\n css='#a {background-color: white;}')\n plugins.connect(fig, tooltip)\n ax.set_ylim([-.2, 1.2])\n ax.set_xlim([-.2, 1.2])\n ax.xaxis.set_ticks([0., 0.5, 1.])\n ax.yaxis.set_ticks([0., 0.5, 1.])\n ax.set_ylabel(category.title() + ' Frequency Percentile', fontdict=font, labelpad=20)\n ax.set_xlabel('Not ' + category.title() + ' Frequency Percentile', fontdict=font, labelpad=20)\n\n for i, row in df_to_annotate.iterrows():\n # alignment_criteria = row['category score rank'] < row['not category score rank']\n alignment_criteria = i % 2 == 0\n horizontalalignment = 'right' if alignment_criteria else 'left'\n verticalalignment = 'bottom' if alignment_criteria else 'top'\n term = row['term']\n ax.annotate(term,\n (self.x_coords[i], self.y_data[i]),\n size=15,\n horizontalalignment=horizontalalignment,\n verticalalignment=verticalalignment,\n )\n # texts.append(\n # ax.text(row['dem freq scaled'], row['rep freq scaled'], row['word'])\n # )\n # adjust_text(texts, arrowprops=dict(arrowstyle=\"->\", color='r', lw=0.5))\n plt.show()\n return df, fig_to_html(fig)", "response": "Outdated. MPLD3 drawing.\n\n Parameters\n ----------\n category\n num_top_words_to_annotate\n words_to_annotate\n scores\n transform\n\n Returns\n -------\n pd.DataFrame, html of fgure"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_zeta_i_j_given_separate_counts(self, y_i, y_j):\n\t\t'''\n\t\tParameters\n\t\t----------\n\t\ty_i, np.array(int)\n\t\t\tArrays of word counts of words occurring in positi ve class\n\t\ty_j, np.array(int)\n\n\t\tReturns\n\t\t-------\n\t\tnp.array of z-scores\n\t\t'''\n\t\tn_i, n_j = y_i.sum(), y_j.sum()\n\t\tdelta_i_j = (np.log((y_i + 1) / (1. + n_i - y_i))\n\t\t - np.log((y_j + 1) / (1. + n_j - y_j)))\n\t\treturn delta_i_j", "response": "This function calculates the zeta_i_j given the counts of words occurring in positi y_i and y_j given the separate counts of words occurring in positi y_j."} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nget scores and p - values from a TermDocMatrix", "response": "def get_scores_and_p_values(self, tdm, category):\n\t\t'''\n\t\tParameters\n\t\t----------\n\t\ttdm: TermDocMatrix\n\t\tcategory: str, category name\n\n\t\tReturns\n\t\t-------\n\t\tpd.DataFrame(['coef', 'p-val'])\n\t\t'''\n\t\tX = tdm._X\n\t\ty = self._make_response_variable_1_or_negative_1(category, tdm)\n\t\tpX = X / X.sum(axis=1)\n\t\tansX = self._anscombe_transform(pX.copy())\n\t\tB, istop, itn, r1norm, r2norm, anorm, acond, arnorm, xnorm, var\\\n\t\t\t= lsqr(A=ansX, b=y, calc_var=True)"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef delete_columns(mat, columns_to_delete):\n\t'''\n\t>>> a = csr_matrix(np.array([[0, 1, 3, 0, 1, 0],\n\t\t [0, 0, 1, 0, 1, 1]])\n\t>>> delete_columns(a, [1,2]).todense()\n\tmatrix([[0, 0, 1, 0],\n [0, 0, 1, 1]])\n\n\tParameters\n\t----------\n\tmat : csr_matrix\n\tcolumns_to_delete : list[int]\n\n\tReturns\n\t-------\n\tcsr_matrix that is stripped of columns indices columns_to_delete\n\t'''\n\tcolumn_mask = np.ones(mat.shape[1], dtype=bool)\n\tcolumn_mask[columns_to_delete] = 0\n\treturn mat.tocsc()[:, column_mask].tocsr()", "response": "Delete columns from a csr_matrix."} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nsetting the last column index in the table.", "response": "def set_last_col_idx(self, last_col_idx):\n\t\t'''\n\t\tParameters\n\t\t----------\n\t\tparam last_col_idx : int\n\t\t\tnumber of columns\n\t\t'''\n\t\tassert last_col_idx >= self._max_col\n\t\tself._max_col = last_col_idx\n\t\treturn self"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef set_last_row_idx(self, last_row_idx):\n\t\t'''\n\t\tParameters\n\t\t----------\n\t\tparam last_row_idx : int\n\t\t\tnumber of rows\n\t\t'''\n\t\tassert last_row_idx >= self._max_row\n\t\tself._max_row = last_row_idx\n\t\treturn self", "response": "Set the last row index in the table."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_feats(self, doc):\n\t\t'''\n\t\tParameters\n\t\t----------\n\t\tdoc, Spacy Docs\n\n\t\tReturns\n\t\t-------\n\t\tCounter noun chunk -> count\n\t\t'''\n\t\treturn Counter([str(c).lower() for c in doc.noun_chunks])", "response": "Return a Counter of the noun chunks in a document."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a function that returns a list of tokens for the given text.", "response": "def nltk_tokenzier_factory(nltk_tokenizer):\n\t'''\n\tParameters\n\t----------\n\tnltk_tokenizer : nltk.tokenize.* instance (e.g., nltk.TreebankWordTokenizer())\n\n\tReturns\n\t-------\n\tDoc of tweets\n\n\tNotes\n\t-------\n\tRequires NLTK to be installed\n\t'''\n\n\tdef tokenize(text):\n\t\ttoks = []\n\t\tfor tok in nltk_tokenizer.tokenize(text):\n\t\t\tif len(tok) > 0:\n\t\t\t\ttoks.append(Tok(_get_pos_tag(tok),\n\t\t\t\t tok.lower(),\n\t\t\t\t tok.lower(),\n\t\t\t\t ent_type='',\n\t\t\t\t tag=''))\n\t\treturn Doc([toks], text)\n\n\treturn tokenize"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef set_embeddings(self, embeddings):\n '''\n Specifies fixed set of embeddings\n :param embeddings: array-like, sparse or dense, shape should be (embedding size, # terms)\n :return: EmbeddingsResolver\n '''\n if self.embeddings_ is not None:\n raise Exception(\"You have already set embeddings by running set_embeddings or set_embeddings_model.\")\n assert embeddings.shape[1] == self.corpus_.get_num_terms()\n self.embeddings_ = embeddings.T\n self.vocab_ = self.corpus_.get_terms()\n return self", "response": "Sets the embeddings for the given corpus."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nprojects the embeddings into a vector vector.", "response": "def project(self, projection_model=None):\n '''\n :param projection_model: sklearn unsupervised model (e.g., PCA) by default the recommended model is umap.UMAP,\n which requires UMAP in to be installed\n\n :return: array, shape (num dimension, vocab size)\n '''\n if self.embeddings_ is None:\n raise Exception(\"Run set_embeddings_model or set_embeddings to get embeddings\")\n if projection_model is None:\n try:\n import umap\n except:\n raise Exception(\"Please install umap (pip install umap-learn) to use the default projection_model.\")\n projection_model = umap.UMAP(min_dist=0.5, metric='cosine')\n axes = projection_model.fit_transform(self.embeddings_)\n return axes"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get_topics_from_terms(self,\n\t terms=None,\n\t num_terms_per_topic=10,\n\t scorer=RankDifference()):\n\t\t'''\n\t\tParameters\n\t\t----------\n\t\tterms : list or None\n\t\t\tIf terms is list, make these the seed terms for the topoics\n\t\t\tIf none, use the first 30 terms in get_scaled_f_scores_vs_background\n\t\tnum_terms_per_topic : int, default 10\n\t\t\tUse this many terms per topic\n\t\tscorer : TermScorer\n\t\t\tImplements get_scores, default is RankDifferce, which tends to work best\n\n\t\tReturns\n\t\t-------\n\t\tdict: {term: [term1, ...], ...}\n\t\t'''\n\t\ttopic_model = {}\n\n\t\tif terms is None:\n\t\t\tterms = self.corpus.get_scaled_f_scores_vs_background().index[:30]\n\n\t\tfor term in terms:\n\t\t\ttermidx = self.termidxstore.getidxstrict(term)\n\t\t\tlabels = self.sentX[:, termidx].astype(bool).todense().A1\n\t\t\tposcnts = self.sentX[labels, :].astype(bool).sum(axis=0).A1\n\t\t\tnegcnts = self.sentX[~labels, :].astype(bool).sum(axis=0).A1\n\t\t\tscores = scorer.get_scores(poscnts, negcnts)\n\t\t\ttopic_model[term] = [self.termidxstore.getval(i) for i in\n\t\t\t np.argsort(-scores)[:num_terms_per_topic]]\n\t\treturn topic_model", "response": "This function returns a dictionary of topics from a list of terms."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_ranks(self):\n\t\t'''\n\t\tReturns\n\t\t-------\n\t\tpd.DataFrame\n\n\t\t'''\n\t\tif self._use_non_text_features:\n\t\t\treturn self._term_doc_matrix.get_metadata_freq_df()\n\t\telse:\n\t\t\treturn self._term_doc_matrix.get_term_freq_df()", "response": "Returns the rank matrix of the term document."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef build(self):\n if self._category_text_iter is None:\n raise CategoryTextIterNotSetError()\n nlp = self.get_nlp()\n\n category_document_iter = (\n (category, self._clean_function(raw_text))\n for category, raw_text\n in self._category_text_iter\n )\n term_doc_matrix = self._build_from_category_spacy_doc_iter(\n (\n (category, nlp(text))\n for (category, text)\n in category_document_iter\n if text.strip() != ''\n )\n )\n return term_doc_matrix", "response": "Generate a TermDocMatrix from data in parameters."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef censor_entity_types(self, entity_types):\n # type: (set) -> TermDocMatrixFactory\n '''\n Entity types to exclude from feature construction. Terms matching\n specificed entities, instead of labeled by their lower case orthographic\n form or lemma, will be labeled by their entity type.\n\n Parameters\n ----------\n entity_types : set of entity types outputted by spaCy\n 'TIME', 'WORK_OF_ART', 'PERSON', 'MONEY', 'ORG', 'ORDINAL', 'DATE',\n 'CARDINAL', 'LAW', 'QUANTITY', 'GPE', 'PERCENT'\n\n Returns\n ---------\n self\n '''\n assert type(entity_types) == set\n self._entity_types_to_censor = entity_types\n self._feats_from_spacy_doc = FeatsFromSpacyDoc(\n use_lemmas=self._use_lemmas,\n entity_types_to_censor=self._entity_types_to_censor\n )\n return self", "response": "Sets the entity types to exclude from feature construction."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _build_from_category_spacy_doc_iter(self, category_doc_iter):\n '''\n Parameters\n ----------\n category_doc_iter : iterator of (string category name, spacy.tokens.doc.Doc) pairs\n\n Returns\n ----------\n t : TermDocMatrix\n '''\n term_idx_store = IndexStore()\n category_idx_store = IndexStore()\n metadata_idx_store = IndexStore()\n X, mX, y = self._get_features_and_labels_from_documents_and_indexes \\\n (category_doc_iter,\n category_idx_store,\n term_idx_store,\n metadata_idx_store)\n return TermDocMatrix(X,\n mX,\n y,\n term_idx_store=term_idx_store,\n category_idx_store=category_idx_store,\n metadata_idx_store=metadata_idx_store)", "response": "Build a TermDocMatrix from a category spacy. tokens. doc. Doc iterator."} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nparses the raw text and returns a CSR matrix that contains the features for the term.", "response": "def feats_from_doc(self, raw_text):\n '''\n Parameters\n ----------\n raw_text, uncleaned text for parsing out features\n\n Returns\n -------\n csr_matrix, feature matrix\n '''\n parsed_text = self._nlp(self._clean_function(raw_text))\n X_factory = CSRMatrixFactory()\n X_factory.set_last_col_idx(self._term_idx_store.getnumvals() - 1)\n term_freq = self._get_features_from_parsed_text(parsed_text, self._term_idx_store)\n self._register_document_features_with_X_factory(X_factory, 0, term_freq)\n return X_factory.get_csr_matrix()"} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn a dictionary of all the available lexicon.", "response": "def get_lexicons(self, num_terms=10):\n '''\n Parameters\n ----------\n num_terms, int\n\n Returns\n -------\n dict\n '''\n return {k: v.index[:num_terms]\n for k, v in self.lexicons.items()}"} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncompacts the term document matrix into a new TermDocMatrix object.", "response": "def compact(self, term_doc_matrix):\n '''\n Parameters\n ----------\n term_doc_matrix : TermDocMatrix\n Term document matrix object to compact\n Returns\n -------\n New term doc matrix\n '''\n rank_df = self.scorer.get_rank_df(term_doc_matrix)\n optimal_rank = self._find_optimal_rank(rank_df)\n compacted_term_doc_matrix = self._prune_higher_ranked_terms(term_doc_matrix, rank_df, optimal_rank)\n print('max terms', self.max_terms, 'optimal_rank', optimal_rank,\n 'num_terms', compacted_term_doc_matrix.get_num_terms())\n return compacted_term_doc_matrix"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ncompacts the term document matrix into a new set of terms.", "response": "def compact(self, term_doc_matrix):\n '''\n Parameters\n ----------\n term_doc_matrix : TermDocMatrix\n Term document matrix object to compact\n Returns\n -------\n TermDocMatrix\n\n\n '''\n rank_df = self.scorer.get_rank_df(term_doc_matrix)\n return self._prune_higher_ranked_terms(term_doc_matrix, rank_df, self.rank)"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef add_phrases(self, corpus):\n\t\t'''\n\t\tParameters\n\t\t----------\n\t\tcorpus: Corpus for phrase augmentation\n\n\t\tReturns\n\t\t-------\n\t\tNew ParsedCorpus containing unigrams in corpus and new phrases\n\t\t'''\n\t\tfrom gensim.models import Phrases\n\n\t\tassert isinstance(corpus, ParsedCorpus)\n\t\tself.phrases = [Phrases(CorpusAdapterForGensim.get_sentences(corpus), delimiter=' ')]\n\n\t\tfor i in range(1, self.max_tokens_per_phrase):\n\t\t\tself.phrases.append(Phrases(self.phrases[-1][CorpusAdapterForGensim.get_sentences(corpus)]))\n\n\t\treturn self", "response": "Adds phrases to the list of phrases that are added to the list of phrases."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ntrains passive aggressive classifier.", "response": "def passive_aggressive_train(self):\n\t\t'''Trains passive aggressive classifier\n\n\t\t'''\n\t\tself._clf = PassiveAggressiveClassifier(n_iter=50, C=0.2, n_jobs=-1, random_state=0)\n\t\tself._clf.fit(self._term_doc_matrix._X, self._term_doc_matrix._y)\n\t\ty_dist = self._clf.decision_function(self._term_doc_matrix._X)\n\t\tpos_ecdf = ECDF(y_dist[y_dist >= 0])\n\t\tneg_ecdf = ECDF(y_dist[y_dist <= 0])\n\n\t\tdef proba_function(distance_from_hyperplane):\n\t\t\tif distance_from_hyperplane > 0:\n\t\t\t\treturn pos_ecdf(distance_from_hyperplane) / 2. + 0.5\n\t\t\telif distance_from_hyperplane < 0:\n\t\t\t\treturn pos_ecdf(distance_from_hyperplane) / 2.\n\t\t\treturn 0.5\n\n\t\tself._proba = proba_function\n\t\treturn self"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nbuild a new deployed classifier.", "response": "def build(self):\n\t\t'''Builds Depoyed Classifier\n\t\t'''\n\t\tif self._clf is None:\n\t\t\traise NeedToTrainExceptionBeforeDeployingException()\n\t\treturn DeployedClassifier(self._category,\n\t\t self._term_doc_matrix._category_idx_store,\n\t\t self._term_doc_matrix._term_idx_store,\n\t\t self._term_doc_matrix_factory)"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef build(term_to_index_dict):\n\t\t'''\n\t\tParameters\n\t\t----------\n\t\tterm_to_index_dict: term -> idx dictionary\n\n\t\tReturns\n\t\t-------\n\t\tIndexStore\n\t\t'''\n\t\tidxstore = IndexStore()\n\t\tidxstore._val2i = term_to_index_dict\n\t\tidxstore._next_i = len(term_to_index_dict)\n\t\tidxstore._i2val = [None for _ in range(idxstore._next_i)]\n\t\tfor term, idx in idxstore._val2i.items():\n\t\t\tidxstore._i2val[idx] = term\n\t\treturn idxstore", "response": "Build a new index store for the given term_to_index_dict."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a new object of the correct type.", "response": "def build(self):\n\t\t'''\n\t\tReturns\n\t\t-------\n\t\tCorpus\n\t\t'''\n\t\tconstructor_kwargs = self._get_build_kwargs()\n\t\tif type(self.raw_texts) == list:\n\t\t\tconstructor_kwargs['raw_texts'] = np.array(self.raw_texts)\n\t\telse:\n\t\t\tconstructor_kwargs['raw_texts'] = self.raw_texts\n\t\treturn Corpus(**constructor_kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncompact the term doc matrix.", "response": "def compact(self, term_doc_matrix):\n\t\t'''\n\t\tParameters\n\t\t----------\n\t\tterm_doc_matrix : TermDocMatrix\n\t\t\tTerm document matrix object to compact\n\n\t\tReturns\n\t\t-------\n\t\tNew term doc matrix\n\t\t'''\n\t\tdomain_mat = CombineDocsIntoDomains(term_doc_matrix).get_new_term_doc_mat(self.doc_domains)\n\t\tdomain_count = (domain_mat > 0).sum(axis=0)\n\t\tvalid_term_mask = (self.max_domain_count >= domain_count) \\\n\t\t & (domain_count >= self.min_domain_count)\n\t\tindices_to_compact = np.arange(term_doc_matrix.get_num_terms())[~valid_term_mask.A1]\n\t\treturn term_doc_matrix.remove_terms_by_indices(indices_to_compact)"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn a pd. Series of all raw documents", "response": "def get_texts(self):\n '''\n Returns\n -------\n pd.Series, all raw documents\n '''\n if sys.version_info[0] == 2:\n return self._df[self._parsed_col]\n return self._df[self._parsed_col].apply(str)"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef term_group_freq_df(self, group_col):\n # type: (str) -> pd.DataFrame\n '''\n Returns a dataframe indexed on the number of groups a term occured in.\n\n Parameters\n ----------\n group_col\n\n Returns\n -------\n pd.DataFrame\n '''\n group_idx_store = IndexStore()\n X = self._X\n group_idx_to_cat_idx, row_group_cat \\\n = self._get_group_docids_and_index_store(X, group_col, group_idx_store)\n newX = self._change_document_type_in_matrix(X, row_group_cat)\n newX = self._make_all_positive_data_ones(newX)\n category_row = newX.tocoo().row\n for group_idx, cat_idx in group_idx_to_cat_idx.items():\n category_row[category_row == group_idx] = cat_idx\n catX = self._change_document_type_in_matrix(newX, category_row)\n return self._term_freq_df_from_matrix(catX)", "response": "Returns a dataframe indexed on the number of groups a term occured in."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef sparse_var(X):\n '''\n Compute variance from\n :param X:\n :return:\n '''\n Xc = X.copy()\n Xc.data **= 2\n return np.array(Xc.mean(axis=0) - np.power(X.mean(axis=0), 2))[0]", "response": "Compute variance from sparse matrix X"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef set_categories(self,\n category_name,\n not_category_names=[],\n neutral_category_names=[]):\n '''\n Specify the category to score. Optionally, score against a specific set of categories.\n '''\n tdf = self.term_ranker_.get_ranks()\n d = {'cat': tdf[category_name + ' freq']}\n if not_category_names == []:\n not_category_names = [c + ' freq' for c in self.corpus_.get_categories()\n if c != category_name]\n else:\n not_category_names = [c + ' freq' for c in not_category_names]\n d['ncat'] = tdf[not_category_names].sum(axis=1)\n if neutral_category_names == []:\n # neutral_category_names = [c + ' freq' for c in self.corpus.get_categories()\n # if c != category_name and c not in not_category_names]\n pass\n else:\n neutral_category_names = [c + ' freq' for c in neutral_category_names]\n for i, c in enumerate(neutral_category_names):\n d['neut%s' % (i)] = tdf[c]\n self.tdf_ = pd.DataFrame(d)\n self.category_name = category_name\n self.not_category_names = [c[:-5] for c in not_category_names]\n self.neutral_category_names = [c[:-5] for c in neutral_category_names]\n self.category_name_is_set_ = True\n return self", "response": "Sets the categories of the term. Optionally score against a specific set of categories."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_t_statistics(self):\n '''\n In this case, parameters a and b aren't used, since this information is taken\n directly from the corpus categories.\n\n Returns\n -------\n\n '''\n\n X = self._get_X()\n cat_X, ncat_X = self._get_cat_and_ncat(X)\n\n mean_delta = self._get_mean_delta(cat_X, ncat_X)\n cat_var = sparse_var(cat_X)\n ncat_var = sparse_var(ncat_X)\n cat_n = cat_X.shape[0]\n ncat_n = ncat_X.shape[0]\n pooled_stderr = np.sqrt(cat_var / cat_n + ncat_var / ncat_n)\n\n tt = mean_delta / pooled_stderr\n\n # Use Satterthaite-Welch adjustment for degrees of freedom\n degs_of_freedom = (cat_var ** 2 / cat_n + ncat_var ** 2 / ncat_n) ** 2 / (\n (cat_var ** 2 / cat_n) ** 2 / (cat_n - 1)\n + (ncat_var ** 2 / ncat_n) ** 2 / (ncat_n - 1)\n )\n\n only_in_neutral_mask = self.tdf_[['cat', 'ncat']].sum(axis=1) == 0\n pvals = stats.t.sf(np.abs(tt), degs_of_freedom)\n tt[only_in_neutral_mask] = 0\n pvals[only_in_neutral_mask] = 0\n\n return tt, pvals", "response": "Returns the t - statistic and p - statistic values for the current level of the log - likelihood of the current level of the log - likelihood of the current level of the log - likelihood of the corpus."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef set_background_corpus(self, background):\n '''\n Parameters\n ----------\n background\n\n '''\n if issubclass(type(background), TermDocMatrixWithoutCategories):\n self._background_corpus = pd.DataFrame(background\n .get_term_freq_df()\n .sum(axis=1),\n columns=['background']).reset_index()\n self._background_corpus.columns = ['word', 'background']\n elif (type(background) == pd.DataFrame\n and set(background.columns) == set(['word', 'background'])):\n self._background_corpus = background\n else:\n raise Exception('The argument named background must be a subclass of TermDocMatrix or a ' \\\n + 'DataFrame with columns \"word\" and \"background\", where \"word\" ' \\\n + 'is the term text, and \"background\" is its frequency.')", "response": "Sets the _background_corpus property of the object."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_term_and_background_counts(self):\n '''\n Returns\n -------\n A pd.DataFrame consisting of unigram term counts of words occurring\n in the TermDocumentMatrix and their corresponding background corpus\n counts. The dataframe has two columns, corpus and background.\n\n >>> corpus.get_unigram_corpus().get_term_and_background_counts()\n corpus background\n obama 702.0 565739.0\n romney 570.0 695398.0\n barack 248.0 227861.0\n ...\n '''\n background_df = self._get_background_unigram_frequencies()\n corpus_freq_df = self.get_term_count_df()\n corpus_unigram_freq = self._get_corpus_unigram_freq(corpus_freq_df)\n df = corpus_unigram_freq.join(background_df, how='outer').fillna(0)\n del df.index.name\n return df", "response": "Returns a pd. DataFrame consisting of unigram term counts of words occurring\n in the TermDocumentMatrix and their corresponding background corpusCounts."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef list_extra_features(self):\n '''\n Returns\n -------\n List of dicts. One dict for each document, keys are metadata, values are counts\n '''\n return FeatureLister(self._mX,\n self._metadata_idx_store,\n self.get_num_docs()).output()", "response": "Returns a list of dicts that contain metadata and counts for each document."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a TermDocumentMatrix consisting of only terms which occur at least minimum_term_count.", "response": "def remove_infrequent_words(self, minimum_term_count, term_ranker=AbsoluteFrequencyRanker):\n '''\n Returns\n -------\n A new TermDocumentMatrix consisting of only terms which occur at least minimum_term_count.\n '''\n tdf = term_ranker(self).get_ranks().sum(axis=1)\n return self.remove_terms(list(tdf[tdf <= minimum_term_count].index))"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a TermDocumentMatrix consisting of only terms in the current TermDocumentMatrix that are not spaCy entity tags.", "response": "def remove_entity_tags(self):\n '''\n Returns\n -------\n A new TermDocumentMatrix consisting of only terms in the current TermDocumentMatrix\n that aren't spaCy entity tags.\n\n Note: Used if entity types are censored using FeatsFromSpacyDoc(tag_types_to_censor=...).\n '''\n terms_to_remove = [term for term in self._term_idx_store._i2val\n if any([word in SPACY_ENTITY_TAGS for word in term.split()])]\n return self.remove_terms(terms_to_remove)"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef remove_terms_used_in_less_than_num_docs(self, threshold):\n '''\n Parameters\n ----------\n threshold: int\n Minimum number of documents term should appear in to be kept\n\n Returns\n -------\n TermDocMatrix, new object with terms removed.\n '''\n term_counts = self._X.astype(bool).astype(int).sum(axis=0).A[0]\n terms_to_remove = np.where(term_counts < threshold)[0]\n return self.remove_terms_by_indices(terms_to_remove)", "response": "Removes terms used in less than num_docs from the term doc matrix."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef get_stoplisted_unigram_corpus(self, stoplist=None):\n '''\n Parameters\n -------\n stoplist : list, optional\n\n Returns\n -------\n A new TermDocumentMatrix consisting of only unigrams in the current TermDocumentMatrix.\n '''\n if stoplist is None:\n stoplist = self.get_default_stoplist()\n else:\n stoplist = [w.lower() for w in stoplist]\n return self._remove_terms_from_list(stoplist)", "response": "Returns a TermDocumentMatrix consisting of only unigrams in the current TermDocumentMatrix."} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns a TermDocumentMatrix consisting of only unigrams in the current TermDocumentMatrix with only custom unigrams in the current TermDocumentMatrix.", "response": "def get_stoplisted_unigram_corpus_and_custom(self,\n custom_stoplist):\n '''\n Parameters\n -------\n stoplist : list of lower-cased words, optional\n\n Returns\n -------\n A new TermDocumentMatrix consisting of only unigrams in the current TermDocumentMatrix.\n '''\n if type(custom_stoplist) == str:\n custom_stoplist = [custom_stoplist]\n return self._remove_terms_from_list(set(self.get_default_stoplist())\n | set(w.lower() for w in custom_stoplist))"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_doc_lengths(self):\n '''\n Returns a list of document lengths in words\n\n Returns\n -------\n np.array\n '''\n idx_to_delete_list = self._build_term_index_list(True, self._get_non_unigrams())\n unigram_X, _ = self._get_X_after_delete_terms(idx_to_delete_list)\n return unigram_X.sum(axis=1).A1", "response": "Returns a list of document lengths in words\n Returns ------- np. array\n Returns ------- np. array\n Returns ------- np. array\n "} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nremove terms from the term doc matrix.", "response": "def remove_terms_by_indices(self, idx_to_delete_list):\n '''\n Parameters\n ----------\n idx_to_delete_list, list\n\n Returns\n -------\n TermDocMatrix\n '''\n new_X, new_term_idx_store = self._get_X_after_delete_terms(idx_to_delete_list)\n\n return self._make_new_term_doc_matrix(new_X=new_X,\n new_mX=self._mX,\n new_y=None,\n new_category_idx_store=None,\n new_term_idx_store=new_term_idx_store,\n new_metadata_idx_store=self._metadata_idx_store,\n new_y_mask=np.ones(new_X.shape[0]).astype(np.bool))"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn a dictionary of terms and doc_ids.", "response": "def term_doc_lists(self):\n '''\n Returns\n -------\n dict\n '''\n doc_ids = self._X.transpose().tolil().rows\n terms = self._term_idx_store.values()\n return dict(zip(terms, doc_ids))"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef apply_ranker(self, term_ranker, use_non_text_features):\n '''\n Parameters\n ----------\n term_ranker : TermRanker\n\n Returns\n -------\n pd.Dataframe\n '''\n if use_non_text_features:\n return term_ranker(self).use_non_text_features().get_ranks()\n return term_ranker(self).get_ranks()", "response": "Applies the ranker to the dataframe."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nadd a list of document names as metadata.", "response": "def add_doc_names_as_metadata(self, doc_names):\n '''\n :param doc_names: array-like[str], document names of reach document\n :return: Corpus-like object with doc names as metadata. If two documents share the same name\n (doc number) will be appended to their names.\n '''\n if len(doc_names) != self.get_num_docs():\n raise Exception(\"The parameter doc_names contains %s elements. \"\n \"It should have %s elements, one per document.\" % (len(doc_names), self.get_num_docs()))\n\n doc_names_counter = collections.Counter(np.array(doc_names))\n metafact = CSRMatrixFactory()\n metaidxstore = IndexStore()\n doc_id_uses = collections.Counter()\n for i in range(self.get_num_docs()):\n doc_id = doc_names[i]\n if doc_names_counter[doc_id] > 1:\n doc_id_uses[doc_id] += 1\n doc_name_idx = metaidxstore.getidx('%s (%s)' % (doc_id, doc_id_uses[doc_id]))\n else:\n doc_name_idx = metaidxstore.getidx(doc_id)\n metafact[i, i] = doc_name_idx\n return self.add_metadata(metafact.get_csr_matrix(), metaidxstore)"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef add_metadata(self, metadata_matrix, meta_index_store):\n '''\n Returns a new corpus with a the metadata matrix and index store integrated.\n\n :param metadata_matrix: scipy.sparse matrix (# docs, # metadata)\n :param meta_index_store: IndexStore of metadata values\n :return: TermDocMatrixWithoutCategories\n '''\n assert isinstance(meta_index_store, IndexStore)\n assert len(metadata_matrix.shape) == 2\n assert metadata_matrix.shape[0] == self.get_num_docs()\n return self._make_new_term_doc_matrix(new_X=self._X,\n new_y=None,\n new_category_idx_store=None,\n new_y_mask=np.ones(self.get_num_docs()).astype(bool),\n new_mX=metadata_matrix,\n new_term_idx_store=self._term_idx_store,\n new_metadata_idx_store=meta_index_store)", "response": "Returns a new corpus with a the metadata matrix and index store integrated."} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ngets the features from the doc.", "response": "def get_feats(self, doc):\n\t\t'''\n\t\tParameters\n\t\t----------\n\t\tdoc, Spacy Docs\n\n\t\tReturns\n\t\t-------\n\t\tCounter (unigram, bigram) -> count\n\t\t'''\n\t\tngram_counter = Counter()\n\t\tfor sent in doc.sents:\n\t\t\tunigrams = self._get_unigram_feats(sent)\n\t\t\tbigrams = self._get_bigram_feats(unigrams)\n\t\t\tngram_counter += Counter(chain(unigrams, bigrams))\n\t\treturn ngram_counter"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ncomputes the score of the Mann Whitney corrected p z - values.", "response": "def get_score_df(self, correction_method=None):\n '''\n Computes Mann Whitney corrected p, z-values. Falls back to normal approximation when numerical limits are reached.\n\n :param correction_method: str or None, correction method from statsmodels.stats.multitest.multipletests\n 'fdr_bh' is recommended.\n :return: pd.DataFrame\n '''\n X = self._get_X().astype(np.float64)\n X = X / X.sum(axis=1)\n cat_X, ncat_X = self._get_cat_and_ncat(X)\n\n def normal_apx(u, x, y):\n # from https://stats.stackexchange.com/questions/116315/problem-with-mann-whitney-u-test-in-scipy\n m_u = len(x) * len(y) / 2\n sigma_u = np.sqrt(len(x) * len(y) * (len(x) + len(y) + 1) / 12)\n z = (u - m_u) / sigma_u\n return 2*norm.cdf(z)\n scores = []\n for i in range(cat_X.shape[1]):\n cat_list = cat_X.T[i].A1\n ncat_list = ncat_X.T[i].A1\n try:\n if cat_list.mean() > ncat_list.mean():\n mw = mannwhitneyu(cat_list, ncat_list, alternative='greater')\n if mw.pvalue in (0, 1):\n mw.pvalue = normal_apx(mw.staistic, cat_list, ncat_list)\n\n scores.append({'mwu': mw.statistic, 'mwu_p': mw.pvalue, 'mwu_z': norm.isf(float(mw.pvalue)), 'valid':True})\n\n else:\n mw = mannwhitneyu(ncat_list, cat_list, alternative='greater')\n if mw.pvalue in (0, 1):\n mw.pvalue = normal_apx(mw.staistic, ncat_list, cat_list)\n\n scores.append({'mwu': -mw.statistic, 'mwu_p': 1 - mw.pvalue, 'mwu_z': 1. - norm.isf(float(mw.pvalue)), 'valid':True})\n except:\n scores.append({'mwu': 0, 'mwu_p': 0, 'mwu_z': 0, 'valid':False})\n\n score_df = pd.DataFrame(scores, index=self.corpus_.get_terms()).fillna(0)\n if correction_method is not None:\n from statsmodels.stats.multitest import multipletests\n for method in ['mwu']:\n valid_pvals = score_df[score_df.valid].mwu_p\n valid_pvals_abs = np.min([valid_pvals, 1-valid_pvals], axis=0)\n valid_pvals_abs_corr = multipletests(valid_pvals_abs, method=correction_method)[1]\n score_df[method + '_p_corr'] = 0.5\n\n valid_pvals_abs_corr[valid_pvals > 0.5] = 1. - valid_pvals_abs_corr[valid_pvals > 0.5]\n valid_pvals_abs_corr[valid_pvals < 0.5] = valid_pvals_abs_corr[valid_pvals < 0.5]\n score_df.loc[score_df.valid, method + '_p_corr'] = valid_pvals_abs_corr\n score_df[method + '_z'] = -norm.ppf(score_df[method + '_p_corr'])\n return score_df"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef reduce_terms(term_doc_matrix, scores, num_term_to_keep=None):\n\t\t'''\n\t\tParameters\n\t\t----------\n\t\tterm_doc_matrix: TermDocMatrix or descendant\n\t\tscores: array-like\n\t\t\tSame length as number of terms in TermDocMatrix.\n\t\tnum_term_to_keep: int, default=4000.\n\t\t\tShould be> 0. Number of terms to keep. Will keep between num_terms_to_keep/2 and num_terms_to_keep.\n\n\t\tReturns\n\t\t-------\n\t\tTermDocMatrix stripped of non-important terms., array of scores\n\t\t'''\n\t\tterms_to_show = AutoTermSelector.get_selected_terms(\n\t\t\tterm_doc_matrix, scores, num_term_to_keep)\n\t\treturn term_doc_matrix.remove_terms(set(term_doc_matrix.get_terms())\n\t\t - set(terms_to_show))", "response": "This function is used to reduce the terms in a TermDocMatrix."} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning the scores of the current category in the corpus.", "response": "def get_scores(self, *args):\n\t\t'''\n\t\tIn this case, args aren't used, since this information is taken\n\t\tdirectly from the corpus categories.\n\n\t\tReturns\n\t\t-------\n\t\tnp.array, scores\n\t\t'''\n\t\tif self.tdf_ is None:\n\t\t\traise Exception(\"Use set_category_name('category name', ['not category name', ...]) \" +\n\t\t\t \"to set the category of interest\")\n\n\t\tavgdl = self.tdf_.sum(axis=0).mean()\n\n\t\tdef idf(cat):\n\t\t\t# Number of categories with term\n\t\t\tn_q = (self.tdf_ > 0).astype(int).max(axis=1).sum()\n\t\t\tN = len(self.tdf_)\n\t\t\treturn (N - n_q + 0.5) / (n_q + 0.5)\n\n\t\tdef length_adjusted_tf(cat):\n\t\t\ttf = self.tdf_[cat]\n\t\t\tdl = self.tdf_[cat].sum()\n\t\t\treturn ((tf * (self.k1 + 1))\n\t\t\t / (tf + self.k1 * (1 - self.b + self.b * (dl / avgdl))))\n\n\t\tdef bm25_score(cat):\n\t\t\treturn - length_adjusted_tf(cat) * np.log(idf(cat))\n\n\t\tscores = bm25_score('cat') - bm25_score('ncat')\n\t\treturn scores"} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn the p - values of the scaled F - scores of the given word counts.", "response": "def get_p_vals(self, X):\n\t\t'''\n\t\tImputes p-values from the Z-scores of `ScaledFScore` scores. Assuming incorrectly\n\t\tthat the scaled f-scores are normally distributed.\n\n\t\tParameters\n\t\t----------\n\t\tX : np.array\n\t\t\tArray of word counts, shape (N, 2) where N is the vocab size. X[:,0] is the\n\t\t\tpositive class, while X[:,1] is the negative class.\n\n\t\tReturns\n\t\t-------\n\t\tnp.array of p-values\n\n\t\t'''\n\t\tz_scores = ScaledFZScore(self.scaler_algo, self.beta).get_scores(X[:,0], X[:,1])\n\t\treturn norm.cdf(z_scores)"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns a projection of the categories", "response": "def project(self, term_doc_mat, x_dim=0, y_dim=1):\n '''\n Returns a projection of the categories\n\n :param term_doc_mat: a TermDocMatrix\n :return: CategoryProjection\n '''\n return self._project_category_corpus(self._get_category_metadata_corpus(term_doc_mat),\n x_dim, y_dim)"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef project_with_metadata(self, term_doc_mat, x_dim=0, y_dim=1):\n '''\n Returns a projection of the\n\n :param term_doc_mat: a TermDocMatrix\n :return: CategoryProjection\n '''\n return self._project_category_corpus(self._get_category_metadata_corpus_and_replace_terms(term_doc_mat),\n x_dim, y_dim)", "response": "Returns a project of the category with metadata"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef comparable(klass):\n # On Python 2, __cmp__ will just work, so no need to add extra methods:\n if not is_py3:\n return klass\n\n def __eq__(self, other):\n c = self.__cmp__(other)\n if c is NotImplemented:\n return c\n\n return c == 0\n\n def __ne__(self, other):\n c = self.__cmp__(other)\n if c is NotImplemented:\n return c\n\n return c != 0\n\n def __lt__(self, other):\n c = self.__cmp__(other)\n if c is NotImplemented:\n return c\n\n return c < 0\n\n def __le__(self, other):\n c = self.__cmp__(other)\n if c is NotImplemented:\n return c\n\n return c <= 0\n\n def __gt__(self, other):\n c = self.__cmp__(other)\n if c is NotImplemented:\n return c\n\n return c > 0\n\n def __ge__(self, other):\n c = self.__cmp__(other)\n if c is NotImplemented:\n return c\n\n return c >= 0\n\n klass.__lt__ = __lt__\n klass.__gt__ = __gt__\n klass.__le__ = __le__\n klass.__ge__ = __ge__\n klass.__eq__ = __eq__\n klass.__ne__ = __ne__\n return klass", "response": "A class decorator that ensures that the objects in the class are comparable."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef validate_arguments_type_of_function(param_type=None):\n\n def inner(function):\n\n def wrapper(self, *args, **kwargs):\n type_ = param_type or type(self)\n for arg in args + tuple(kwargs.values()):\n if not isinstance(arg, type_):\n raise TypeError(\n (\n 'Invalid Type: {}.{}() accepts only the '\n 'arguments of type \"<{}>\"'\n ).format(\n type(self).__name__,\n function.__name__,\n type_.__name__,\n )\n )\n\n return function(self, *args, **kwargs)\n\n return wrapper\n\n return inner", "response": "Decorator to validate the arguments of a function of the specified type."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef utc_offset(time_struct=None):\n if time_struct:\n ts = time_struct\n else:\n ts = time.localtime()\n\n if ts[-1]:\n offset = time.altzone\n else:\n offset = time.timezone\n return offset", "response": "Returns the time offset from UTC accounting for DST\nWorkItem"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning a MayaDT instance for the human moment specified.", "response": "def when(string, timezone='UTC', prefer_dates_from='current_period'):\n \"\"\"\"Returns a MayaDT instance for the human moment specified.\n\n Powered by dateparser. Useful for scraping websites.\n\n Examples:\n 'next week', 'now', 'tomorrow', '300 years ago', 'August 14, 2015'\n\n Keyword Arguments:\n string -- string to be parsed\n timezone -- timezone referenced from (default: 'UTC')\n prefer_dates_from -- what dates are prefered when `string` is ambigous.\n options are 'past', 'future', and 'current_period'\n (default: 'current_period'). see: [1]\n\n Reference:\n [1] dateparser.readthedocs.io/en/latest/usage.html#handling-incomplete-dates\n \"\"\"\n settings = {\n 'TIMEZONE': timezone,\n 'RETURN_AS_TIMEZONE_AWARE': True,\n 'TO_TIMEZONE': 'UTC',\n 'PREFER_DATES_FROM': prefer_dates_from,\n }\n\n dt = dateparser.parse(string, settings=settings)\n if dt is None:\n raise ValueError('invalid datetime input specified.')\n\n return MayaDT.from_datetime(dt)"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef parse(string, timezone='UTC', day_first=False, year_first=True, strict=False):\n options = {}\n options['tz'] = timezone\n options['day_first'] = day_first\n options['year_first'] = year_first\n options['strict'] = strict\n\n dt = pendulum.parse(str(string), **options)\n return MayaDT.from_datetime(dt)", "response": "Returns a MayaDT instance for the machine - produced moment specified."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _seconds_or_timedelta(duration):\n if isinstance(duration, int):\n dt_timedelta = timedelta(seconds=duration)\n elif isinstance(duration, timedelta):\n dt_timedelta = duration\n else:\n raise TypeError(\n 'Expects argument as `datetime.timedelta` object '\n 'or seconds in `int` format'\n )\n\n return dt_timedelta", "response": "Returns a datetime. timedelta object for the passed duration."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nyield MayaDT objects between start and end at a given interval.", "response": "def intervals(start, end, interval):\n \"\"\"\n Yields MayaDT objects between the start and end MayaDTs given,\n at a given interval (seconds or timedelta).\n \"\"\"\n interval = _seconds_or_timedelta(interval)\n current_timestamp = start\n while current_timestamp.epoch < end.epoch:\n yield current_timestamp\n\n current_timestamp = current_timestamp.add(\n seconds=interval.total_seconds()\n )"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef add(self, **kwargs):\n return self.from_datetime(\n pendulum.instance(self.datetime()).add(**kwargs)\n )", "response": "Returns a new MayaDT object with the given offsets."} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn a new MayaDT object with the given offsets.", "response": "def subtract(self, **kwargs):\n \"\"\"Returns a new MayaDT object with the given offsets.\"\"\"\n return self.from_datetime(\n pendulum.instance(self.datetime()).subtract(**kwargs)\n )"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef snap(self, instruction):\n return self.from_datetime(snaptime.snap(self.datetime(), instruction))", "response": "Returns a new MayaDT object modified by the given instruction."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning the name of the local timezone.", "response": "def local_timezone(self):\n \"\"\"Returns the name of the local timezone.\"\"\"\n if self._local_tz.zone in pytz.all_timezones:\n return self._local_tz.zone\n\n return self.timezone"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef __dt_to_epoch(dt):\n # Assume UTC if no datetime is provided.\n if dt.tzinfo is None:\n dt = dt.replace(tzinfo=pytz.utc)\n epoch_start = Datetime(*MayaDT.__EPOCH_START, tzinfo=pytz.timezone('UTC'))\n return (dt - epoch_start).total_seconds()", "response": "Converts a datetime into an epoch."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns MayaDT instance from a 9 - tuple struct.", "response": "def from_struct(klass, struct, timezone=pytz.UTC):\n \"\"\"Returns MayaDT instance from a 9-tuple struct\n\n It's assumed to be from gmtime().\n \"\"\"\n struct_time = time.mktime(struct) - utc_offset(struct)\n dt = Datetime.fromtimestamp(struct_time, timezone)\n return klass(klass.__dt_to_epoch(dt))"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef datetime(self, to_timezone=None, naive=False):\n if to_timezone:\n dt = self.datetime().astimezone(pytz.timezone(to_timezone))\n else:\n dt = Datetime.utcfromtimestamp(self._epoch)\n dt.replace(tzinfo=self._tz)\n # Strip the timezone info if requested to do so.\n if naive:\n return dt.replace(tzinfo=None)\n\n else:\n if dt.tzinfo is None:\n dt = dt.replace(tzinfo=self._tz)\n return dt", "response": "Returns a timezone - aware datetime..."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef iso8601(self):\n # Get a timezone-naive datetime.\n dt = self.datetime(naive=True)\n return '{}Z'.format(dt.isoformat())", "response": "Returns an ISO 8601 representation of the MayaDT."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef slang_date(self, locale=\"en\"):\n dt = pendulum.instance(self.datetime())\n\n try:\n return _translate(dt, locale)\n except KeyError:\n pass\n\n delta = humanize.time.abs_timedelta(\n timedelta(seconds=(self.epoch - now().epoch)))\n\n format_string = \"DD MMM\"\n if delta.days >= 365:\n format_string += \" YYYY\"\n\n return dt.format(format_string, locale=locale).title()", "response": "Returns human slang representation of date."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning human slang representation of time.", "response": "def slang_time(self, locale=\"en\"):\n \"\"\"\"Returns human slang representation of time.\n\n Keyword Arguments:\n locale -- locale to translate to, e.g. 'fr' for french.\n (default: 'en' - English)\n \"\"\"\n dt = self.datetime()\n return pendulum.instance(dt).diff_for_humans(locale=locale)"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef angvel(target, current, scale):\n '''Use sigmoid function to choose a delta that will help smoothly steer from current angle to target angle.'''\n delta = target - current\n while delta < -180:\n delta += 360;\n while delta > 180:\n delta -= 360;\n return (old_div(2.0, (1.0 + math.exp(old_div(-delta,scale))))) - 1.0", "response": "Use sigmoid function to choose a delta that will help smoothly steer from current angle to target angle."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nsteering towards the target pitch / yaw return True when within the given tolerance threshold.", "response": "def pointTo(agent_host, ob, target_pitch, target_yaw, threshold):\n '''Steer towards the target pitch/yaw, return True when within the given tolerance threshold.'''\n pitch = ob.get(u'Pitch', 0)\n yaw = ob.get(u'Yaw', 0)\n delta_yaw = angvel(target_yaw, yaw, 50.0)\n delta_pitch = angvel(target_pitch, pitch, 50.0)\n agent_host.sendCommand(\"turn \" + str(delta_yaw)) \n agent_host.sendCommand(\"pitch \" + str(delta_pitch))\n if abs(pitch-target_pitch) + abs(yaw-target_yaw) < threshold:\n agent_host.sendCommand(\"turn 0\")\n agent_host.sendCommand(\"pitch 0\")\n return True\n return False"} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ndownload Malmo from github and optionaly build the Minecraft Mod.", "response": "def download(branch=None, buildMod=False):\n \"\"\"Download Malmo from github and optionaly build the Minecraft Mod.\n Args:\n branch: optional branch to clone. Default is release version.\n buildMod: don't build the Mod unless build arg is given as True.\n Returns:\n The path for the Malmo Minecraft mod.\n \"\"\"\n gradlew = \"./gradlew\"\n if os.name == 'nt':\n gradlew = \"gradlew.bat\"\n\n if branch is None:\n branch = malmo_version\n\n subprocess.check_call([\"git\", \"clone\", \"-b\", branch, \"https://github.com/Microsoft/malmo.git\" , malmo_install_dir])\n\n os.chdir(malmo_install_dir)\n os.chdir(\"Minecraft\")\n try:\n # Create the version properties file.\n pathlib.Path(\"src/main/resources/version.properties\").write_text(\"malmomod.version={}\\n\".format(malmo_version))\n\n # Optionally do a test build.\n if buildMod:\n subprocess.check_call([gradlew, \"setupDecompWorkspace\", \"build\", \"testClasses\", \"-x\", \"test\", \"--stacktrace\", \"-Pversion={}\"\n .format(malmo_version)])\n\n minecraft_dir = os.getcwd()\n finally:\n os.chdir(\"../..\")\n\n if \"MALMO_XSD_PATH\" not in os.environ:\n print(\"Please make sure you set the MALMO_XSD_PATH environment variable to \\\"{}/Schemas\\\"!\"\n .format(str(pathlib.Path(malmo_install_dir).absolute())))\n return minecraft_dir"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nlaunching Malmo Minecraft Mod in one or more clients from the Minecraft directory on the given ports.", "response": "def launch_minecraft(ports = [], wait_timeout = 360):\n \"\"\"Launch Malmo Minecraft Mod in one or more clients from \n the Minecraft directory on the (optionally) given ports.\n Args:\n ports: an optionsl list of ports to start minecraft clients on. \n Defaults to a single Minecraft client on port 10000.\n wait_timeout: optional time in seconds to wait (defaults to 3 mins).\n \"\"\"\n if \"MALMO_XSD_PATH\" not in os.environ:\n print(\"Please set the MALMO_XSD_PATH environment variable.\")\n return\n cwd = os.getcwd()\n try:\n os.chdir(malmo_install_dir + \"/Minecraft\")\n launch_minecraft_in_background(os.getcwd(), ports, wait_timeout)\n finally:\n os.chdir(cwd)"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nsetting the MAMLMO_XSD_PATH environment variable in current process.", "response": "def set_malmo_xsd_path():\n \"\"\"Set the MAMLMO_XSD_PATH environment variable in current process.\"\"\"\n \n os.environ[\"MALMO_XSD_PATH\"] = str(pathlib.Path(malmo_install_dir + \"/Schemas\").absolute())\n print(os.environ[\"MALMO_XSD_PATH\"])"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns the index in arr of the closest float value to val.", "response": "def indexOfClosest( arr, val ):\n '''Return the index in arr of the closest float value to val.'''\n i_closest = None\n for i,v in enumerate(arr):\n d = math.fabs( v - val )\n if i_closest == None or d < d_closest:\n i_closest = i\n d_closest = d\n return i_closest"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nwaiting for an observation of the world and a frame.", "response": "def waitForInitialState( self ):\n '''Before a command has been sent we wait for an observation of the world and a frame.'''\n # wait for a valid observation\n world_state = self.agent_host.peekWorldState()\n while world_state.is_mission_running and all(e.text=='{}' for e in world_state.observations):\n world_state = self.agent_host.peekWorldState()\n # wait for a frame to arrive after that\n num_frames_seen = world_state.number_of_video_frames_since_last_state\n while world_state.is_mission_running and world_state.number_of_video_frames_since_last_state == num_frames_seen:\n world_state = self.agent_host.peekWorldState()\n world_state = self.agent_host.getWorldState()\n\n if world_state.is_mission_running:\n \n assert len(world_state.video_frames) > 0, 'No video frames!?'\n \n obs = json.loads( world_state.observations[-1].text )\n self.prev_x = obs[u'XPos']\n self.prev_y = obs[u'YPos']\n self.prev_z = obs[u'ZPos']\n self.prev_yaw = obs[u'Yaw']\n print('Initial position:',self.prev_x,',',self.prev_y,',',self.prev_z,'yaw',self.prev_yaw)\n \n if save_images:\n # save the frame, for debugging\n frame = world_state.video_frames[-1]\n image = Image.frombytes('RGB', (frame.width, frame.height), bytes(frame.pixels) )\n self.iFrame = 0\n self.rep = self.rep + 1\n image.save( 'rep_' + str(self.rep).zfill(3) + '_saved_frame_' + str(self.iFrame).zfill(4) + '.png' )\n \n return world_state"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nwaiting for the next state of the agent.", "response": "def waitForNextState( self ):\n '''After each command has been sent we wait for the observation to change as expected and a frame.'''\n # wait for the observation position to have changed\n print('Waiting for observation...', end=' ')\n while True:\n world_state = self.agent_host.peekWorldState()\n if not world_state.is_mission_running:\n print('mission ended.')\n break\n if not all(e.text=='{}' for e in world_state.observations):\n obs = json.loads( world_state.observations[-1].text )\n self.curr_x = obs[u'XPos']\n self.curr_y = obs[u'YPos']\n self.curr_z = obs[u'ZPos']\n self.curr_yaw = obs[u'Yaw']\n if self.require_move:\n if math.fabs( self.curr_x - self.prev_x ) > self.tolerance or\\\n math.fabs( self.curr_y - self.prev_y ) > self.tolerance or\\\n math.fabs( self.curr_z - self.prev_z ) > self.tolerance:\n print('received a move.')\n break\n elif self.require_yaw_change:\n if math.fabs( self.curr_yaw - self.prev_yaw ) > self.tolerance:\n print('received a turn.')\n break\n else:\n print('received.')\n break\n # wait for the render position to have changed\n print('Waiting for render...', end=' ')\n while True:\n world_state = self.agent_host.peekWorldState()\n if not world_state.is_mission_running:\n print('mission ended.')\n break\n if len(world_state.video_frames) > 0:\n frame = world_state.video_frames[-1]\n curr_x_from_render = frame.xPos\n curr_y_from_render = frame.yPos\n curr_z_from_render = frame.zPos\n curr_yaw_from_render = frame.yaw\n if self.require_move:\n if math.fabs( curr_x_from_render - self.prev_x ) > self.tolerance or\\\n math.fabs( curr_y_from_render - self.prev_y ) > self.tolerance or\\\n math.fabs( curr_z_from_render - self.prev_z ) > self.tolerance:\n print('received a move.')\n break\n elif self.require_yaw_change:\n if math.fabs( curr_yaw_from_render - self.prev_yaw ) > self.tolerance:\n print('received a turn.')\n break\n else:\n print('received.')\n break\n \n num_frames_before_get = len(world_state.video_frames)\n world_state = self.agent_host.getWorldState()\n\n if save_images:\n # save the frame, for debugging\n if world_state.is_mission_running:\n assert len(world_state.video_frames) > 0, 'No video frames!?'\n frame = world_state.video_frames[-1]\n image = Image.frombytes('RGB', (frame.width, frame.height), bytes(frame.pixels) )\n self.iFrame = self.iFrame + 1\n image.save( 'rep_' + str(self.rep).zfill(3) + '_saved_frame_' + str(self.iFrame).zfill(4) + '.png' )\n \n if world_state.is_mission_running:\n assert len(world_state.video_frames) > 0, 'No video frames!?'\n num_frames_after_get = len(world_state.video_frames)\n assert num_frames_after_get >= num_frames_before_get, 'Fewer frames after getWorldState!?'\n frame = world_state.video_frames[-1]\n obs = json.loads( world_state.observations[-1].text )\n self.curr_x = obs[u'XPos']\n self.curr_y = obs[u'YPos']\n self.curr_z = obs[u'ZPos']\n self.curr_yaw = obs[u'Yaw']\n print('New position from observation:',self.curr_x,',',self.curr_y,',',self.curr_z,'yaw',self.curr_yaw, end=' ')\n if math.fabs( self.curr_x - self.expected_x ) > self.tolerance or\\\n math.fabs( self.curr_y - self.expected_y ) > self.tolerance or\\\n math.fabs( self.curr_z - self.expected_z ) > self.tolerance or\\\n math.fabs( self.curr_yaw - self.expected_yaw ) > self.tolerance:\n print(' - ERROR DETECTED! Expected:',self.expected_x,',',self.expected_y,',',self.expected_z,'yaw',self.expected_yaw)\n exit(1)\n else:\n print('as expected.')\n curr_x_from_render = frame.xPos\n curr_y_from_render = frame.yPos\n curr_z_from_render = frame.zPos\n curr_yaw_from_render = frame.yaw\n print('New position from render:',curr_x_from_render,',',curr_y_from_render,',',curr_z_from_render,'yaw',curr_yaw_from_render, end=' ')\n if math.fabs( curr_x_from_render - self.expected_x ) > self.tolerance or\\\n math.fabs( curr_y_from_render - self.expected_y ) > self.tolerance or \\\n math.fabs( curr_z_from_render - self.expected_z ) > self.tolerance or \\\n math.fabs( curr_yaw_from_render - self.expected_yaw ) > self.tolerance:\n print(' - ERROR DETECTED! Expected:',self.expected_x,',',self.expected_y,',',self.expected_z,'yaw',self.expected_yaw)\n exit(1)\n else:\n print('as expected.')\n self.prev_x = self.curr_x\n self.prev_y = self.curr_y\n self.prev_z = self.curr_z\n self.prev_yaw = self.curr_yaw\n \n return world_state"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef act( self ):\n '''Take an action from the action_set and set the expected outcome so we can wait for it.'''\n if self.action_set == 'discrete_absolute':\n actions = ['movenorth 1', 'movesouth 1', 'movewest 1', 'moveeast 1']\n elif self.action_set == 'discrete_relative':\n actions = ['move 1', 'move -1', 'turn 1', 'turn -1']\n elif self.action_set == 'teleport':\n while True:\n dx = random.randint(-10,10)\n dz = random.randint(-10,10)\n if not dx == 0 or not dz == 0:\n break\n self.expected_x = self.prev_x + dx\n self.expected_y = self.prev_y\n self.expected_z = self.prev_z + dz\n self.expected_yaw = self.prev_yaw\n self.require_move = True\n self.require_yaw_change = False\n actions = ['tp '+str(self.expected_x)+' '+str(self.expected_y)+' '+str(self.expected_z)]\n else:\n print('ERROR: Unsupported action set:',self.action_set)\n exit(1)\n i_action = random.randint(0,len(actions)-1)\n action = actions[ i_action ]\n print('Sending',action)\n self.agent_host.sendCommand( action )\n if self.action_set == 'discrete_absolute':\n self.expected_x = self.prev_x + [0,0,-1,1][i_action]\n self.expected_y = self.prev_y\n self.expected_z = self.prev_z + [-1,1,0,0][i_action]\n self.expected_yaw = self.prev_yaw\n self.require_move = True\n self.require_yaw_change = False\n elif self.action_set == 'discrete_relative':\n if i_action == 0 or i_action == 1:\n i_yaw = indexOfClosest( [0,90,180,270], self.prev_yaw )\n forward = [ (0,1), (-1,0), (0,-1), (1,0) ][ i_yaw ]\n if i_action == 0:\n self.expected_x = self.prev_x + forward[0]\n self.expected_z = self.prev_z + forward[1]\n else:\n self.expected_x = self.prev_x - forward[0]\n self.expected_z = self.prev_z - forward[1]\n self.expected_y = self.prev_y\n self.expected_yaw = self.prev_yaw\n self.require_move = True\n self.require_yaw_change = False\n else:\n self.expected_x = self.prev_x\n self.expected_y = self.prev_y\n self.expected_z = self.prev_z\n self.expected_yaw = math.fmod( 360 + self.prev_yaw + [90,-90][i_action-2], 360 )\n self.require_move = False\n self.require_yaw_change = True\n elif self.action_set == 'teleport':\n pass\n else:\n print('ERROR: Unsupported action set:',self.action_set)\n exit(1)", "response": "Take an action from the action_set and set the expected outcome so we can wait for it."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef processFrame(frame):\n # The pixels are grey-scale values from 0-255 - white = 255, black = 0\n # We want to turn the image into two values, one for the left \"sensor\" and one for the right.\n # There are many possible ways to do this. The very simplest way would be\n # to examine the values of two pixels - one in the centre of the left half of the image,\n # and one in the centre of the right half. What are the obvious problems with this?\n\n # NB: The pixels are stored in a flat (1 dimensional) array, so to calculate the index of the pixels\n # at (x,y), we do:\n # index = x + (y * width)\n\n width = frame.width\n height = frame.height\n pixels = frame.pixels\n\n USE_SIMPLE_APPROACH = False # Change this to False to use more sophisticated approach.\n\n if USE_SIMPLE_APPROACH:\n # Simplest approach.\n left_centre_x = int(width / 4) # Centre of left half\n right_centre_x = int(3 * width / 4) # Centre of right half\n centre_y = int(height / 2) # Middle row\n left_pixel_index = left_centre_x + (centre_y * width)\n right_pixel_index = right_centre_x + (centre_y * width)\n left_sensor = float(pixels[left_pixel_index]) / 255.0\n right_sensor = float(pixels[right_pixel_index]) / 255.0\n else:\n # Obviously, the simple approach misses a lot of data, and is very susceptible to noise.\n # A better approach would be to consider more pixels.\n # You could take the average of all the pixels - it's slower but better.\n # Or you could use the median value, which is less noisy than the mean.\n # Here we calculate both, and estimate the median using histograms:\n left_total = 0\n right_total = 0\n left_hist = [0 for i in range(256)]\n right_hist = [0 for i in range(256)]\n # Create a histogram for each half of the image:\n for y in range(height):\n for x in range(int(width/2)):\n i = pixels[x + y*width]\n left_hist[i] += 1\n left_total += float(i)/255.0\n for x in range(int(width/2), width):\n i = pixels[x + y*width]\n right_hist[i] += 1\n right_total += float(i)/255.0\n # Calculate the mean values:\n left_mean, right_mean = left_total / (width*height/2), right_total / (width*height/2)\n # Now use the histogram to estimate the median value\n left_total, right_total = 0, 0\n pixels_per_half = width * height / 2\n cut_off_value = pixels_per_half / 2\n left_cut_off_point, right_cut_off_point = 0, 0\n while (left_total < cut_off_value):\n left_total += left_hist[left_cut_off_point]\n left_cut_off_point += 1\n while(right_total < cut_off_value):\n right_total += right_hist[right_cut_off_point]\n right_cut_off_point += 1\n left_median, right_median = left_cut_off_point / 255.0, right_cut_off_point / 255.0\n \n # Use the median values:\n left_sensor, right_sensor = left_median, right_median\n # Or uncomment this line to use the mean values:\n # left_sensor, right_sensor = left_mean, right_mean\n \n # In our gloomy arena, we never get particularly bright, so the sensor values tend to be low.\n # To get more action from our vehicle, we can scale up (keeping 1.0 as the max).\n # What values work well for GAIN? What happens if it's too high or too low?\n GAIN = 1.8\n left_sensor = min(1, left_sensor * GAIN)\n right_sensor = min(1, right_sensor * GAIN)\n # Done - return the values:\n return left_sensor, right_sensor", "response": "Simulate a left and right sensor output from the input image."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef getCitiesDrawingXML(points):\n ''' Build an XML string that contains a square for each city'''\n xml = \"\"\n for p in points:\n x = str(p.x)\n z = str(p.y)\n xml += ''\n xml += ''\n return xml", "response": "Build an XML string that contains a square for each city"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nsetting a mission running.", "response": "def runMission( self, mission_spec, mission_record_spec, role = 0 ):\n '''Sets a mission running.\n \n Parameters:\n mission_spec : MissionSpec instance, specifying the mission.\n mission_record_spec : MissionRecordSpec instance, specifying what should be recorded.\n role : int, the index of the role this human agent is to play. Zero based.\n '''\n \n self.world_state = None\n total_reward = 0\n \n # decide on the action space\n command_handlers = mission_spec.getListOfCommandHandlers(role)\n if 'ContinuousMovement' in command_handlers and 'DiscreteMovement' in command_handlers:\n print('ERROR: Ambiguous action space in supplied mission: both continuous and discrete command handlers present.')\n exit(1)\n elif 'ContinuousMovement' in command_handlers:\n self.action_space = 'continuous'\n elif 'DiscreteMovement' in command_handlers:\n self.action_space = 'discrete'\n else:\n print('ERROR: Unknown action space in supplied mission: neither continuous or discrete command handlers present.')\n exit(1)\n\n self.createGUI()\n \n if mission_spec.isVideoRequested(0):\n self.canvas.config( width=mission_spec.getVideoWidth(0), height=mission_spec.getVideoHeight(0) )\n\n # show the mission summary\n start_time = time.time()\n while time.time() - start_time < 4:\n canvas_id = self.canvas.create_rectangle(100, 100, 540, 200, fill=\"white\", outline=\"red\", width=\"5\")\n self.canvas.create_text(320, 120, text=mission_spec.getSummary(), font=('Helvetica', '16'))\n self.canvas.create_text(320, 150, text=str(3 - int(time.time() - start_time)), font=('Helvetica', '16'), fill=\"red\")\n self.root.update()\n time.sleep(0.2)\n \n try:\n self.agent_host.startMission( mission_spec, mission_record_spec )\n except RuntimeError as e:\n tkinter.messagebox.showerror(\"Error\",\"Error starting mission: \"+str(e))\n return\n\n print(\"Waiting for the mission to start\", end=' ')\n self.world_state = self.agent_host.peekWorldState()\n while not self.world_state.has_mission_begun:\n print(\".\", end=\"\")\n time.sleep(0.1)\n self.world_state = self.agent_host.peekWorldState()\n for error in self.world_state.errors:\n print(\"Error:\",error.text)\n print()\n if self.action_space == 'continuous':\n self.canvas.config(cursor='none') # hide the mouse cursor while over the canvas\n self.canvas.event_generate('', warp=True, x=old_div(self.canvas.winfo_width(),2), y=old_div(self.canvas.winfo_height(),2)) # put cursor at center\n self.root.after(50, self.update)\n self.canvas.focus_set()\n\n while self.world_state.is_mission_running:\n self.world_state = self.agent_host.getWorldState()\n if self.world_state.number_of_observations_since_last_state > 0:\n self.observation.config(text = self.world_state.observations[0].text )\n if mission_spec.isVideoRequested(0) and self.world_state.number_of_video_frames_since_last_state > 0:\n frame = self.world_state.video_frames[-1]\n image = Image.frombytes('RGB', (frame.width,frame.height), bytes(frame.pixels) )\n photo = ImageTk.PhotoImage(image)\n self.canvas.delete(\"all\")\n self.canvas.create_image(old_div(frame.width,2), old_div(frame.height,2), image=photo)\n self.canvas.create_line( old_div(self.canvas.winfo_width(),2)-5, old_div(self.canvas.winfo_height(),2), old_div(self.canvas.winfo_width(),2)+6, old_div(self.canvas.winfo_height(),2), fill='white' )\n self.canvas.create_line( old_div(self.canvas.winfo_width(),2), old_div(self.canvas.winfo_height(),2)-5, old_div(self.canvas.winfo_width(),2), old_div(self.canvas.winfo_height(),2)+6, fill='white' )\n # parse reward\n for reward in self.world_state.rewards:\n total_reward += reward.getValue()\n self.reward.config(text = str(total_reward) )\n self.root.update()\n time.sleep(0.01)\n if self.action_space == 'continuous':\n self.canvas.config(cursor='arrow') # restore the mouse cursor\n print('Mission stopped')\n if not self.agent_host.receivedArgument(\"test\"):\n tkinter.messagebox.showinfo(\"Mission ended\",\"Mission has ended. Total reward: \" + str(total_reward) )\n self.root_frame.destroy()"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncreating the graphical user interface.", "response": "def createGUI( self ):\n '''Create the graphical user interface.'''\n our_font = \"Helvetica 16 bold\"\n small_font = \"Helvetica 9 bold\"\n self.root_frame = Frame(self.root)\n if self.action_space == 'continuous':\n desc = \"Running continuous-action mission.\\nUse the mouse to turn, WASD to move.\"\n else:\n desc = \"Running discrete-action mission.\\nUse the arrow keys to turn and move.\"\n Label(self.root_frame, text=desc,font = our_font,wraplength=640).pack(padx=5, pady=5)\n self.canvas = Canvas(self.root_frame, borderwidth=0, highlightthickness=0, width=640, height=480, bg=\"gray\" )\n self.canvas.bind('',self.onMouseMoveInCanvas)\n self.canvas.bind('',self.onLeftMouseDownInCanvas)\n self.canvas.bind('',self.onLeftMouseUpInCanvas)\n if sys.platform == 'darwin': right_mouse_button = '2' # on MacOSX, the right button is 'Button-2'\n else: right_mouse_button = '3' # on Windows and Linux the right button is 'Button-3'\n self.canvas.bind('',self.onRightMouseDownInCanvas)\n self.canvas.bind('',self.onRightMouseUpInCanvas)\n self.canvas.bind('',self.onKeyPressInCanvas)\n self.canvas.bind('',self.onKeyReleaseInCanvas)\n self.canvas.pack(padx=5, pady=5)\n self.entry_frame = Frame(self.root_frame)\n Label(self.entry_frame, text=\"Type '/' to enter command:\",font = small_font).pack(padx=5, pady=5, side=LEFT)\n self.command_entry = Entry(self.entry_frame,font = small_font)\n self.command_entry.bind('',self.onKeyInCommandEntry)\n self.command_entry.pack(padx=5, pady=5, side=LEFT)\n Button(self.entry_frame, text='Send', command=self.onSendCommand,font = small_font).pack(padx=5, pady=5, side=LEFT)\n self.entry_frame.pack()\n self.observation = Label(self.root_frame, text='observations will appear here', wraplength=640, font = small_font)\n self.observation.pack()\n self.reward = Label(self.root_frame, text='rewards will appear here', wraplength=640, font = small_font)\n self.reward.pack()\n self.root_frame.pack()\n self.mouse_event = self.prev_mouse_event = None"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncalling when the user presses the send button or presses Enter while the command entry box has focus.", "response": "def onSendCommand(self):\n '''Called when user presses the 'send' button or presses 'Enter' while the command entry box has focus.'''\n self.agent_host.sendCommand(self.command_entry.get())\n self.command_entry.delete(0,END)\n self.canvas.focus_set()"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef update(self):\n '''Called at regular intervals to poll the mouse position to send continuous commands.'''\n if self.action_space == 'continuous': # mouse movement only used for continuous action space\n if self.world_state and self.world_state.is_mission_running:\n if self.mouse_event and self.prev_mouse_event:\n rotation_speed = 0.1\n turn_speed = ( self.mouse_event.x - self.prev_mouse_event.x ) * rotation_speed\n pitch_speed = ( self.mouse_event.y - self.prev_mouse_event.y ) * rotation_speed\n self.agent_host.sendCommand( 'turn '+str(turn_speed) )\n self.agent_host.sendCommand( 'pitch '+str(pitch_speed) )\n if self.mouse_event:\n if os.name == 'nt': # (moving the mouse cursor only seems to work on Windows)\n self.canvas.event_generate('', warp=True, x=old_div(self.canvas.winfo_width(),2), y=old_div(self.canvas.winfo_height(),2)) # put cursor at center\n self.mouse_event.x = old_div(self.canvas.winfo_width(),2)\n self.mouse_event.y = old_div(self.canvas.winfo_height(),2)\n self.prev_mouse_event = self.mouse_event\n if self.world_state.is_mission_running:\n self.root.after(50, self.update)", "response": "Called at regular intervals to poll the mouse position to send continuous commands."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef onKeyInCommandEntry(self, event):\n '''Called when a key is pressed when the command entry box has focus.'''\n if event.char == '\\r':\n self.onSendCommand()\n self.canvas.focus_set()", "response": "Called when a key is pressed when the command entry box has focus."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef onKeyPressInCanvas(self, event):\n '''Called when a key is pressed when the canvas has focus.'''\n char_map = { 'w':'move 1', 'a':'strafe -1', 's':'move -1', 'd':'strafe 1', ' ':'jump 1' }\n keysym_map = { 'continuous': { 'Left':'turn -1', 'Right':'turn 1', 'Up':'pitch -1', 'Down':'pitch 1', 'Shift_L':'crouch 1',\n 'Shift_R':'crouch 1', \n '1':'hotbar.1 1', '2':'hotbar.2 1', '3':'hotbar.3 1', '4':'hotbar.4 1', '5':'hotbar.5 1',\n '6':'hotbar.6 1', '7':'hotbar.7 1', '8':'hotbar.8 1', '9':'hotbar.9 1' },\n 'discrete': { 'Left':'turn -1', 'Right':'turn 1', 'Up':'move 1', 'Down':'move -1', \n '1':'hotbar.1 1', '2':'hotbar.2 1', '3':'hotbar.3 1', '4':'hotbar.4 1', '5':'hotbar.5 1',\n '6':'hotbar.6 1', '7':'hotbar.7 1', '8':'hotbar.8 1', '9':'hotbar.9 1' } }\n if event.char == '/':\n self.command_entry.focus_set() # interlude to allow user to type command\n elif event.char.lower() in char_map:\n self.agent_host.sendCommand( char_map[ event.char.lower() ] )\n elif event.keysym in keysym_map[self.action_space]:\n self.agent_host.sendCommand( keysym_map[self.action_space][ event.keysym ] )", "response": "Called when a key is pressed when the canvas has focus."} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncalls when a key is released when the command entry box has focus.", "response": "def onKeyReleaseInCanvas(self, event):\n '''Called when a key is released when the command entry box has focus.'''\n char_map = { 'w':'move 0', 'a':'strafe 0', 's':'move 0', 'd':'strafe 0', ' ':'jump 0' }\n keysym_map = { 'Left':'turn 0', 'Right':'turn 0', 'Up':'pitch 0', 'Down':'pitch 0', 'Shift_L':'crouch 0', 'Shift_R':'crouch 0', \n '1':'hotbar.1 0', '2':'hotbar.2 0', '3':'hotbar.3 0', '4':'hotbar.4 0', '5':'hotbar.5 0',\n '6':'hotbar.6 0', '7':'hotbar.7 0', '8':'hotbar.8 0', '9':'hotbar.9 0' }\n if event.char.lower() in char_map:\n self.agent_host.sendCommand( char_map[ event.char.lower() ] )\n elif event.keysym in keysym_map:\n self.agent_host.sendCommand( keysym_map[ event.keysym ] )"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef getItemXML():\n ''' Build an XML string that contains some randomly positioned goal items'''\n xml=\"\"\n for item in range(NUM_GOALS):\n x = str(random.randint(old_div(-ARENA_WIDTH,2),old_div(ARENA_WIDTH,2)))\n z = str(random.randint(old_div(-ARENA_BREADTH,2),old_div(ARENA_BREADTH,2)))\n xml += ''''''\n return xml", "response": "Build an XML string that contains some randomly positioned goal items"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn part of the XML string that defines the requested corner", "response": "def getCorner(index,top,left,expand=0,y=206):\n ''' Return part of the XML string that defines the requested corner'''\n x = str(-(expand+old_div(ARENA_WIDTH,2))) if left else str(expand+old_div(ARENA_WIDTH,2))\n z = str(-(expand+old_div(ARENA_BREADTH,2))) if top else str(expand+old_div(ARENA_BREADTH,2))\n return 'x'+index+'=\"'+x+'\" y'+index+'=\"' +str(y)+'\" z'+index+'=\"'+z+'\"'"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef getMissionXML(summary):\n ''' Build an XML mission string.'''\n spawn_end_tag = ' type=\"mob_spawner\" variant=\"' + MOB_TYPE + '\"/>'\n return '''\n \n \n ''' + summary + '''\n \n\n \n 20\n \n \n \n \n true\n ''' + MOB_TYPE + '''\n \n \n \n \n \n \n\n \n \n \n \n\n \n The Hunted\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n ''' + malmoutils.get_video_xml(agent_host) + '''\n \n \n\n '''", "response": "Build an XML mission string."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nscanning through 360 degrees looking for the best direction in which to take the next step.", "response": "def getBestAngle(entities, current_yaw, current_health):\n '''Scan through 360 degrees, looking for the best direction in which to take the next step.'''\n us = findUs(entities)\n scores=[]\n # Normalise current yaw:\n while current_yaw < 0:\n current_yaw += 360\n while current_yaw > 360:\n current_yaw -= 360\n\n # Look for best option\n for i in range(agent_search_resolution):\n # Calculate cost of turning:\n ang = 2 * math.pi * (old_div(i, float(agent_search_resolution)))\n yaw = i * 360.0 / float(agent_search_resolution)\n yawdist = min(abs(yaw-current_yaw), 360-abs(yaw-current_yaw))\n turncost = agent_turn_weight * yawdist\n score = turncost\n\n # Calculate entity proximity cost for new (x,z):\n x = us[\"x\"] + agent_stepsize - math.sin(ang)\n z = us[\"z\"] + agent_stepsize * math.cos(ang)\n for ent in entities:\n dist = (ent[\"x\"] - x)*(ent[\"x\"] - x) + (ent[\"z\"] - z)*(ent[\"z\"] - z)\n if (dist == 0):\n continue\n weight = 0.0\n if ent[\"name\"] == MOB_TYPE:\n weight = agent_mob_weight\n dist -= 1 # assume mobs are moving towards us\n if dist <= 0:\n dist = 0.1\n elif ent[\"name\"] == GOAL_TYPE:\n weight = agent_goal_weight * current_health / 20.0\n score += old_div(weight, float(dist))\n\n # Calculate cost of proximity to edges:\n distRight = (2+old_div(ARENA_WIDTH,2)) - x\n distLeft = (-2-old_div(ARENA_WIDTH,2)) - x\n distTop = (2+old_div(ARENA_BREADTH,2)) - z\n distBottom = (-2-old_div(ARENA_BREADTH,2)) - z\n score += old_div(agent_edge_weight, float(distRight * distRight * distRight * distRight))\n score += old_div(agent_edge_weight, float(distLeft * distLeft * distLeft * distLeft))\n score += old_div(agent_edge_weight, float(distTop * distTop * distTop * distTop))\n score += old_div(agent_edge_weight, float(distBottom * distBottom * distBottom * distBottom))\n scores.append(score)\n\n # Find best score:\n i = scores.index(max(scores))\n # Return as an angle in degrees:\n return i * 360.0 / float(agent_search_resolution)"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef restart_minecraft(world_state, agent_host, client_info, message):\n if world_state.is_mission_running:\n agent_host.sendCommand(\"quit\")\n time.sleep(10)\n agent_host.killClient(client_info)\n raise MissionTimeoutException(message)", "response": "Attempt to quit mission if running and kill the client"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef init(self, xml, port, server=None,\n server2=None, port2=None,\n role=0, exp_uid=None, episode=0,\n action_filter=None, resync=0, step_options=0, action_space=None):\n \"\"\"\"Initialize a Malmo environment.\n xml - the mission xml.\n port - the MalmoEnv service's port.\n server - the MalmoEnv service address. Default is localhost.\n server2 - the MalmoEnv service address for given role if not 0.\n port2 - the MalmoEnv service port for given role if not 0.\n role - the agent role (0..N-1) for missions with N agents. Defaults to 0.\n exp_uid - the experiment's unique identifier. Generated if not given.\n episode - the \"reset\" start count for experiment re-starts. Defaults to 0.\n action_filter - an optional list of valid actions to filter by. Defaults to simple commands.\n step_options - encodes withTurnKey and withInfo in step messages. Defaults to info included,\n turn if required.\n \"\"\"\n if action_filter is None:\n action_filter = {\"move\", \"turn\", \"use\", \"attack\"}\n\n if not xml.startswith(' tag.\")\n xml = xml[i:]\n\n self.xml = etree.fromstring(xml)\n self.role = role\n if exp_uid is None:\n self.exp_uid = str(uuid.uuid4())\n else:\n self.exp_uid = exp_uid\n\n command_parser = CommandParser(action_filter)\n commands = command_parser.get_commands_from_xml(self.xml, self.role)\n actions = command_parser.get_actions(commands)\n # print(\"role \" + str(self.role) + \" actions \" + str(actions)\n\n if action_space:\n self.action_space = action_space\n else:\n self.action_space = ActionSpace(actions)\n\n self.port = port\n if server is not None:\n self.server = server\n if server2 is not None:\n self.server2 = server2\n else:\n self.server2 = self.server\n if port2 is not None:\n self.port2 = port2\n else:\n self.port2 = self.port + self.role\n\n self.agent_count = len(self.xml.findall(self.ns + 'AgentSection'))\n turn_based = self.xml.find('.//' + self.ns + 'TurnBasedCommands') is not None\n if turn_based:\n self.turn_key = 'AKWozEre'\n else:\n self.turn_key = \"\"\n if step_options is None:\n self.step_options = 0 if not turn_based else 2\n else:\n self.step_options = step_options\n self.done = True\n # print(\"agent count \" + str(self.agent_count) + \" turn based \" + turn_based)\n self.resync_period = resync\n self.resets = episode\n\n e = etree.fromstring(\"\"\"\n \n 0\n \n 127.0.0.1\n 0\n 0\n 127.0.0.1\n 0\n 0\n 0\n 0\n 0\n 0\n 0\n \n \"\"\")\n e.insert(0, self.xml)\n self.xml = e\n self.xml.find(self.ns + 'ClientRole').text = str(self.role)\n self.xml.find(self.ns + 'ExperimentUID').text = self.exp_uid\n if self.role != 0 and self.agent_count > 1:\n e = etree.Element(self.ns + 'MinecraftServerConnection',\n attrib={'address': self.server,\n 'port': str(0)\n })\n self.xml.insert(2, e)\n\n video_producers = self.xml.findall('.//' + self.ns + 'VideoProducer')\n assert len(video_producers) == self.agent_count\n video_producer = video_producers[self.role]\n self.width = int(video_producer.find(self.ns + 'Width').text)\n self.height = int(video_producer.find(self.ns + 'Height').text)\n want_depth = video_producer.attrib[\"want_depth\"]\n self.depth = 4 if want_depth is not None and (want_depth == \"true\" or want_depth == \"1\") else 3\n # print(str(self.width) + \"x\" + str(self.height) + \"x\" + str(self.depth))\n self.observation_space = VisualObservationSpace(self.width, self.height, self.depth)", "response": "Initialize a Malmo environment."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nuses carefully to reset the episode count to 0.", "response": "def reinit(self):\n \"\"\"Use carefully to reset the episode count to 0.\"\"\"\n sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)\n sock.connect((self.server, self.port))\n self._hello(sock)\n\n comms.send_message(sock, (\"\" + self._get_token() + \"\").encode())\n reply = comms.recv_message(sock)\n sock.close()\n ok, = struct.unpack('!I', reply)\n return ok != 0"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nget the status from the server.", "response": "def status(self, head):\n \"\"\"Get status from server.\n head - Ping the the head node if True.\n \"\"\"\n sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)\n if head:\n sock.connect((self.server, self.port))\n else:\n sock.connect((self.server2, self.port2))\n self._hello(sock)\n\n comms.send_message(sock, \"\".encode())\n status = comms.recv_message(sock).decode('utf-8')\n sock.close()\n return status"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nusing carefully to cause the Minecraft service to exit.", "response": "def exit(self):\n \"\"\"Use carefully to cause the Minecraft service to exit (and hopefully restart).\n Likely to throw communication errors so wrap in exception handler.\n \"\"\"\n sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)\n sock.connect((self.server2, self.port2))\n self._hello(sock)\n\n comms.send_message(sock, (\"\" + self._get_token() + \"\").encode())\n reply = comms.recv_message(sock)\n sock.close()\n ok, = struct.unpack('!I', reply)\n return ok != 0"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef resync(self):\n success = 0\n for head in [True, False]:\n for _ in range(30):\n try:\n self.status(head)\n success += 1\n break\n except Exception as e:\n self._log_error(e)\n time.sleep(10)\n\n if success != 2:\n raise EnvException(\"Failed to contact service\" + (\" head\" if success == 0 else \"\"))", "response": "check if we can ping the head and assigned node."} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nexit the current Minecraft and wait for new one to replace it.", "response": "def exit_resync(self):\n \"\"\"Exit the current Minecraft and wait for new one to replace it.\"\"\"\n print(\"********** exit & resync **********\")\n try:\n if self.client_socket:\n self.client_socket.close()\n self.client_socket = None\n try:\n self.exit()\n except Exception as e:\n self._log_error(e)\n print(\"Pause for exit(s) ...\")\n time.sleep(60)\n except (socket.error, ConnectionError):\n pass\n self.resync()"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef updateQTable( self, reward, current_state ):\n \n # retrieve the old action value from the Q-table (indexed by the previous state and the previous action)\n old_q = self.q_table[self.prev_s][self.prev_a]\n \n # TODO: what should the new action value be?\n new_q = old_q\n \n # assign the new action value to the Q-table\n self.q_table[self.prev_s][self.prev_a] = new_q", "response": "Update the q_table to reflect what we have learnt."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nupdating the q_table to reflect what we have learnt from a terminal state.", "response": "def updateQTableFromTerminatingState( self, reward ):\n \"\"\"Change q_table to reflect what we have learnt, after reaching a terminal state.\"\"\"\n \n # retrieve the old action value from the Q-table (indexed by the previous state and the previous action)\n old_q = self.q_table[self.prev_s][self.prev_a]\n \n # TODO: what should the new action value be?\n new_q = old_q\n \n # assign the new action value to the Q-table\n self.q_table[self.prev_s][self.prev_a] = new_q"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ntake 1 action in response to the current world state", "response": "def act(self, world_state, agent_host, current_r ):\n \"\"\"take 1 action in response to the current world state\"\"\"\n \n obs_text = world_state.observations[-1].text\n obs = json.loads(obs_text) # most recent observation\n self.logger.debug(obs)\n if not u'XPos' in obs or not u'ZPos' in obs:\n self.logger.error(\"Incomplete observation received: %s\" % obs_text)\n return 0\n current_s = \"%d:%d\" % (int(obs[u'XPos']), int(obs[u'ZPos']))\n self.logger.debug(\"State: %s (x = %.2f, z = %.2f)\" % (current_s, float(obs[u'XPos']), float(obs[u'ZPos'])))\n if current_s not in self.q_table:\n self.q_table[current_s] = ([0] * len(self.actions))\n\n # update Q values\n if self.prev_s is not None and self.prev_a is not None:\n self.updateQTable( current_r, current_s )\n\n self.drawQ( curr_x = int(obs[u'XPos']), curr_y = int(obs[u'ZPos']) )\n\n # select the next action\n rnd = random.random()\n if rnd < self.epsilon:\n a = random.randint(0, len(self.actions) - 1)\n self.logger.info(\"Random action: %s\" % self.actions[a])\n else:\n m = max(self.q_table[current_s])\n self.logger.debug(\"Current values: %s\" % \",\".join(str(x) for x in self.q_table[current_s]))\n l = list()\n for x in range(0, len(self.actions)):\n if self.q_table[current_s][x] == m:\n l.append(x)\n y = random.randint(0, len(l)-1)\n a = l[y]\n self.logger.info(\"Taking q action: %s\" % self.actions[a])\n\n # try to send the selected action, only update prev_s if this succeeds\n try:\n agent_host.sendCommand(self.actions[a])\n self.prev_s = current_s\n self.prev_a = a\n\n except RuntimeError as e:\n self.logger.error(\"Failed to send command: %s\" % e)\n\n return current_r"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef run(self, agent_host):\n\n total_reward = 0\n \n self.prev_s = None\n self.prev_a = None\n \n is_first_action = True\n \n # main loop:\n world_state = agent_host.getWorldState()\n while world_state.is_mission_running:\n\n current_r = 0\n \n if is_first_action:\n # wait until have received a valid observation\n while True:\n time.sleep(0.1)\n world_state = agent_host.getWorldState()\n for error in world_state.errors:\n self.logger.error(\"Error: %s\" % error.text)\n for reward in world_state.rewards:\n current_r += reward.getValue()\n if world_state.is_mission_running and len(world_state.observations)>0 and not world_state.observations[-1].text==\"{}\":\n total_reward += self.act(world_state, agent_host, current_r)\n break\n if not world_state.is_mission_running:\n break\n is_first_action = False\n else:\n # wait for non-zero reward\n while world_state.is_mission_running and current_r == 0:\n time.sleep(0.1)\n world_state = agent_host.getWorldState()\n for error in world_state.errors:\n self.logger.error(\"Error: %s\" % error.text)\n for reward in world_state.rewards:\n current_r += reward.getValue()\n # allow time to stabilise after action\n while True:\n time.sleep(0.1)\n world_state = agent_host.getWorldState()\n for error in world_state.errors:\n self.logger.error(\"Error: %s\" % error.text)\n for reward in world_state.rewards:\n current_r += reward.getValue()\n if world_state.is_mission_running and len(world_state.observations)>0 and not world_state.observations[-1].text==\"{}\":\n total_reward += self.act(world_state, agent_host, current_r)\n break\n if not world_state.is_mission_running:\n break\n\n # process final reward\n self.logger.debug(\"Final reward: %d\" % current_r)\n total_reward += current_r\n\n # update Q values\n if self.prev_s is not None and self.prev_a is not None:\n self.updateQTableFromTerminatingState( current_r )\n \n self.drawQ()\n \n return total_reward", "response": "run the agent on the world"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_commands(self, mission_xml, role):\n mission = etree.fromstring(mission_xml)\n return self.get_commands_from_xml(mission, role)", "response": "Get commands from xml string as a list of ( command_type int turnbased boolean command"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_commands_from_file(self, mission_file, role):\n doc = etree.parse(mission_file)\n mission = doc.getroot()\n return self.get_commands_from_xml(mission, role)", "response": "Get commands from xml file as a list of"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ngets commands from xml", "response": "def get_commands_from_xml(self, mission, role):\n \"\"\"Get commands from etree\"\"\"\n handlers = mission.findall(CommandParser.ns + \"AgentSection\" + \"/\" + CommandParser.ns + \"AgentHandlers\")\n if len(handlers) <= role:\n raise CommandHandlerException(\"Not enough agents sections in XML\")\n commands = []\n self._command_hander(handlers[role], False, commands)\n return commands"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nget parameterized actions from command list based on command type and verb.", "response": "def get_actions(self, commands):\n \"\"\"Get parameterized actions from command list based on command type and verb.\"\"\"\n actions = []\n for type, turn_based, verb in commands:\n if len(self.action_filter) != 0 and verb not in self.action_filter:\n continue\n if type == 'DiscreteMovement':\n if verb in {\"move\", \"turn\", \"look\",\n \"strafe\", \"jumpmove\", \"jumpstrafe\"}:\n actions.append(verb + \" 1\")\n actions.append(verb + \" -1\")\n elif verb in {\"jumpeast\", \"jumpnorth\", \"jumpsouth\",\n \"jumpwest\", \"movenorth\", \"moveeast\",\n \"movesouth\", \"movewest\", \"jumpuse\",\n \"use\", \"attack\", \"jump\"}:\n actions.append(verb + \" 1\")\n else:\n raise CommandHandlerException(\"Invalid discrete command\")\n elif type == 'ContinuousMovement':\n # Translate to discrete.\n if verb in {\"move\", \"strafe\", \"pitch\", \"turn\"}:\n actions.append(verb + \" 1\")\n actions.append(verb + \" -1\")\n elif verb in {\"crouch\", \"jump\", \"attack\", \"use\"}:\n actions.append(verb + \" 1\")\n actions.append(verb + \" 0\")\n else:\n raise CommandHandlerException(\"Invalid continuous command\")\n elif type == 'HumanLevel':\n if verb == 'moveMouse':\n actions.append('mouseMove 0 0')\n elif verb in {'forward', 'back', 'left', 'right'}:\n actions.append(verb + ' 1')\n actions.append(verb + ' 0')\n else:\n actions.append(verb)\n elif type == 'MissionQuit':\n if verb != 'quit':\n raise CommandHandlerException(\"Invalid quit command\")\n actions.append(verb)\n elif type == 'Chat':\n if verb != 'chat':\n raise CommandHandlerException(\"Invalid chat command\")\n actions.append(verb)\n elif type == 'SimpleCraft':\n if verb != 'craft':\n raise CommandHandlerException(\"Invalid craft command\")\n actions.append(verb)\n elif type == 'AbsoluteMovement' or 'Inventory':\n actions.append(verb)\n return actions"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nloading q table from model_file", "response": "def loadModel(self, model_file):\n \"\"\"load q table from model_file\"\"\"\n with open(model_file) as f:\n self.q_table = json.load(f)"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef run(self, agent_host):\n\n total_reward = 0\n current_r = 0\n tol = 0.01\n \n self.prev_s = None\n self.prev_a = None\n \n # wait for a valid observation\n world_state = agent_host.peekWorldState()\n while world_state.is_mission_running and all(e.text=='{}' for e in world_state.observations):\n world_state = agent_host.peekWorldState()\n # wait for a frame to arrive after that\n num_frames_seen = world_state.number_of_video_frames_since_last_state\n while world_state.is_mission_running and world_state.number_of_video_frames_since_last_state == num_frames_seen:\n world_state = agent_host.peekWorldState()\n world_state = agent_host.getWorldState()\n for err in world_state.errors:\n print(err)\n\n if not world_state.is_mission_running:\n return 0 # mission already ended\n \n assert len(world_state.video_frames) > 0, 'No video frames!?'\n \n obs = json.loads( world_state.observations[-1].text )\n prev_x = obs[u'XPos']\n prev_z = obs[u'ZPos']\n print('Initial position:',prev_x,',',prev_z)\n \n if save_images:\n # save the frame, for debugging\n frame = world_state.video_frames[-1]\n image = Image.frombytes('RGB', (frame.width, frame.height), bytes(frame.pixels) )\n iFrame = 0\n self.rep = self.rep + 1\n image.save( 'rep_' + str(self.rep).zfill(3) + '_saved_frame_' + str(iFrame).zfill(4) + '.png' )\n \n # take first action\n total_reward += self.act(world_state,agent_host,current_r)\n \n require_move = True\n check_expected_position = True\n \n # main loop:\n while world_state.is_mission_running:\n \n # wait for the position to have changed and a reward received\n print('Waiting for data...', end=' ')\n while True:\n world_state = agent_host.peekWorldState()\n if not world_state.is_mission_running:\n print('mission ended.')\n break\n if len(world_state.rewards) > 0 and not all(e.text=='{}' for e in world_state.observations):\n obs = json.loads( world_state.observations[-1].text )\n curr_x = obs[u'XPos']\n curr_z = obs[u'ZPos']\n if require_move:\n if math.hypot( curr_x - prev_x, curr_z - prev_z ) > tol:\n print('received.')\n break\n else:\n print('received.')\n break\n # wait for a frame to arrive after that\n num_frames_seen = world_state.number_of_video_frames_since_last_state\n while world_state.is_mission_running and world_state.number_of_video_frames_since_last_state == num_frames_seen:\n world_state = agent_host.peekWorldState()\n \n num_frames_before_get = len(world_state.video_frames)\n \n world_state = agent_host.getWorldState()\n for err in world_state.errors:\n print(err)\n current_r = sum(r.getValue() for r in world_state.rewards)\n\n if save_images:\n # save the frame, for debugging\n if world_state.is_mission_running:\n assert len(world_state.video_frames) > 0, 'No video frames!?'\n frame = world_state.video_frames[-1]\n image = Image.frombytes('RGB', (frame.width, frame.height), bytes(frame.pixels) )\n iFrame = iFrame + 1\n image.save( 'rep_' + str(self.rep).zfill(3) + '_saved_frame_' + str(iFrame).zfill(4) + '_after_' + self.actions[self.prev_a] + '.png' )\n \n if world_state.is_mission_running:\n assert len(world_state.video_frames) > 0, 'No video frames!?'\n num_frames_after_get = len(world_state.video_frames)\n assert num_frames_after_get >= num_frames_before_get, 'Fewer frames after getWorldState!?'\n frame = world_state.video_frames[-1]\n obs = json.loads( world_state.observations[-1].text )\n curr_x = obs[u'XPos']\n curr_z = obs[u'ZPos']\n print('New position from observation:',curr_x,',',curr_z,'after action:',self.actions[self.prev_a], end=' ') #NSWE\n if check_expected_position:\n expected_x = prev_x + [0,0,-1,1][self.prev_a]\n expected_z = prev_z + [-1,1,0,0][self.prev_a]\n if math.hypot( curr_x - expected_x, curr_z - expected_z ) > tol:\n print(' - ERROR DETECTED! Expected:',expected_x,',',expected_z)\n input(\"Press Enter to continue...\")\n else:\n print('as expected.')\n curr_x_from_render = frame.xPos\n curr_z_from_render = frame.zPos\n print('New position from render:',curr_x_from_render,',',curr_z_from_render,'after action:',self.actions[self.prev_a], end=' ') #NSWE\n if math.hypot( curr_x_from_render - expected_x, curr_z_from_render - expected_z ) > tol:\n print(' - ERROR DETECTED! Expected:',expected_x,',',expected_z)\n input(\"Press Enter to continue...\")\n else:\n print('as expected.')\n else:\n print()\n prev_x = curr_x\n prev_z = curr_z\n # act\n total_reward += self.act(world_state, agent_host, current_r)\n \n # process final reward\n self.logger.debug(\"Final reward: %d\" % current_r)\n total_reward += current_r\n\n # update Q values\n if self.training and self.prev_s is not None and self.prev_a is not None:\n old_q = self.q_table[self.prev_s][self.prev_a]\n self.q_table[self.prev_s][self.prev_a] = old_q + self.alpha * ( current_r - old_q )\n \n self.drawQ()\n \n return total_reward", "response": "run the agent on the world"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ndownload Malmo from github and build the Mod.", "response": "def download(branch=None, build=True, installdir=\"MalmoPlatform\"):\n \"\"\"Download Malmo from github and build (by default) the Minecraft Mod.\n Example usage: import malmoenv.bootstrap; malmoenv.bootstrap.download()\n Args:\n branch: optional branch to clone. TODO Default is release version.\n build: build the Mod unless build arg is given as False.\n installdir: the install dir name. Defaults to MalmoPlatform.\n Returns:\n The path for the Malmo Minecraft mod.\n \"\"\"\n\n if branch is None:\n branch = malmo_version\n\n subprocess.check_call([\"git\", \"clone\", \"-b\", branch, \"https://github.com/Microsoft/malmo.git\", installdir])\n\n return setup(build=build, installdir=installdir)"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nset up Minecraft for use with the MalmoEnv gym environment", "response": "def setup(build=True, installdir=\"MalmoPlatform\"):\n \"\"\"Set up Minecraft for use with the MalmoEnv gym environment\"\"\"\n\n gradlew = './gradlew'\n if os.name == 'nt':\n gradlew = 'gradlew.bat'\n\n cwd = os.getcwd()\n os.chdir(installdir)\n os.chdir(\"Minecraft\")\n try:\n # Create the version properties file.\n pathlib.Path(\"src/main/resources/version.properties\").write_text(\"malmomod.version={}\\n\".format(malmo_version))\n # Optionally do a test build.\n if build:\n subprocess.check_call([gradlew, \"setupDecompWorkspace\", \"build\", \"testClasses\",\n \"-x\", \"test\", \"--stacktrace\", \"-Pversion={}\".format(malmo_version)])\n minecraft_dir = os.getcwd()\n finally:\n os.chdir(cwd)\n return minecraft_dir"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef launch_minecraft(port, installdir=\"MalmoPlatform\", replaceable=False):\n launch_script = './launchClient.sh'\n if os.name == 'nt':\n launch_script = 'launchClient.bat'\n cwd = os.getcwd()\n os.chdir(installdir)\n os.chdir(\"Minecraft\")\n try:\n cmd = [launch_script, '-port', str(port), '-env']\n if replaceable:\n cmd.append('-replaceable')\n subprocess.check_call(cmd)\n finally:\n os.chdir(cwd)", "response": "Launch Minecraft listening for malmoenv connections."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nprocess the frame and return the depth map.", "response": "def processFrame( frame ):\n '''Track through the middle line of the depth data and find the max discontinuities'''\n global current_yaw_delta_from_depth\n\n y = int(old_div(video_height, 2))\n rowstart = y * video_width\n \n v = 0\n v_max = 0\n v_max_pos = 0\n v_min = 0\n v_min_pos = 0\n \n dv = 0\n dv_max = 0\n dv_max_pos = 0\n dv_max_sign = 0\n \n d2v = 0\n d2v_max = 0\n d2v_max_pos = 0\n d2v_max_sign = 0\n \n for x in range(0, video_width):\n nv = frame[(rowstart + x) * 4 + 3]\n ndv = nv - v\n nd2v = ndv - dv\n\n if nv > v_max or x == 0:\n v_max = nv\n v_max_pos = x\n \n if nv < v_min or x == 0:\n v_min = nv\n v_min_pos = x\n\n if abs(ndv) > dv_max or x == 1:\n dv_max = abs(ndv)\n dv_max_pos = x\n dv_max_sign = ndv > 0\n \n if abs(nd2v) > d2v_max or x == 2:\n d2v_max = abs(nd2v)\n d2v_max_pos = x\n d2v_max_sign = nd2v > 0\n \n d2v = nd2v\n dv = ndv\n v = nv\n \n logger.info(\"d2v, dv, v: \" + str(d2v) + \", \" + str(dv) + \", \" + str(v))\n\n # We want to steer towards the greatest d2v (ie the biggest discontinuity in the gradient of the depth map).\n # If it's a possitive value, then it represents a rapid change from close to far - eg the left-hand edge of a gap.\n # Aiming to put this point in the leftmost quarter of the screen will cause us to aim for the gap.\n # If it's a negative value, it represents a rapid change from far to close - eg the right-hand edge of a gap.\n # Aiming to put this point in the rightmost quarter of the screen will cause us to aim for the gap.\n if dv_max_sign:\n edge = old_div(video_width, 4)\n else:\n edge = 3 * video_width / 4\n\n # Now, if there is something noteworthy in d2v, steer according to the above comment:\n if d2v_max > 8:\n current_yaw_delta_from_depth = (old_div(float(d2v_max_pos - edge), video_width))\n else:\n # Nothing obvious to aim for, so aim for the farthest point:\n if v_max < 255:\n current_yaw_delta_from_depth = (old_div(float(v_max_pos), video_width)) - 0.5\n else:\n # No real data to be had in d2v or v, so just go by the direction we were already travelling in:\n if current_yaw_delta_from_depth < 0:\n current_yaw_delta_from_depth = -1\n else:\n current_yaw_delta_from_depth = 1"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef checkFuelPosition(obs, agent_host):\n '''Make sure our coal, if we have any, is in slot 0.'''\n # (We need to do this because the furnace crafting commands - cooking the potato and the rabbit -\n # take the first available item of fuel in the inventory. If this isn't the coal, it could end up burning the wood\n # that we need for making the bowl.)\n for i in range(1,39):\n key = 'InventorySlot_'+str(i)+'_item'\n if key in obs:\n item = obs[key]\n if item == 'coal':\n agent_host.sendCommand(\"swapInventoryItems 0 \" + str(i))\n return", "response": "Make sure our coal is in slot 0."} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nbuilds an XML mission string that uses the RewardForCollectingItem mission handler.", "response": "def GetMissionXML(summary):\n ''' Build an XML mission string that uses the RewardForCollectingItem mission handler.'''\n \n positions = buildPositionList(items)\n \n return '''\n \n \n ''' + summary + '''\n \n\n \n \n \n \n \n \n \n \n ''' + getItemDrawing(positions) + '''\n \n \n \n \n \n\n \n Delia\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n ''' + getSubgoalPositions(positions) + '''\n \n \n \n \n ''' + video_requirements + '''\n \n \n\n '''"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef refresh_content(self, order=None, name=None):\n # reddit.get_my_subreddits() does not support sorting by order\n if order:\n self.term.flash()\n return\n\n with self.term.loader():\n self.content = SubscriptionContent.from_user(\n self.reddit, self.term.loader, self.content_type)\n if not self.term.loader.exception:\n self.nav = Navigator(self.content.get)", "response": "Re - download all subscriptions and reset the page index"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nstores the selected subreddit and return to the subreddit page", "response": "def select_subreddit(self):\n \"\"\"\n Store the selected subreddit and return to the subreddit page\n \"\"\"\n name = self.get_selected_item()['name']\n self.selected_page = self.open_subreddit_page(name)"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef alias_function(function, class_name):\n @wraps(function)\n def wrapped(self, *args, **kwargs):\n func_args = _make_func_args(function)\n if 'subreddit' in func_args and func_args.index('subreddit') != 1:\n # Only happens for search\n kwargs['subreddit'] = self\n return function(self.reddit_session, *args, **kwargs)\n else:\n return function(self.reddit_session, self, *args, **kwargs)\n # Only grab the short-line doc and add a link to the complete doc\n if wrapped.__doc__ is not None:\n wrapped.__doc__ = wrapped.__doc__.split('\\n', 1)[0]\n wrapped.__doc__ += ('\\n\\nSee :meth:`.{0}.{1}` for complete usage. '\n 'Note that you should exclude the subreddit '\n 'parameter when calling this convenience method.'\n .format(class_name, function.__name__))\n # Don't hide from sphinx as this is a parameter modifying decorator\n return wrapped", "response": "Create a RedditContentObject function mapped to a BaseReddit function."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef limit_chars(function, *args, **kwargs):\n output_chars_limit = args[0].reddit_session.config.output_chars_limit\n output_string = function(*args, **kwargs)\n if -1 < output_chars_limit < len(output_string):\n output_string = output_string[:output_chars_limit - 3] + '...'\n return output_string", "response": "Truncate the string returned from a function and return the result."} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nset the _use_oauth keyword argument to True when appropriate. This is needed because generator functions may be called at anytime, and PRAW relies on the Reddit._use_oauth value at original call time to know when to make OAuth requests. Returned data is not modified.", "response": "def oauth_generator(function, *args, **kwargs):\n \"\"\"Set the _use_oauth keyword argument to True when appropriate.\n\n This is needed because generator functions may be called at anytime, and\n PRAW relies on the Reddit._use_oauth value at original call time to know\n when to make OAuth requests.\n\n Returned data is not modified.\n\n \"\"\"\n if getattr(args[0], '_use_oauth', False):\n kwargs['_use_oauth'] = True\n return function(*args, **kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nraises client side exception when present in the API response.", "response": "def raise_api_exceptions(function, *args, **kwargs):\n \"\"\"Raise client side exception(s) when present in the API response.\n\n Returned data is not modified.\n\n \"\"\"\n try:\n return_value = function(*args, **kwargs)\n except errors.HTTPException as exc:\n if exc._raw.status_code != 400: # pylint: disable=W0212\n raise # Unhandled HTTPErrors\n try: # Attempt to convert v1 errors into older format (for now)\n data = exc._raw.json() # pylint: disable=W0212\n assert len(data) == 2\n return_value = {'errors': [(data['reason'],\n data['explanation'], '')]}\n except Exception:\n raise exc\n if isinstance(return_value, dict):\n if return_value.get('error') == 304: # Not modified exception\n raise errors.NotModified(return_value)\n elif return_value.get('errors'):\n error_list = []\n for error_type, msg, value in return_value['errors']:\n if error_type in errors.ERROR_MAPPING:\n if error_type == 'RATELIMIT':\n args[0].evict(args[1])\n error_class = errors.ERROR_MAPPING[error_type]\n else:\n error_class = errors.APIException\n error_list.append(error_class(error_type, msg, value,\n return_value))\n if len(error_list) == 1:\n raise error_list[0]\n else:\n raise errors.ExceptionList(error_list)\n return return_value"} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns a decorator for methods that require captchas.", "response": "def require_captcha(function, *args, **kwargs):\n \"\"\"Return a decorator for methods that require captchas.\"\"\"\n raise_captcha_exception = kwargs.pop('raise_captcha_exception', False)\n captcha_id = None\n\n # Get a handle to the reddit session\n if hasattr(args[0], 'reddit_session'):\n reddit_session = args[0].reddit_session\n else:\n reddit_session = args[0]\n\n while True:\n try:\n if captcha_id:\n captcha_answer = _get_captcha(reddit_session, captcha_id)\n\n # When the method is being decorated, all of its default\n # parameters become part of this *args tuple. This means that\n # *args currently contains a None where the captcha answer\n # needs to go. If we put the captcha in the **kwargs,\n # we get a TypeError for having two values of the same param.\n func_args = _make_func_args(function)\n if 'captcha' in func_args:\n captcha_index = func_args.index('captcha')\n args = list(args)\n args[captcha_index] = captcha_answer\n else:\n kwargs['captcha'] = captcha_answer\n return function(*args, **kwargs)\n except errors.InvalidCaptcha as exception:\n if raise_captcha_exception or \\\n not hasattr(sys.stdin, 'closed') or sys.stdin.closed:\n raise\n captcha_id = exception.response['captcha']"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nrestricting the access to the given user.", "response": "def restrict_access(scope, mod=None, login=None, oauth_only=False,\n generator_called=False):\n \"\"\"Restrict function access unless the user has the necessary permissions.\n\n Raises one of the following exceptions when appropriate:\n * LoginRequired\n * LoginOrOAuthRequired\n * the scope attribute will provide the necessary scope name\n * ModeratorRequired\n * ModeratorOrOAuthRequired\n * the scope attribute will provide the necessary scope name\n\n :param scope: Indicate the scope that is required for the API call. None or\n False must be passed to indicate that no scope handles the API call.\n All scopes save for `read` imply login=True. Scopes with 'mod' in their\n name imply mod=True.\n :param mod: Indicate that a moderator is required. Implies login=True.\n :param login: Indicate that a login is required.\n :param oauth_only: Indicate that only OAuth is supported for the function.\n :param generator_called: Indicate that the function consists solely of\n exhausting one or more oauth_generator wrapped generators. This is\n because the oauth_generator itself will determine whether or not to\n use the oauth domain.\n\n Returned data is not modified.\n\n This decorator assumes that all mod required functions fit one of these\n categories:\n\n * have the subreddit as the first argument (Reddit instance functions) or\n have a subreddit keyword argument\n * are called upon a subreddit object (Subreddit RedditContentObject)\n * are called upon a RedditContent object with attribute subreddit\n\n \"\"\"\n if not scope and oauth_only:\n raise TypeError('`scope` must be set when `oauth_only` is set')\n\n mod = mod is not False and (mod or scope and 'mod' in scope)\n login = login is not False and (login or mod or scope and scope != 'read')\n\n @decorator.decorator\n def wrap(function, *args, **kwargs):\n if args[0] is None: # Occurs with (un)friend\n assert login\n raise errors.LoginRequired(function.__name__)\n # This segment of code uses hasattr to determine what instance type\n # the function was called on. We could use isinstance if we wanted\n # to import the types at runtime (decorators is used by all the\n # types).\n if mod:\n if hasattr(args[0], 'reddit_session'):\n # Defer access until necessary for RedditContentObject.\n # This is because scoped sessions may not require this\n # attribute to exist, thus it might not be set.\n from .objects import Subreddit\n subreddit = args[0] if isinstance(args[0], Subreddit) \\\n else False\n else:\n subreddit = kwargs.get(\n 'subreddit', args[1] if len(args) > 1 else None)\n if subreddit is None: # Try the default value\n defaults = six.get_function_defaults(function)\n subreddit = defaults[0] if defaults else None\n else:\n subreddit = None\n\n obj = getattr(args[0], 'reddit_session', args[0])\n # This function sets _use_oauth for one time use only.\n # Verify that statement is actually true.\n assert not obj._use_oauth # pylint: disable=W0212\n\n if scope and obj.has_scope(scope):\n obj._use_oauth = not generator_called # pylint: disable=W0212\n elif oauth_only:\n raise errors.OAuthScopeRequired(function.__name__, scope)\n elif login and obj.is_logged_in():\n if subreddit is False:\n # Now fetch the subreddit attribute. There is no good\n # reason for it to not be set during a logged in session.\n subreddit = args[0].subreddit\n if mod and not _is_mod_of_all(obj.user, subreddit):\n if scope:\n raise errors.ModeratorOrScopeRequired(\n function.__name__, scope)\n raise errors.ModeratorRequired(function.__name__)\n elif login:\n if scope:\n raise errors.LoginOrScopeRequired(function.__name__, scope)\n raise errors.LoginRequired(function.__name__)\n try:\n return function(*args, **kwargs)\n finally:\n obj._use_oauth = False # pylint: disable=W0212\n return wrap"} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nverifies that the OAuth functions can be used prior to use.", "response": "def require_oauth(function, *args, **kwargs):\n \"\"\"Verify that the OAuth functions can be used prior to use.\n\n Returned data is not modified.\n\n \"\"\"\n if not args[0].has_oauth_app_info:\n err_msg = (\"The OAuth app config parameters client_id, client_secret \"\n \"and redirect_url must be specified to use this function.\")\n raise errors.OAuthAppRequired(err_msg)\n return function(*args, **kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\niterate over the items in the archive.", "response": "def iterate(self, index, step, n_cols=70):\n \"\"\"\n Return an iterator that starts and the current index and increments\n by the given step.\n \"\"\"\n\n while True:\n if step < 0 and index < 0:\n # Hack to prevent displaying a submission's post if iterating\n # comments in the negative direction\n break\n try:\n yield self.get(index, n_cols=n_cols)\n except IndexError:\n break\n index += step"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef flatten_comments(comments, root_level=0):\n\n stack = comments[:]\n for item in stack:\n item.nested_level = root_level\n\n retval, parent_candidates = [], {}\n while stack:\n item = stack.pop(0)\n\n # The MoreComments item count should never be zero, discard it if\n # it is. Need to look into this further.\n if isinstance(item, praw.objects.MoreComments) and item.count == 0:\n continue\n\n if item.parent_id:\n # Search the list of previous comments for a possible parent\n # The match is based off of the parent_id parameter E.g.\n # parent.id = c0tprcm\n # child.parent_id = t1_c0tprcm\n parent = parent_candidates.get(item.parent_id[3:])\n if parent:\n item.nested_level = parent.nested_level + 1\n\n # Add all of the attached replies to the front of the stack to be\n # parsed separately\n if hasattr(item, 'replies'):\n for n in item.replies:\n n.nested_level = item.nested_level + 1\n stack[0:0] = item.replies\n\n # The comment is now a potential parent for the items that are\n # remaining on the stack.\n parent_candidates[item.id] = item\n\n retval.append(item)\n return retval", "response": "Flatten a PRAW comment tree while preserving the nested level of each comment."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef strip_praw_comment(cls, comment):\n\n data = {}\n data['object'] = comment\n\n if isinstance(comment, praw.objects.MoreComments):\n data['type'] = 'MoreComments'\n data['level'] = comment.nested_level\n data['count'] = comment.count\n data['body'] = 'More comments'\n data['hidden'] = True\n\n elif hasattr(comment, 'nested_level'):\n author = getattr(comment, 'author', '[deleted]')\n name = getattr(author, 'name', '[deleted]')\n sub = getattr(comment, 'submission', '[deleted]')\n sub_author = getattr(sub, 'author', '[deleted]')\n sub_name = getattr(sub_author, 'name', '[deleted]')\n flair = getattr(comment, 'author_flair_text', '')\n permalink = getattr(comment, 'permalink', None)\n stickied = getattr(comment, 'stickied', False)\n\n data['type'] = 'Comment'\n data['level'] = comment.nested_level\n data['body'] = comment.body\n data['html'] = comment.body_html\n data['created'] = cls.humanize_timestamp(comment.created_utc)\n data['score'] = '{0} pts'.format(\n '-' if comment.score_hidden else comment.score)\n data['author'] = name\n data['is_author'] = (name == sub_name)\n data['flair'] = flair\n data['likes'] = comment.likes\n data['gold'] = comment.gilded\n data['permalink'] = permalink\n data['stickied'] = stickied\n data['hidden'] = False\n data['saved'] = comment.saved\n if comment.edited:\n data['edited'] = '(edit {})'.format(\n cls.humanize_timestamp(comment.edited))\n else:\n data['edited'] = ''\n else:\n # Saved comments don't have a nested level and are missing a couple\n # of fields like ``submission``. As a result, we can only load a\n # subset of fields to avoid triggering a separate api call to load\n # the full comment.\n author = getattr(comment, 'author', '[deleted]')\n stickied = getattr(comment, 'stickied', False)\n flair = getattr(comment, 'author_flair_text', '')\n\n data['type'] = 'SavedComment'\n data['level'] = None\n data['title'] = '[Comment] {0}'.format(comment.body)\n data['comments'] = None\n data['url_full'] = comment._fast_permalink\n data['url'] = comment._fast_permalink\n data['permalink'] = comment._fast_permalink\n data['nsfw'] = comment.over_18\n data['subreddit'] = six.text_type(comment.subreddit)\n data['url_type'] = 'selfpost'\n data['score'] = '{0} pts'.format(\n '-' if comment.score_hidden else comment.score)\n data['likes'] = comment.likes\n data['created'] = cls.humanize_timestamp(comment.created_utc)\n data['saved'] = comment.saved\n data['stickied'] = stickied\n data['gold'] = comment.gilded\n data['author'] = author\n data['flair'] = flair\n data['hidden'] = False\n if comment.edited:\n data['edited'] = '(edit {})'.format(\n cls.humanize_timestamp(comment.edited))\n else:\n data['edited'] = ''\n\n return data", "response": "Parse a submission comment and return a dict with data ready to be displayed through the terminal."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nparsing through a submission and return a dict with the data ready to be displayed through the terminal.", "response": "def strip_praw_submission(cls, sub):\n \"\"\"\n Parse through a submission and return a dict with data ready to be\n displayed through the terminal.\n\n Definitions:\n permalink - URL to the reddit page with submission comments.\n url_full - URL that the submission points to.\n url - URL that will be displayed on the subreddit page, may be\n \"selfpost\", \"x-post submission\", \"x-post subreddit\", or an\n external link.\n \"\"\"\n\n reddit_link = re.compile(\n r'https?://(www\\.)?(np\\.)?redd(it\\.com|\\.it)/r/.*')\n author = getattr(sub, 'author', '[deleted]')\n name = getattr(author, 'name', '[deleted]')\n flair = getattr(sub, 'link_flair_text', '')\n\n data = {}\n data['object'] = sub\n data['type'] = 'Submission'\n data['title'] = sub.title\n data['text'] = sub.selftext\n data['html'] = sub.selftext_html or ''\n data['created'] = cls.humanize_timestamp(sub.created_utc)\n data['created_long'] = cls.humanize_timestamp(sub.created_utc, True)\n data['comments'] = '{0} comments'.format(sub.num_comments)\n data['score'] = '{0} pts'.format('-' if sub.hide_score else sub.score)\n data['author'] = name\n data['permalink'] = sub.permalink\n data['subreddit'] = six.text_type(sub.subreddit)\n data['flair'] = '[{0}]'.format(flair.strip(' []')) if flair else ''\n data['url_full'] = sub.url\n data['likes'] = sub.likes\n data['gold'] = sub.gilded\n data['nsfw'] = sub.over_18\n data['stickied'] = sub.stickied\n data['hidden'] = sub.hidden\n data['xpost_subreddit'] = None\n data['index'] = None # This is filled in later by the method caller\n data['saved'] = sub.saved\n if sub.edited:\n data['edited'] = '(edit {})'.format(\n cls.humanize_timestamp(sub.edited))\n data['edited_long'] = '(edit {})'.format(\n cls.humanize_timestamp(sub.edited, True))\n else:\n data['edited'] = ''\n data['edited_long'] = ''\n\n if sub.url.split('/r/')[-1] == sub.permalink.split('/r/')[-1]:\n data['url'] = 'self.{0}'.format(data['subreddit'])\n data['url_type'] = 'selfpost'\n elif reddit_link.match(sub.url):\n # Strip the subreddit name from the permalink to avoid having\n # submission.subreddit.url make a separate API call\n url_parts = sub.url.split('/')\n data['xpost_subreddit'] = url_parts[4]\n data['url'] = 'self.{0}'.format(url_parts[4])\n if 'comments' in url_parts:\n data['url_type'] = 'x-post submission'\n else:\n data['url_type'] = 'x-post subreddit'\n else:\n data['url'] = sub.url\n data['url_type'] = 'external'\n\n return data"} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nparse through a subscription and return a dict with data ready to be displayed through the terminal.", "response": "def strip_praw_subscription(subscription):\n \"\"\"\n Parse through a subscription and return a dict with data ready to be\n displayed through the terminal.\n \"\"\"\n\n data = {}\n data['object'] = subscription\n if isinstance(subscription, praw.objects.Multireddit):\n data['type'] = 'Multireddit'\n data['name'] = subscription.path\n data['title'] = subscription.description_md\n else:\n data['type'] = 'Subscription'\n data['name'] = \"/r/\" + subscription.display_name\n data['title'] = subscription.title\n\n return data"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef strip_praw_message(cls, msg):\n author = getattr(msg, 'author', None)\n\n data = {}\n data['object'] = msg\n\n if isinstance(msg, praw.objects.Message):\n data['type'] = 'Message'\n data['level'] = msg.nested_level\n data['distinguished'] = msg.distinguished\n data['permalink'] = None\n data['submission_permalink'] = None\n data['subreddit_name'] = None\n data['link_title'] = None\n data['context'] = None\n else:\n data['type'] = 'InboxComment'\n data['level'] = 0\n data['distinguished'] = None\n data['permalink'] = msg._fast_permalink\n data['submission_permalink'] = '/'.join(data['permalink'].split('/')[:-2])\n data['subreddit_name'] = msg.subreddit_name_prefixed\n data['link_title'] = msg.link_title\n data['context'] = msg.context\n\n data['id'] = msg.id\n data['subject'] = msg.subject\n data['body'] = msg.body\n data['html'] = msg.body_html\n data['created'] = cls.humanize_timestamp(msg.created_utc)\n data['created_long'] = cls.humanize_timestamp(msg.created_utc, True)\n data['recipient'] = msg.dest\n data['distinguished'] = msg.distinguished\n data['author'] = author.name if author else '[deleted]'\n data['is_new'] = msg.new\n data['was_comment'] = msg.was_comment\n return data", "response": "Parse a praw. objects. Message and return a dict with data ready to be\n displayed through the terminal."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nconverts a utc timestamp into a human readable relative - time.", "response": "def humanize_timestamp(utc_timestamp, verbose=False):\n \"\"\"\n Convert a utc timestamp into a human readable relative-time.\n \"\"\"\n\n timedelta = datetime.utcnow() - datetime.utcfromtimestamp(utc_timestamp)\n\n seconds = int(timedelta.total_seconds())\n if seconds < 60:\n return 'moments ago' if verbose else '0min'\n\n minutes = seconds // 60\n if minutes < 60:\n if verbose and minutes == 1:\n return '1 minutes ago'\n elif verbose:\n return '%d minutes ago' % minutes\n else:\n return '%dmin' % minutes\n\n hours = minutes // 60\n if hours < 24:\n if verbose and hours == 1:\n return '1 hour ago'\n elif verbose:\n return '%d hours ago' % hours\n else:\n return '%dhr' % hours\n\n days = hours // 24\n if days < 30:\n if verbose and days == 1:\n return '1 day ago'\n elif verbose:\n return '%d days ago' % days\n else:\n return '%dday' % days\n\n months = days // 30.4\n if months < 12:\n if verbose and months == 1:\n return '1 month ago'\n elif verbose:\n return '%d months ago' % months\n else:\n return '%dmonth' % months\n\n years = months // 12\n if verbose and years == 1:\n return '1 year ago'\n elif verbose:\n return '%d years ago' % years\n else:\n return '%dyr' % years"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nwrap text paragraphs to the given character width while preserving newlines.", "response": "def wrap_text(text, width):\n \"\"\"\n Wrap text paragraphs to the given character width while preserving\n newlines.\n \"\"\"\n out = []\n for paragraph in text.splitlines():\n # Wrap returns an empty list when paragraph is a newline. In order\n # to preserve newlines we substitute a list containing an empty\n # string.\n lines = wrap(paragraph, width=width) or ['']\n out.extend(lines)\n return out"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nextracting a list of hyperlinks from an HTML document.", "response": "def extract_links(html):\n \"\"\"\n Extract a list of hyperlinks from an HTML document.\n \"\"\"\n links = []\n soup = BeautifulSoup(html, 'html.parser')\n for link in soup.findAll('a'):\n href = link.get('href')\n if not href:\n continue\n if href.startswith('/'):\n href = 'https://www.reddit.com' + href\n links.append({'text': link.text, 'href': href})\n return links"} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ngrabs the ith submission with the title field formatted to fit inside the ith submission with the text field formatted to fit inside the ith comment.", "response": "def get(self, index, n_cols=70):\n \"\"\"\n Grab the `i`th submission, with the title field formatted to fit inside\n of a window of width `n`\n \"\"\"\n\n if index < -1:\n raise IndexError\n\n elif index == -1:\n data = self._submission_data\n data['split_title'] = self.wrap_text(data['title'], width=n_cols-2)\n data['split_text'] = self.wrap_text(data['text'], width=n_cols-2)\n data['n_rows'] = len(data['split_title'] + data['split_text']) + 5\n data['h_offset'] = 0\n\n else:\n data = self._comment_data[index]\n indent_level = min(data['level'], self.max_indent_level)\n data['h_offset'] = indent_level * self.indent_size\n\n if data['type'] == 'Comment':\n width = min(n_cols - data['h_offset'], self._max_comment_cols)\n data['split_body'] = self.wrap_text(data['body'], width=width)\n data['n_rows'] = len(data['split_body']) + 1\n else:\n data['n_rows'] = 1\n\n return data"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ntoggle the state of the object at the given index.", "response": "def toggle(self, index, n_cols=70):\n \"\"\"\n Toggle the state of the object at the given index.\n\n If it is a comment, pack it into a hidden comment.\n If it is a hidden comment, unpack it.\n If it is more comments, load the comments.\n \"\"\"\n data = self.get(index)\n\n if data['type'] == 'Submission':\n # Can't hide the submission!\n pass\n\n elif data['type'] == 'Comment':\n cache = [data]\n count = 1\n for d in self.iterate(index + 1, 1, n_cols):\n if d['level'] <= data['level']:\n break\n\n count += d.get('count', 1)\n cache.append(d)\n\n comment = {\n 'type': 'HiddenComment',\n 'cache': cache,\n 'count': count,\n 'level': data['level'],\n 'body': 'Hidden',\n 'hidden': True}\n\n self._comment_data[index:index + len(cache)] = [comment]\n\n elif data['type'] == 'HiddenComment':\n self._comment_data[index:index + 1] = data['cache']\n\n elif data['type'] == 'MoreComments':\n with self._loader('Loading comments'):\n # Undefined behavior if using a nested loader here\n assert self._loader.depth == 1\n comments = data['object'].comments(update=True)\n if not self._loader.exception:\n comments = self.flatten_comments(comments, data['level'])\n comment_data = [self.strip_praw_comment(c) for c in comments]\n self._comment_data[index:index + 1] = comment_data\n\n else:\n raise ValueError('%s type not recognized' % data['type'])"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef from_name(cls, reddit, name, loader, order=None, query=None):\n # TODO: This desperately needs to be refactored\n\n # Strip leading, trailing, and redundant backslashes\n parts = [seg for seg in name.strip(' /').split('/') if seg]\n\n # Check for the resource type, assume /r/ as the default\n if len(parts) >= 3 and parts[2] == 'm':\n # E.g. /u/civilization_phaze_3/m/multireddit ->\n # resource_root = \"u/civilization_phaze_3/m\"\n # parts = [\"multireddit\"]\n resource_root, parts = '/'.join(parts[:3]), parts[3:]\n elif len(parts) > 1 and parts[0] in ['r', 'u', 'user', 'domain']:\n # E.g. /u/civilization_phaze_3 ->\n # resource_root = \"u\"\n # parts = [\"civilization_phaze_3\"]\n #\n # E.g. /r/python/top-week ->\n # resource_root = \"r\"\n # parts = [\"python\", \"top-week\"]\n resource_root = parts.pop(0)\n else:\n resource_root = 'r'\n\n if resource_root == 'user':\n resource_root = 'u'\n elif resource_root.startswith('user/'):\n # Special check for multi-reddit resource roots\n # E.g.\n # before: resource_root = \"user/civilization_phaze_3/m\"\n # After: resource_root = \"u/civilization_phaze_3/m\"\n resource_root = 'u' + resource_root[4:]\n\n # The parts left should be in one of the following forms:\n # [resource]\n # [resource, order]\n # [resource, user_room, order]\n\n user_rooms = ['overview', 'submitted', 'comments']\n private_user_rooms = ['upvoted', 'downvoted', 'hidden', 'saved']\n user_room = None\n\n if len(parts) == 1:\n # E.g. /r/python\n # parts = [\"python\"]\n # resource = \"python\"\n # resource_order = None\n resource, resource_order = parts[0], None\n elif resource_root == 'u' and len(parts) in [2, 3] \\\n and parts[1] in user_rooms + private_user_rooms:\n # E.g. /u/spez/submitted/top ->\n # parts = [\"spez\", \"submitted\", \"top\"]\n # resource = \"spez\"\n # user_room = \"submitted\"\n # resource_order = \"top\"\n resource, user_room = parts[:2]\n resource_order = parts[2] if len(parts) == 3 else None\n elif len(parts) == 2:\n # E.g. /r/python/top\n # parts = [\"python\", \"top\"]\n # resource = \"python\n # resource_order = \"top\"\n resource, resource_order = parts\n else:\n raise InvalidSubreddit('`{}` is an invalid format'.format(name))\n\n if not resource:\n # Praw does not correctly handle empty strings\n # https://github.com/praw-dev/praw/issues/615\n raise InvalidSubreddit('Subreddit cannot be empty')\n\n # If the order was explicitly passed in, it will take priority over\n # the order that was extracted from the name\n order = order or resource_order\n\n display_order = order\n display_name = '/'.join(['', resource_root, resource])\n if user_room and resource_root == 'u':\n display_name += '/' + user_room\n\n # Split the order from the period E.g. controversial-all, top-hour\n if order and '-' in order:\n order, period = order.split('-', 1)\n else:\n period = None\n\n if query:\n # The allowed orders for sorting search results are different\n orders = ['relevance', 'top', 'comments', 'new', None]\n period_allowed = ['top', 'comments']\n else:\n orders = ['hot', 'top', 'rising', 'new', 'controversial', 'gilded', None]\n period_allowed = ['top', 'controversial']\n\n if order not in orders:\n raise InvalidSubreddit('Invalid order `%s`' % order)\n if period not in ['all', 'day', 'hour', 'month', 'week', 'year', None]:\n raise InvalidSubreddit('Invalid period `%s`' % period)\n if period and order not in period_allowed:\n raise InvalidSubreddit(\n '`%s` order does not allow sorting by period' % order)\n\n # On some objects, praw doesn't allow you to pass arguments for the\n # order and period. Instead you need to call special helper functions\n # such as Multireddit.get_controversial_from_year(). Build the method\n # name here for convenience.\n if period:\n method_alias = 'get_{0}_from_{1}'.format(order, period)\n elif order:\n method_alias = 'get_{0}'.format(order)\n else:\n method_alias = 'get_hot'\n\n # Here's where we start to build the submission generators\n if query:\n if resource_root == 'u':\n search = '/r/{subreddit}/search'\n author = reddit.user.name if resource == 'me' else resource\n query = 'author:{0} {1}'.format(author, query)\n subreddit = None\n else:\n search = resource_root + '/{subreddit}/search'\n subreddit = None if resource == 'front' else resource\n\n reddit.config.API_PATHS['search'] = search\n submissions = reddit.search(query, subreddit=subreddit,\n sort=order, period=period)\n\n elif resource_root == 'domain':\n order = order or 'hot'\n submissions = reddit.get_domain_listing(\n resource, sort=order, period=period, limit=None)\n\n elif resource_root.endswith('/m'):\n redditor = resource_root.split('/')[1]\n\n if redditor == 'me':\n if not reddit.is_oauth_session():\n raise exceptions.AccountError('Not logged in')\n else:\n redditor = reddit.user.name\n display_name = display_name.replace(\n '/me/', '/{0}/'.format(redditor))\n\n multireddit = reddit.get_multireddit(redditor, resource)\n submissions = getattr(multireddit, method_alias)(limit=None)\n\n elif resource_root == 'u' and resource == 'me':\n if not reddit.is_oauth_session():\n raise exceptions.AccountError('Not logged in')\n else:\n user_room = user_room or 'overview'\n order = order or 'new'\n period = period or 'all'\n method = getattr(reddit.user, 'get_%s' % user_room)\n submissions = method(sort=order, time=period, limit=None)\n\n elif resource_root == 'u':\n user_room = user_room or 'overview'\n if user_room not in user_rooms:\n # Tried to access a private room like \"u/me/hidden\" for a\n # different redditor\n raise InvalidSubreddit('Unavailable Resource')\n order = order or 'new'\n period = period or 'all'\n redditor = reddit.get_redditor(resource)\n method = getattr(redditor, 'get_%s' % user_room)\n submissions = method(sort=order, time=period, limit=None)\n\n elif resource == 'front':\n if order in (None, 'hot'):\n submissions = reddit.get_front_page(limit=None)\n elif period:\n # For the front page, praw makes you send the period as `t`\n # instead of calling reddit.get_hot_from_week()\n method_alias = 'get_{0}'.format(order)\n method = getattr(reddit, method_alias)\n submissions = method(limit=None, params={'t': period})\n else:\n submissions = getattr(reddit, method_alias)(limit=None)\n\n else:\n subreddit = reddit.get_subreddit(resource)\n submissions = getattr(subreddit, method_alias)(limit=None)\n\n # For special subreddits like /r/random we want to replace the\n # display name with the one returned by the request.\n display_name = '/r/{0}'.format(subreddit.display_name)\n\n filter_nsfw = (reddit.user and reddit.user.over_18 is False)\n\n # We made it!\n return cls(display_name, submissions, loader, order=display_order,\n query=query, filter_nsfw=filter_nsfw)", "response": "Return a new object from a name."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ngrabs the ith submission with the title field formatted to fit inside the n_cols window of width n_cols", "response": "def get(self, index, n_cols=70):\n \"\"\"\n Grab the `i`th submission, with the title field formatted to fit inside\n of a window of width `n_cols`\n \"\"\"\n\n if index < 0:\n raise IndexError\n\n nsfw_count = 0\n while index >= len(self._submission_data):\n try:\n with self._loader('Loading more submissions'):\n submission = next(self._submissions)\n if self._loader.exception:\n raise IndexError\n except StopIteration:\n raise IndexError\n else:\n\n # Skip NSFW posts based on the reddit user's profile settings.\n # If we see 20+ NSFW posts at the beginning, assume the subreddit\n # only has NSFW content and abort. This allows us to avoid making\n # an additional API call to check if a subreddit is over18 (which\n # doesn't work for things like multireddits anyway)\n if self.filter_nsfw and submission.over_18:\n nsfw_count += 1\n if not self._submission_data and nsfw_count >= 20:\n raise exceptions.SubredditError(\n 'You must be over 18+ to view this subreddit')\n continue\n else:\n nsfw_count = 0\n\n if hasattr(submission, 'title'):\n data = self.strip_praw_submission(submission)\n else:\n # when submission is a saved comment\n data = self.strip_praw_comment(submission)\n\n data['index'] = len(self._submission_data) + 1\n # Add the post number to the beginning of the title\n data['title'] = '{0}. {1}'.format(data['index'], data['title'])\n self._submission_data.append(data)\n\n # Modifies the original dict, faster than copying\n data = self._submission_data[index]\n data['split_title'] = self.wrap_text(data['title'], width=n_cols)\n if len(data['split_title']) > self.max_title_rows:\n data['split_title'] = data['split_title'][:self.max_title_rows-1]\n data['split_title'].append('(Not enough space to display)')\n data['n_rows'] = len(data['split_title']) + 3\n data['h_offset'] = 0\n\n return data"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get(self, index, n_cols=70):\n\n if index < 0:\n raise IndexError\n\n while index >= len(self._subscription_data):\n try:\n with self._loader('Loading content'):\n subscription = next(self._subscriptions)\n if self._loader.exception:\n raise IndexError\n except StopIteration:\n raise IndexError\n else:\n data = self.strip_praw_subscription(subscription)\n self._subscription_data.append(data)\n\n data = self._subscription_data[index]\n data['split_title'] = self.wrap_text(data['title'], width=n_cols)\n data['n_rows'] = len(data['split_title']) + 1\n data['h_offset'] = 0\n\n return data", "response": "Grab the ith object with the title field formatted to fit n_cols inside of a window of width n_cols"} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\npauses before making the next HTTP request.", "response": "def _delay(self):\n \"\"\"\n Pause before making the next HTTP request.\n \"\"\"\n if self.next_request_timestamp is None:\n return\n\n sleep_seconds = self.next_request_timestamp - time.time()\n if sleep_seconds <= 0:\n return\n time.sleep(sleep_seconds)"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nclearing the cache of timed out results.", "response": "def _clear_timeouts(self, cache_timeout):\n \"\"\"\n Clear the cache of timed out results.\n \"\"\"\n\n for key in list(self.timeouts):\n if timer() - self.timeouts[key] > cache_timeout:\n del self.timeouts[key]\n del self.cache[key]"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef evict(self, urls):\n if isinstance(urls, six.text_type):\n urls = [urls]\n urls = set(normalize_url(url) for url in urls)\n retval = 0\n for key in list(self.cache):\n if key[0] in urls:\n retval += 1\n del self.cache[key]\n del self.timeouts[key]\n return retval", "response": "Remove items from cache matching URLs. Returns the number of items removed."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef request(self, _cache_key, _cache_ignore, _cache_timeout, **kwargs):\n if _cache_key:\n # Pop the request's session cookies from the cache key.\n # These appear to be unreliable and change with every\n # request. Also, with the introduction of OAuth I don't think\n # that cookies are being used to store anything that\n # differentiates API requests anyways\n url, items = _cache_key\n _cache_key = (url, (items[0], items[1], items[3], items[4]))\n\n if kwargs['request'].method != 'GET':\n # I added this check for RTV, I have no idea why PRAW would ever\n # want to cache POST/PUT/DELETE requests\n _cache_ignore = True\n\n if _cache_ignore:\n return self._request(**kwargs)\n\n self._clear_timeouts(_cache_timeout)\n if _cache_key in self.cache:\n return self.cache[_cache_key]\n\n result = self._request(**kwargs)\n\n # The handlers don't call `raise_for_status` so we need to ignore\n # status codes that will result in an exception that should not be\n # cached.\n if result.status_code not in (200, 302):\n return result\n\n self.timeouts[_cache_key] = timer()\n self.cache[_cache_key] = result\n return result", "response": "This is a wrapper function that handles the caching of the request."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _request(self, request, proxies, timeout, verify, **_):\n\n settings = self.http.merge_environment_settings(\n request.url, proxies, False, verify, None)\n\n self._delay()\n response = self.http.send(\n request, timeout=timeout, allow_redirects=False, **settings)\n self._update(response.headers)\n\n return response", "response": "This is where we apply rate limiting and make the HTTP request."} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn the user - agent string.", "response": "def ua_string(praw_info):\n \"\"\"Return the user-agent string.\n\n The user-agent string contains PRAW version and platform version info.\n\n \"\"\"\n if os.environ.get('SERVER_SOFTWARE') is not None:\n # Google App Engine information\n # https://developers.google.com/appengine/docs/python/\n info = os.environ.get('SERVER_SOFTWARE')\n else:\n # Standard platform information\n info = platform.platform(True).encode('ascii', 'ignore')\n\n return '{0} PRAW/{1} Python/{2} {3}'.format(\n praw_info, __version__, sys.version.split()[0], info)"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ngives a page url and a dict of params, open and return the page. :param url: the url to grab content from. :param params: a dictionary containing the GET data to put in the url :param data: a dictionary containing the extra data to submit :param files: a dictionary specifying the files to upload :param auth: Add the HTTP authentication headers (see requests) :param timeout: Specifies the maximum time that the actual HTTP request can take. :param raw_response: return the response object rather than the response body :param retry_on_error: if True retry the request, if it fails, for up to 3 attempts :returns: either the response body or the response object", "response": "def _request(self, url, params=None, data=None, files=None, auth=None,\n timeout=None, raw_response=False, retry_on_error=True,\n method=None):\n \"\"\"Given a page url and a dict of params, open and return the page.\n\n :param url: the url to grab content from.\n :param params: a dictionary containing the GET data to put in the url\n :param data: a dictionary containing the extra data to submit\n :param files: a dictionary specifying the files to upload\n :param auth: Add the HTTP authentication headers (see requests)\n :param timeout: Specifies the maximum time that the actual HTTP request\n can take.\n :param raw_response: return the response object rather than the\n response body\n :param retry_on_error: if True retry the request, if it fails, for up\n to 3 attempts\n :returns: either the response body or the response object\n\n \"\"\"\n def build_key_items(url, params, data, auth, files, method):\n request = _prepare_request(self, url, params, data, auth, files,\n method)\n\n # Prepare extra arguments\n key_items = []\n oauth = request.headers.get('Authorization', None)\n for key_value in (params, data, request.cookies, auth, oauth):\n if isinstance(key_value, dict):\n key_items.append(tuple(key_value.items()))\n elif isinstance(key_value, http_cookiejar.CookieJar):\n key_items.append(tuple(key_value.get_dict().items()))\n else:\n key_items.append(key_value)\n kwargs = {'_rate_domain': self.config.domain,\n '_rate_delay': int(self.config.api_request_delay),\n '_cache_ignore': bool(files) or raw_response,\n '_cache_timeout': int(self.config.cache_timeout)}\n\n return (request, key_items, kwargs)\n\n def decode(match):\n return six.unichr(html_entities.name2codepoint[match.group(1)])\n\n def handle_redirect():\n response = None\n url = request.url\n while url: # Manually handle 302 redirects\n request.url = url\n kwargs['_cache_key'] = (normalize_url(request.url),\n tuple(key_items))\n response = self.handler.request(\n request=request.prepare(),\n proxies=self.http.proxies,\n timeout=timeout,\n verify=self.http.validate_certs, **kwargs)\n\n if self.config.log_requests >= 2:\n msg = 'status: {0}\\n'.format(response.status_code)\n sys.stderr.write(msg)\n url = _raise_redirect_exceptions(response)\n assert url != request.url\n return response\n\n timeout = self.config.timeout if timeout is None else timeout\n request, key_items, kwargs = build_key_items(url, params, data,\n auth, files, method)\n\n tempauth = self._use_oauth\n remaining_attempts = 3 if retry_on_error else 1\n attempt_oauth_refresh = bool(self.refresh_token)\n while True:\n try:\n self._use_oauth = self.is_oauth_session()\n response = handle_redirect()\n _raise_response_exceptions(response)\n self.http.cookies.update(response.cookies)\n if raw_response:\n return response\n else:\n return re.sub('&([^;]+);', decode, response.text)\n except errors.OAuthInvalidToken as error:\n if not attempt_oauth_refresh:\n raise\n attempt_oauth_refresh = False\n self._use_oauth = False\n self.refresh_access_information()\n self._use_oauth = tempauth\n request, key_items, kwargs = build_key_items(url, params,\n data, auth, files,\n method)\n except errors.HTTPException as error:\n remaining_attempts -= 1\n # pylint: disable=W0212\n if error._raw.status_code not in self.RETRY_CODES or \\\n remaining_attempts == 0:\n raise\n finally:\n self._use_oauth = tempauth"} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns an appropriate RedditObject from json_data when possible.", "response": "def _json_reddit_objecter(self, json_data):\n \"\"\"Return an appropriate RedditObject from json_data when possible.\"\"\"\n try:\n object_class = self.config.by_kind[json_data['kind']]\n except KeyError:\n if 'json' in json_data:\n if len(json_data) != 1:\n msg = 'Unknown object type: {0}'.format(json_data)\n warn_explicit(msg, UserWarning, '', 0)\n return json_data['json']\n else:\n return object_class.from_api_response(self, json_data['data'])\n return json_data"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef evict(self, urls):\n if isinstance(urls, six.string_types):\n urls = (urls,)\n return self.handler.evict(urls)", "response": "Evicts the given url from the cache."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nmake a HTTP request and return the response.", "response": "def request(self, url, params=None, data=None, retry_on_error=True,\n method=None):\n \"\"\"Make a HTTP request and return the response.\n\n :param url: the url to grab content from.\n :param params: a dictionary containing the GET data to put in the url\n :param data: a dictionary containing the extra data to submit\n :param retry_on_error: if True retry the request, if it fails, for up\n to 3 attempts\n :param method: The HTTP method to use in the request.\n :returns: The HTTP response.\n \"\"\"\n return self._request(url, params, data, raw_response=True,\n retry_on_error=retry_on_error, method=method)"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef request_json(self, url, params=None, data=None, as_objects=True,\n retry_on_error=True, method=None):\n \"\"\"Get the JSON processed from a page.\n\n :param url: the url to grab content from.\n :param params: a dictionary containing the GET data to put in the url\n :param data: a dictionary containing the extra data to submit\n :param as_objects: if True return reddit objects else raw json dict.\n :param retry_on_error: if True retry the request, if it fails, for up\n to 3 attempts\n :returns: JSON processed page\n\n \"\"\"\n if not url.endswith('.json'):\n url += '.json'\n response = self._request(url, params, data, method=method,\n retry_on_error=retry_on_error)\n hook = self._json_reddit_objecter if as_objects else None\n # Request url just needs to be available for the objecter to use\n self._request_url = url # pylint: disable=W0201\n\n if response == '':\n # Some of the v1 urls don't return anything, even when they're\n # successful.\n return response\n\n data = json.loads(response, object_hook=hook)\n delattr(self, '_request_url')\n # Update the modhash\n if isinstance(data, dict) and 'data' in data \\\n and 'modhash' in data['data']:\n self.modhash = data['data']['modhash']\n return data", "response": "Request the content from a url and return the JSON processed page."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_access_information(self, code):\n if self.config.grant_type == 'password':\n data = {'grant_type': 'password',\n 'username': self.config.user,\n 'password': self.config.pswd}\n else:\n data = {'code': code, 'grant_type': 'authorization_code',\n 'redirect_uri': self.redirect_uri}\n retval = self._handle_oauth_request(data)\n return {'access_token': retval['access_token'],\n 'refresh_token': retval.get('refresh_token'),\n 'scope': set(retval['scope'].split(' '))}", "response": "Return the access information for an OAuth2 authorization grant."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning the URL to send the user to for OAuth2 authorization.", "response": "def get_authorize_url(self, state, scope='identity', refreshable=False):\n \"\"\"Return the URL to send the user to for OAuth2 authorization.\n\n :param state: a unique string of your choice that represents this\n individual client\n :param scope: the reddit scope to ask permissions for. Multiple scopes\n can be enabled by passing in a container of strings.\n :param refreshable: when True, a permanent \"refreshable\" token is\n issued\n\n \"\"\"\n params = {'client_id': self.client_id, 'response_type': 'code',\n 'redirect_uri': self.redirect_uri, 'state': state,\n 'scope': _to_reddit_list(scope)}\n params['duration'] = 'permanent' if refreshable else 'temporary'\n request = Request('GET', self.config['authorize'], params=params)\n return request.prepare().url"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef refresh_access_information(self, refresh_token):\n if self.config.grant_type == 'password':\n data = {'grant_type': 'password',\n 'username': self.config.user,\n 'password': self.config.pswd}\n else:\n data = {'grant_type': 'refresh_token',\n 'redirect_uri': self.redirect_uri,\n 'refresh_token': refresh_token}\n retval = self._handle_oauth_request(data)\n return {'access_token': retval['access_token'],\n 'refresh_token': refresh_token,\n 'scope': set(retval['scope'].split(' '))}", "response": "Return the updated access information for an OAuth2 authorization grant."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nsetting the client_id client_secret and redirect_uri values for the application.", "response": "def set_oauth_app_info(self, client_id, client_secret, redirect_uri):\n \"\"\"Set the app information to use with OAuth2.\n\n This function need only be called if your praw.ini site configuration\n does not already contain the necessary information.\n\n Go to https://www.reddit.com/prefs/apps/ to discover the appropriate\n values for your application.\n\n :param client_id: the client_id of your application\n :param client_secret: the client_secret of your application\n :param redirect_uri: the redirect_uri of your application\n\n \"\"\"\n self.client_id = client_id\n self.client_secret = client_secret\n self.redirect_uri = redirect_uri"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef create_redditor(self, user_name, password, email=''):\n data = {'email': email,\n 'passwd': password,\n 'passwd2': password,\n 'user': user_name}\n return self.request_json(self.config['register'], data=data)", "response": "Register a new user."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef default_subreddits(self, *args, **kwargs):\n url = self.config['default_subreddits']\n return self.get_content(url, *args, **kwargs)", "response": "Return a get_content generator for the default subreddits."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get_comments(self, subreddit, gilded_only=False, *args, **kwargs):\n key = 'sub_comments_gilded' if gilded_only else 'subreddit_comments'\n url = self.config[key].format(subreddit=six.text_type(subreddit))\n return self.get_content(url, *args, **kwargs)", "response": "Return a get_content generator for comments in the given subreddit."} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns a get_content generator for controversial submissions.", "response": "def get_controversial(self, *args, **kwargs):\n \"\"\"Return a get_content generator for controversial submissions.\n\n Corresponds to submissions provided by\n ``https://www.reddit.com/controversial/`` for the session.\n\n The additional parameters are passed directly into\n :meth:`.get_content`. Note: the `url` parameter cannot be altered.\n\n \"\"\"\n return self.get_content(self.config['controversial'], *args, **kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_domain_listing(self, domain, sort='hot', period=None, *args,\n **kwargs):\n \"\"\"Return a get_content generator for submissions by domain.\n\n Corresponds to the submissions provided by\n ``https://www.reddit.com/domain/{domain}``.\n\n :param domain: The domain to generate a submission listing for.\n :param sort: When provided must be one of 'hot', 'new', 'rising',\n 'controversial, 'gilded', or 'top'. Defaults to 'hot'.\n :param period: When sort is either 'controversial', or 'top' the period\n can be either None (for account default), 'all', 'year', 'month',\n 'week', 'day', or 'hour'.\n\n The additional parameters are passed directly into\n :meth:`.get_content`. Note: the `url` parameter cannot be altered.\n\n \"\"\"\n # Verify arguments\n if sort not in ('controversial', 'hot', 'new', 'rising', 'top',\n 'gilded'):\n raise TypeError('Invalid sort parameter.')\n if period not in (None, 'all', 'day', 'hour', 'month', 'week', 'year'):\n raise TypeError('Invalid period parameter.')\n if sort not in ('controversial', 'top') and period:\n raise TypeError('Period cannot be set for that sort argument.')\n\n url = self.config['domain'].format(domain=domain)\n if sort != 'hot':\n url += sort\n if period: # Set or overwrite params 't' parameter\n kwargs.setdefault('params', {})['t'] = period\n return self.get_content(url, *args, **kwargs)", "response": "Return a get_content generator for submissions by domain."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning the flair for a user on the given subreddit.", "response": "def get_flair(self, subreddit, redditor, **params):\n \"\"\"Return the flair for a user on the given subreddit.\n\n :param subreddit: Can be either a Subreddit object or the name of a\n subreddit.\n :param redditor: Can be either a Redditor object or the name of a\n redditor.\n :returns: None if the user doesn't exist, otherwise a dictionary\n containing the keys `flair_css_class`, `flair_text`, and `user`.\n\n \"\"\"\n name = six.text_type(redditor)\n params.update(name=name)\n url = self.config['flairlist'].format(\n subreddit=six.text_type(subreddit))\n data = self.request_json(url, params=params)\n if not data['users'] or \\\n data['users'][0]['user'].lower() != name.lower():\n return None\n return data['users'][0]"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_front_page(self, *args, **kwargs):\n return self.get_content(self.config['reddit_url'], *args, **kwargs)", "response": "Return a get_content generator for the front page submissions."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nlook up existing items by thing_id (fullname) or url. :param url: A url to lookup. :param thing_id: A single thing_id, or a list of thing_ids. A thing_id can be any one of Comment (``t1_``), Link (``t3_``), or Subreddit (``t5_``) to lookup by fullname. :returns: When a single ``thing_id`` is provided, return the corresponding thing object, or ``None`` if not found. When a list of ``thing_id``s or a ``url`` is provided return a list of thing objects (up to ``limit``). ``None`` is returned if all of the thing_ids or the URL is invalid. The additional parameters are passed into :meth:`.get_content` after the `params` parameter is exctracted and used to update the dictionary of url parameters this function sends. Note: the `url` parameter cannot be altered. Also, if using thing_id and the `limit` parameter passed to :meth:`.get_content` is used to slice the list of retreived things before returning it to the user, for when `limit > 100` and `(limit % 100) > 0`, to ensure a maximum of `limit` thigns are returned.", "response": "def get_info(self, url=None, thing_id=None, *args, **kwargs):\n \"\"\"Look up existing items by thing_id (fullname) or url.\n\n :param url: A url to lookup.\n :param thing_id: A single thing_id, or a list of thing_ids. A thing_id\n can be any one of Comment (``t1_``), Link (``t3_``), or Subreddit\n (``t5_``) to lookup by fullname.\n :returns: When a single ``thing_id`` is provided, return the\n corresponding thing object, or ``None`` if not found. When a list\n of ``thing_id``s or a ``url`` is provided return a list of thing\n objects (up to ``limit``). ``None`` is returned if all of the\n thing_ids or the URL is invalid.\n\n The additional parameters are passed into :meth:`.get_content` after\n the `params` parameter is exctracted and used to update the dictionary\n of url parameters this function sends. Note: the `url` parameter\n cannot be altered.\n\n Also, if using thing_id and the `limit` parameter passed to\n :meth:`.get_content` is used to slice the list of retreived things\n before returning it to the user, for when `limit > 100` and\n `(limit % 100) > 0`, to ensure a maximum of `limit` thigns are\n returned.\n\n \"\"\"\n if bool(url) == bool(thing_id):\n raise TypeError('Only one of url or thing_id is required!')\n\n # In these cases, we will have a list of things to return.\n # Otherwise, it will just be one item.\n if isinstance(thing_id, six.string_types) and ',' in thing_id:\n thing_id = thing_id.split(',')\n return_list = bool(url) or not isinstance(thing_id, six.string_types)\n\n if url:\n param_groups = [{'url': url}]\n else:\n if isinstance(thing_id, six.string_types):\n thing_id = [thing_id]\n id_chunks = chunk_sequence(thing_id, 100)\n param_groups = [{'id': ','.join(id_chunk)} for\n id_chunk in id_chunks]\n\n items = []\n update_with = kwargs.pop('params', {})\n for param_group in param_groups:\n param_group.update(update_with)\n kwargs['params'] = param_group\n chunk = self.get_content(self.config['info'], *args, **kwargs)\n items.extend(list(chunk))\n\n # if using ids, manually set the limit\n if kwargs.get('limit'):\n items = items[:kwargs['limit']]\n\n if return_list:\n return items if items else None\n elif items:\n return items[0]\n else:\n return None"} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn the list of moderators for the given subreddit.", "response": "def get_moderators(self, subreddit, **kwargs):\n \"\"\"Return the list of moderators for the given subreddit.\"\"\"\n url = self.config['moderators'].format(\n subreddit=six.text_type(subreddit))\n return self.request_json(url, **kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_new(self, *args, **kwargs):\n return self.get_content(self.config['new'], *args, **kwargs)", "response": "Return a get_content generator for new submissions."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_new_subreddits(self, *args, **kwargs):\n url = self.config['new_subreddits']\n return self.get_content(url, *args, **kwargs)", "response": "Return a get_content generator for the newest subreddits."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a get_content generator for the most active subreddits.", "response": "def get_popular_subreddits(self, *args, **kwargs):\n \"\"\"Return a get_content generator for the most active subreddits.\n\n The additional parameters are passed directly into\n :meth:`.get_content`. Note: the `url` parameter cannot be altered.\n\n \"\"\"\n url = self.config['popular_subreddits']\n return self.get_content(url, *args, **kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_random_subreddit(self, nsfw=False):\n path = 'random'\n if nsfw:\n self.http.cookies.set('over18', '1')\n path = 'randnsfw'\n url = self.config['subreddit'].format(subreddit=path)\n response = self._request(url, params={'unique': self._unique_count},\n raw_response=True)\n self._unique_count += 1\n return self.get_subreddit(response.url.rsplit('/', 2)[-2])", "response": "Return a random Subreddit object."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a random Submission object.", "response": "def get_random_submission(self, subreddit='all'):\n \"\"\"Return a random Submission object.\n\n :param subreddit: Limit the submission to the specified\n subreddit(s). Default: all\n\n \"\"\"\n url = self.config['subreddit_random'].format(\n subreddit=six.text_type(subreddit))\n try:\n item = self.request_json(url,\n params={'unique': self._unique_count})\n self._unique_count += 1 # Avoid network-level caching\n return objects.Submission.from_json(item)\n except errors.RedirectException as exc:\n self._unique_count += 1\n return self.get_submission(exc.response_url)\n raise errors.ClientException('Expected exception not raised.')"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_redditor(self, user_name, *args, **kwargs):\n return objects.Redditor(self, user_name, *args, **kwargs)", "response": "Returns a Redditor instance for the user_name specified."} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn a get_content generator for rising submissions.", "response": "def get_rising(self, *args, **kwargs):\n \"\"\"Return a get_content generator for rising submissions.\n\n Corresponds to the submissions provided by\n ``https://www.reddit.com/rising/`` for the session.\n\n The additional parameters are passed directly into\n :meth:`.get_content`. Note: the `url` parameter cannot be altered.\n\n \"\"\"\n return self.get_content(self.config['rising'], *args, **kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_rules(self, subreddit, bottom=False):\n url = self.config['rules'].format(subreddit=six.text_type(subreddit))\n return self.request_json(url)", "response": "Return the json dictionary containing rules for a subreddit."} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn a Submission object for the sticky of the subreddit.", "response": "def get_sticky(self, subreddit, bottom=False):\n \"\"\"Return a Submission object for the sticky of the subreddit.\n\n :param bottom: Get the top or bottom sticky. If the subreddit has only\n a single sticky, it is considered the top one.\n\n \"\"\"\n url = self.config['sticky'].format(subreddit=six.text_type(subreddit))\n param = {'num': 2} if bottom else None\n return objects.Submission.from_json(self.request_json(url,\n params=param))"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning a Submission object for the given url or submission_id.", "response": "def get_submission(self, url=None, submission_id=None, comment_limit=0,\n comment_sort=None, params=None):\n \"\"\"Return a Submission object for the given url or submission_id.\n\n :param comment_limit: The desired number of comments to fetch. If <= 0\n fetch the default number for the session's user. If None, fetch the\n maximum possible.\n :param comment_sort: The sort order for retrieved comments. When None\n use the default for the session's user.\n :param params: Dictionary containing extra GET data to put in the url.\n\n \"\"\"\n if bool(url) == bool(submission_id):\n raise TypeError('One (and only one) of id or url is required!')\n if submission_id:\n url = urljoin(self.config['comments'], submission_id)\n return objects.Submission.from_url(self, url,\n comment_limit=comment_limit,\n comment_sort=comment_sort,\n params=params)"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nyield Submission objects for each item provided in fullnames.", "response": "def get_submissions(self, fullnames, *args, **kwargs):\n \"\"\"Generate Submission objects for each item provided in `fullnames`.\n\n A submission fullname looks like `t3_`. Submissions are\n yielded in the same order they appear in `fullnames`.\n\n Up to 100 items are batched at a time -- this happens transparently.\n\n The additional parameters are passed directly into\n :meth:`.get_content`. Note: the `url` and `limit` parameters cannot be\n altered.\n\n \"\"\"\n fullnames = fullnames[:]\n while fullnames:\n cur = fullnames[:100]\n fullnames[:100] = []\n url = self.config['by_id'] + ','.join(cur)\n for item in self.get_content(url, limit=len(cur), *args, **kwargs):\n yield item"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_subreddit(self, subreddit_name, *args, **kwargs):\n sr_name_lower = subreddit_name.lower()\n if sr_name_lower == 'random':\n return self.get_random_subreddit()\n elif sr_name_lower == 'randnsfw':\n return self.get_random_subreddit(nsfw=True)\n return objects.Subreddit(self, subreddit_name, *args, **kwargs)", "response": "Return a Subreddit object for the subreddit_name specified."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a list of recommended subreddits as Subreddit objects.", "response": "def get_subreddit_recommendations(self, subreddits, omit=None):\n \"\"\"Return a list of recommended subreddits as Subreddit objects.\n\n Subreddits with activity less than a certain threshold, will not have\n any recommendations due to lack of data.\n\n :param subreddits: A list of subreddits (either names or Subreddit\n objects) to base the recommendations on.\n :param omit: A list of subreddits (either names or Subreddit\n objects) that will be filtered out of the result.\n\n \"\"\"\n params = {'omit': _to_reddit_list(omit or [])}\n url = self.config['sub_recommendations'].format(\n subreddits=_to_reddit_list(subreddits))\n result = self.request_json(url, params=params)\n return [objects.Subreddit(self, sub['sr_name']) for sub in result]"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a get_content generator for top submissions.", "response": "def get_top(self, *args, **kwargs):\n \"\"\"Return a get_content generator for top submissions.\n\n Corresponds to the submissions provided by\n ``https://www.reddit.com/top/`` for the session.\n\n The additional parameters are passed directly into\n :meth:`.get_content`. Note: the `url` parameter cannot be altered.\n\n \"\"\"\n return self.get_content(self.config['top'], *args, **kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_gilded(self, *args, **kwargs):\n return self.get_content(self.config['gilded'], *args, **kwargs)", "response": "Return a get_content generator for gilded submissions."} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns the json dictionary containing the traffic stats for a subreddit.", "response": "def get_traffic(self, subreddit):\n \"\"\"Return the json dictionary containing traffic stats for a subreddit.\n\n :param subreddit: The subreddit whose /about/traffic page we will\n collect.\n\n \"\"\"\n url = self.config['subreddit_traffic'].format(\n subreddit=six.text_type(subreddit))\n return self.request_json(url)"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_wiki_page(self, subreddit, page):\n return objects.WikiPage(self, six.text_type(subreddit), page.lower())", "response": "Return a WikiPage object for the subreddit and page provided."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns True if username is valid and available otherwise False.", "response": "def is_username_available(self, username):\n \"\"\"Return True if username is valid and available, otherwise False.\"\"\"\n params = {'user': username}\n try:\n result = self.request_json(self.config['username_available'],\n params=params)\n except errors.BadUsername:\n return False\n return result"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a generator for submissions that match the search query.", "response": "def search(self, query, subreddit=None, sort=None, syntax=None,\n period=None, *args, **kwargs):\n \"\"\"Return a generator for submissions that match the search query.\n\n :param query: The query string to search for. If query is a URL only\n submissions which link to that URL will be returned.\n :param subreddit: Limit search results to the subreddit if provided.\n :param sort: The sort order of the results.\n :param syntax: The syntax of the search query.\n :param period: The time period of the results.\n\n The additional parameters are passed directly into\n :meth:`.get_content`. Note: the `url` parameter cannot be altered.\n\n See https://www.reddit.com/wiki/search for more information on how to\n build a search query.\n\n \"\"\"\n params = {'q': query}\n if 'params' in kwargs:\n params.update(kwargs['params'])\n kwargs.pop('params')\n if sort:\n params['sort'] = sort\n if syntax:\n params['syntax'] = syntax\n if period:\n params['t'] = period\n if subreddit:\n params['restrict_sr'] = 'on'\n subreddit = six.text_type(subreddit)\n else:\n subreddit = 'all'\n url = self.config['search'].format(subreddit=subreddit)\n\n depth = 2\n while depth > 0:\n depth -= 1\n try:\n for item in self.get_content(url, params=params, *args,\n **kwargs):\n yield item\n break\n except errors.RedirectException as exc:\n parsed = urlparse(exc.response_url)\n params = dict((k, \",\".join(v)) for k, v in\n parse_qs(parsed.query).items())\n url = urlunparse(parsed[:3] + (\"\", \"\", \"\"))\n # Handle redirects from URL searches\n if 'already_submitted' in params:\n yield self.get_submission(url)\n break"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef search_reddit_names(self, query):\n data = {'query': query}\n results = self.request_json(self.config['search_reddit_names'],\n data=data)\n return [self.get_subreddit(name) for name in results['names']]", "response": "Return a list of subreddits whose display name contains the query."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\naccept a moderator invite to the given subreddit. Callable upon an instance of Subreddit with no arguments. :returns: The json response from the server.", "response": "def accept_moderator_invite(self, subreddit):\n \"\"\"Accept a moderator invite to the given subreddit.\n\n Callable upon an instance of Subreddit with no arguments.\n\n :returns: The json response from the server.\n\n \"\"\"\n data = {'r': six.text_type(subreddit)}\n # Clear moderated subreddits and cache\n self.user._mod_subs = None # pylint: disable=W0212\n self.evict(self.config['my_mod_subreddits'])\n return self.request_json(self.config['accept_mod_invite'], data=data)"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nclearing any existing authentication on the object.", "response": "def clear_authentication(self):\n \"\"\"Clear any existing authentication on the reddit object.\n\n This function is implicitly called on `login` and\n `set_access_credentials`.\n\n \"\"\"\n self._authentication = None\n self.access_token = None\n self.refresh_token = None\n self.http.cookies.clear()\n self.user = None"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef delete(self, password, message=\"\"):\n data = {'user': self.user.name,\n 'passwd': password,\n 'delete_message': message,\n 'confirm': True}\n return self.request_json(self.config['delete_redditor'], data=data)", "response": "Delete the currently authenticated reddit account."} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncreates or edit a wiki page with title page for subreddit.", "response": "def edit_wiki_page(self, subreddit, page, content, reason=''):\n \"\"\"Create or edit a wiki page with title `page` for `subreddit`.\n\n :returns: The json response from the server.\n\n \"\"\"\n data = {'content': content,\n 'page': page,\n 'r': six.text_type(subreddit),\n 'reason': reason}\n evict = self.config['wiki_page'].format(\n subreddit=six.text_type(subreddit), page=page.lower())\n self.evict(evict)\n return self.request_json(self.config['wiki_edit'], data=data)"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning the access information for an OAuth2 authorization grant.", "response": "def get_access_information(self, code, # pylint: disable=W0221\n update_session=True):\n \"\"\"Return the access information for an OAuth2 authorization grant.\n\n :param code: the code received in the request from the OAuth2 server\n :param update_session: Update the current session with the retrieved\n token(s).\n :returns: A dictionary with the key/value pairs for access_token,\n refresh_token and scope. The refresh_token value will be done when\n the OAuth2 grant is not refreshable.\n\n \"\"\"\n retval = super(AuthenticatedReddit, self).get_access_information(code)\n if update_session:\n self.set_access_credentials(**retval)\n return retval"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning available flair choices and current flair.", "response": "def get_flair_choices(self, subreddit, link=None):\n \"\"\"Return available flair choices and current flair.\n\n :param link: If link is given, return the flair options for this\n submission. Not normally given directly, but instead set by calling\n the flair_choices method for Submission objects.\n Use the default for the session's user.\n\n :returns: A dictionary with 2 keys. 'current' containing current flair\n settings for the authenticated user and 'choices' containing a list\n of possible flair choices.\n\n \"\"\"\n data = {'r': six.text_type(subreddit), 'link': link}\n return self.request_json(self.config['flairselector'], data=data)"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_friends(self, **params):\n url = self.config['friends']\n return self.request_json(url, params=params)[0]", "response": "Return a UserList of Redditors with whom the user is friends."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_me(self):\n response = self.request_json(self.config['me'])\n user = objects.Redditor(self, response['name'], response)\n user.__class__ = objects.LoggedInRedditor\n return user", "response": "Return a LoggedInRedditor object."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef has_scope(self, scope):\n if not self.is_oauth_session():\n return False\n if '*' in self._authentication:\n return True\n if isinstance(scope, six.string_types):\n scope = [scope]\n return all(s in self._authentication for s in scope)", "response": "Return True if OAuth2 authorized for the passed in scope."} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nlogging-in to a reddit site.", "response": "def login(self, username=None, password=None, **kwargs):\n \"\"\"Login to a reddit site.\n\n **DEPRECATED**. Will be removed in a future version of PRAW.\n\n https://www.reddit.com/comments/2ujhkr/\n https://www.reddit.com/comments/37e2mv/\n\n Look for username first in parameter, then praw.ini and finally if both\n were empty get it from stdin. Look for password in parameter, then\n praw.ini (but only if username matches that in praw.ini) and finally\n if they both are empty get it with getpass. Add the variables ``user``\n (username) and ``pswd`` (password) to your praw.ini file to allow for\n auto-login.\n\n A successful login will overwrite any existing authentication.\n\n \"\"\"\n if password and not username:\n raise Exception('Username must be provided when password is.')\n user = username or self.config.user\n if not user:\n sys.stdout.write('Username: ')\n sys.stdout.flush()\n user = sys.stdin.readline().strip()\n pswd = None\n else:\n pswd = password or self.config.pswd\n if not pswd:\n import getpass\n pswd = getpass.getpass('Password for {0}: '.format(user)\n .encode('ascii', 'ignore'))\n\n data = {'passwd': pswd,\n 'user': user}\n self.clear_authentication()\n self.request_json(self.config['login'], data=data)\n # Update authentication settings\n self._authentication = True\n self.user = self.get_redditor(user)\n self.user.__class__ = objects.LoggedInRedditor"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef refresh_access_information(self, # pylint: disable=W0221\n refresh_token=None,\n update_session=True):\n \"\"\"Return updated access information for an OAuth2 authorization grant.\n\n :param refresh_token: The refresh token used to obtain the updated\n information. When not provided, use the stored refresh_token.\n :param update_session: Update the session with the returned data.\n :returns: A dictionary with the key/value pairs for ``access_token``,\n ``refresh_token`` and ``scope``. The ``refresh_token`` value will\n be None when the OAuth2 grant is not refreshable. The ``scope``\n value will be a set containing the scopes the tokens are valid for.\n\n \"\"\"\n response = super(AuthenticatedReddit, self).refresh_access_information(\n refresh_token=refresh_token or self.refresh_token)\n if update_session:\n self.set_access_credentials(**response)\n return response", "response": "Refreshes the access information for an OAuth2 authorization grant."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nselecting user flair or link flair on subreddits.", "response": "def select_flair(self, item, flair_template_id='', flair_text=''):\n \"\"\"Select user flair or link flair on subreddits.\n\n This can only be used for assigning your own name flair or link flair\n on your own submissions. For assigning other's flairs using moderator\n access, see :meth:`~praw.__init__.ModFlairMixin.set_flair`.\n\n :param item: A string, Subreddit object (for user flair), or\n Submission object (for link flair). If ``item`` is a string it\n will be treated as the name of a Subreddit.\n :param flair_template_id: The id for the desired flair template. Use\n the :meth:`~praw.objects.Subreddit.get_flair_choices` and\n :meth:`~praw.objects.Submission.get_flair_choices` methods to find\n the ids for the available user and link flair choices.\n :param flair_text: A string containing the custom flair text.\n Used on subreddits that allow it.\n\n :returns: The json response from the server.\n\n \"\"\"\n data = {'flair_template_id': flair_template_id or '',\n 'text': flair_text or ''}\n if isinstance(item, objects.Submission):\n # Link flair\n data['link'] = item.fullname\n evict = item.permalink\n else:\n # User flair\n data['name'] = self.user.name\n data['r'] = six.text_type(item)\n evict = self.config['flairlist'].format(\n subreddit=six.text_type(item))\n response = self.request_json(self.config['select_flair'], data=data)\n self.evict(evict)\n return response"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nsets the credentials used for OAuth2 authentication.", "response": "def set_access_credentials(self, scope, access_token, refresh_token=None,\n update_user=True):\n \"\"\"Set the credentials used for OAuth2 authentication.\n\n Calling this function will overwrite any currently existing access\n credentials.\n\n :param scope: A set of reddit scopes the tokens provide access to\n :param access_token: the access token of the authentication\n :param refresh_token: the refresh token of the authentication\n :param update_user: Whether or not to set the user attribute for\n identity scopes\n\n \"\"\"\n if isinstance(scope, (list, tuple)):\n scope = set(scope)\n elif isinstance(scope, six.string_types):\n scope = set(scope.split())\n if not isinstance(scope, set):\n raise TypeError('`scope` parameter must be a set')\n self.clear_authentication()\n # Update authentication settings\n self._authentication = scope\n self.access_token = access_token\n self.refresh_token = refresh_token\n # Update the user object\n if update_user and ('identity' in scope or '*' in scope):\n self.user = self.get_me()"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ncreating a new subreddit.", "response": "def create_subreddit(self, name, title, description='', language='en',\n subreddit_type='public', content_options='any',\n over_18=False, default_set=True, show_media=False,\n domain='', wikimode='disabled', captcha=None,\n **kwargs):\n \"\"\"Create a new subreddit.\n\n :returns: The json response from the server.\n\n This function may result in a captcha challenge. PRAW will\n automatically prompt you for a response. See :ref:`handling-captchas`\n if you want to manually handle captchas.\n\n \"\"\"\n data = {'name': name,\n 'title': title,\n 'description': description,\n 'lang': language,\n 'type': subreddit_type,\n 'link_type': content_options,\n 'over_18': 'on' if over_18 else 'off',\n 'allow_top': 'on' if default_set else 'off',\n 'show_media': 'on' if show_media else 'off',\n 'wikimode': wikimode,\n 'domain': domain}\n if captcha:\n data.update(captcha)\n return self.request_json(self.config['site_admin'], data=data)"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ndeletes an image from the subreddit.", "response": "def delete_image(self, subreddit, name=None, header=False):\n \"\"\"Delete an image from the subreddit.\n\n :param name: The name of the image if removing a CSS image.\n :param header: When true, delete the subreddit header.\n :returns: The json response from the server.\n\n \"\"\"\n subreddit = six.text_type(subreddit)\n if name and header:\n raise TypeError('Both name and header cannot be set.')\n elif name:\n data = {'img_name': name}\n url = self.config['delete_sr_image']\n self.evict(self.config['stylesheet'].format(subreddit=subreddit))\n else:\n data = True\n url = self.config['delete_sr_header']\n url = url.format(subreddit=subreddit)\n return self.request_json(url, data=data)"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef set_settings(self, subreddit, title, public_description='',\n description='', language='en', subreddit_type='public',\n content_options='any', over_18=False, default_set=True,\n show_media=False, domain='', domain_css=False,\n domain_sidebar=False, header_hover_text='',\n wikimode='disabled', wiki_edit_age=30,\n wiki_edit_karma=100,\n submit_link_label='', submit_text_label='',\n exclude_banned_modqueue=False, comment_score_hide_mins=0,\n public_traffic=False, collapse_deleted_comments=False,\n spam_comments='low', spam_links='high',\n spam_selfposts='high', submit_text='',\n hide_ads=False, suggested_comment_sort='',\n key_color='',\n **kwargs):\n \"\"\"Set the settings for the given subreddit.\n\n :param subreddit: Must be a subreddit object.\n :returns: The json response from the server.\n\n \"\"\"\n data = {'sr': subreddit.fullname,\n 'allow_top': default_set,\n 'comment_score_hide_mins': comment_score_hide_mins,\n 'collapse_deleted_comments': collapse_deleted_comments,\n 'description': description,\n 'domain': domain or '',\n 'domain_css': domain_css,\n 'domain_sidebar': domain_sidebar,\n 'exclude_banned_modqueue': exclude_banned_modqueue,\n 'header-title': header_hover_text or '',\n 'hide_ads': hide_ads,\n 'key_color': key_color,\n 'lang': language,\n 'link_type': content_options,\n 'over_18': over_18,\n 'public_description': public_description,\n 'public_traffic': public_traffic,\n 'show_media': show_media,\n 'submit_link_label': submit_link_label or '',\n 'submit_text': submit_text,\n 'submit_text_label': submit_text_label or '',\n 'suggested_comment_sort': suggested_comment_sort or '',\n 'spam_comments': spam_comments,\n 'spam_links': spam_links,\n 'spam_selfposts': spam_selfposts,\n 'title': title,\n 'type': subreddit_type,\n 'wiki_edit_age': six.text_type(wiki_edit_age),\n 'wiki_edit_karma': six.text_type(wiki_edit_karma),\n 'wikimode': wikimode}\n\n if kwargs:\n msg = 'Extra settings fields: {0}'.format(kwargs.keys())\n warn_explicit(msg, UserWarning, '', 0)\n data.update(kwargs)\n evict = self.config['subreddit_settings'].format(\n subreddit=six.text_type(subreddit))\n self.evict(evict)\n return self.request_json(self.config['site_admin'], data=data)", "response": "Set the settings for the given subreddit."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nsetting the stylesheet for the given subreddit.", "response": "def set_stylesheet(self, subreddit, stylesheet):\n \"\"\"Set stylesheet for the given subreddit.\n\n :returns: The json response from the server.\n\n \"\"\"\n subreddit = six.text_type(subreddit)\n data = {'r': subreddit,\n 'stylesheet_contents': stylesheet,\n 'op': 'save'} # Options: save / preview\n self.evict(self.config['stylesheet'].format(subreddit=subreddit))\n return self.request_json(self.config['subreddit_css'], data=data)"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef upload_image(self, subreddit, image_path, name=None,\n header=False, upload_as=None):\n \"\"\"Upload an image to the subreddit.\n\n :param image_path: A path to the jpg or png image you want to upload.\n :param name: The name to provide the image. When None the name will be\n filename less any extension.\n :param header: When True, upload the image as the subreddit header.\n :param upload_as: Must be `'jpg'`, `'png'` or `None`. When None, this\n will match the format of the image itself. In all cases where both\n this value and the image format is not png, reddit will also\n convert the image mode to RGBA. reddit optimizes the image\n according to this value.\n :returns: A link to the uploaded image. Raises an exception otherwise.\n\n \"\"\"\n if name and header:\n raise TypeError('Both name and header cannot be set.')\n if upload_as not in (None, 'png', 'jpg'):\n raise TypeError(\"upload_as must be 'jpg', 'png', or None.\")\n with open(image_path, 'rb') as image:\n image_type = upload_as or _image_type(image)\n data = {'r': six.text_type(subreddit), 'img_type': image_type}\n if header:\n data['header'] = 1\n else:\n if not name:\n name = os.path.splitext(os.path.basename(image.name))[0]\n data['name'] = name\n\n response = json.loads(self._request(\n self.config['upload_image'], data=data, files={'file': image},\n method=to_native_string('POST'), retry_on_error=False))\n\n if response['errors']:\n raise errors.APIException(response['errors'], None)\n return response['img_src']", "response": "Upload an image to the subreddit."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef update_settings(self, subreddit, **kwargs):\n settings = self.get_settings(subreddit)\n settings.update(kwargs)\n del settings['subreddit_id']\n return self.set_settings(subreddit, **settings)", "response": "Update only the given settings for the given subreddit."} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nadding a flair template to the given subreddit.", "response": "def add_flair_template(self, subreddit, text='', css_class='',\n text_editable=False, is_link=False):\n \"\"\"Add a flair template to the given subreddit.\n\n :returns: The json response from the server.\n\n \"\"\"\n data = {'r': six.text_type(subreddit),\n 'text': text,\n 'css_class': css_class,\n 'text_editable': six.text_type(text_editable),\n 'flair_type': 'LINK_FLAIR' if is_link else 'USER_FLAIR'}\n return self.request_json(self.config['flairtemplate'], data=data)"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nclearing flair templates for the given subreddit.", "response": "def clear_flair_templates(self, subreddit, is_link=False):\n \"\"\"Clear flair templates for the given subreddit.\n\n :returns: The json response from the server.\n\n \"\"\"\n data = {'r': six.text_type(subreddit),\n 'flair_type': 'LINK_FLAIR' if is_link else 'USER_FLAIR'}\n return self.request_json(self.config['clearflairtemplates'], data=data)"} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nconfigures the flair setting for the given subreddit.", "response": "def configure_flair(self, subreddit, flair_enabled=False,\n flair_position='right',\n flair_self_assign=False,\n link_flair_enabled=False,\n link_flair_position='left',\n link_flair_self_assign=False):\n \"\"\"Configure the flair setting for the given subreddit.\n\n :returns: The json response from the server.\n\n \"\"\"\n flair_enabled = 'on' if flair_enabled else 'off'\n flair_self_assign = 'on' if flair_self_assign else 'off'\n if not link_flair_enabled:\n link_flair_position = ''\n link_flair_self_assign = 'on' if link_flair_self_assign else 'off'\n data = {'r': six.text_type(subreddit),\n 'flair_enabled': flair_enabled,\n 'flair_position': flair_position,\n 'flair_self_assign_enabled': flair_self_assign,\n 'link_flair_position': link_flair_position,\n 'link_flair_self_assign_enabled': link_flair_self_assign}\n return self.request_json(self.config['flairconfig'], data=data)"} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ndelete the flair for the given user on the given subreddit.", "response": "def delete_flair(self, subreddit, user):\n \"\"\"Delete the flair for the given user on the given subreddit.\n\n :returns: The json response from the server.\n\n \"\"\"\n data = {'r': six.text_type(subreddit),\n 'name': six.text_type(user)}\n return self.request_json(self.config['deleteflair'], data=data)"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get_flair_list(self, subreddit, *args, **kwargs):\n url = self.config['flairlist'].format(\n subreddit=six.text_type(subreddit))\n return self.get_content(url, *args, root_field=None,\n thing_field='users', after_field='next',\n **kwargs)", "response": "Return a get_content generator of flair mappings for the given subreddit."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef set_flair(self, subreddit, item, flair_text='', flair_css_class=''):\n data = {'r': six.text_type(subreddit),\n 'text': flair_text or '',\n 'css_class': flair_css_class or ''}\n if isinstance(item, objects.Submission):\n data['link'] = item.fullname\n evict = item.permalink\n else:\n data['name'] = six.text_type(item)\n evict = self.config['flairlist'].format(\n subreddit=six.text_type(subreddit))\n response = self.request_json(self.config['flair'], data=data)\n self.evict(evict)\n return response", "response": "Set flair for the user in the given subreddit."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nsetting flair for a group of users in the given subreddit.", "response": "def set_flair_csv(self, subreddit, flair_mapping):\n \"\"\"Set flair for a group of users in the given subreddit.\n\n flair_mapping should be a list of dictionaries with the following keys:\n `user`: the user name,\n `flair_text`: the flair text for the user (optional),\n `flair_css_class`: the flair css class for the user (optional)\n\n :returns: The json response from the server.\n\n \"\"\"\n if not flair_mapping:\n raise errors.ClientException('flair_mapping must be set')\n item_order = ['user', 'flair_text', 'flair_css_class']\n lines = []\n for mapping in flair_mapping:\n if 'user' not in mapping:\n raise errors.ClientException('flair_mapping must '\n 'contain `user` key')\n lines.append(','.join([mapping.get(x, '') for x in item_order]))\n response = []\n while len(lines):\n data = {'r': six.text_type(subreddit),\n 'flair_csv': '\\n'.join(lines[:100])}\n response.extend(self.request_json(self.config['flaircsv'],\n data=data))\n lines = lines[100:]\n evict = self.config['flairlist'].format(\n subreddit=six.text_type(subreddit))\n self.evict(evict)\n return response"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get_mod_log(self, subreddit, mod=None, action=None, *args, **kwargs):\n params = kwargs.setdefault('params', {})\n if mod is not None:\n params['mod'] = six.text_type(mod)\n if action is not None:\n params['type'] = six.text_type(action)\n url = self.config['modlog'].format(subreddit=six.text_type(subreddit))\n return self.get_content(url, *args, **kwargs)", "response": "Return a get_content generator for moderation log items."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn a get_content generator of banned users for the subreddit.", "response": "def get_banned(self, subreddit, user_only=True, *args, **kwargs):\n \"\"\"Return a get_content generator of banned users for the subreddit.\n\n :param subreddit: The subreddit to get the banned user list for.\n :param user_only: When False, the generator yields a dictionary of data\n associated with the server response for that user. In such cases,\n the Redditor will be in key 'name' (default: True).\n\n \"\"\"\n url = self.config['banned'].format(subreddit=six.text_type(subreddit))\n return self._get_userlist(url, user_only, *args, **kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get_contributors(self, subreddit, *args, **kwargs):\n # pylint: disable=W0613\n def get_contributors_helper(self, subreddit):\n # It is necessary to have the 'self' argument as it's needed in\n # restrict_access to determine what class the decorator is\n # operating on.\n url = self.config['contributors'].format(\n subreddit=six.text_type(subreddit))\n return self._get_userlist(url, user_only=True, *args, **kwargs)\n\n if self.is_logged_in():\n if not isinstance(subreddit, objects.Subreddit):\n subreddit = self.get_subreddit(subreddit)\n if subreddit.subreddit_type == \"public\":\n decorator = decorators.restrict_access(scope='read', mod=True)\n return decorator(get_contributors_helper)(self, subreddit)\n return get_contributors_helper(self, subreddit)", "response": "Return a get_content generator of contributors for the given subreddit."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_edited(self, subreddit='mod', *args, **kwargs):\n url = self.config['edited'].format(subreddit=six.text_type(subreddit))\n return self.get_content(url, *args, **kwargs)", "response": "Return a get_content generator of edited items."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef get_stylesheet(self, subreddit, **params):\n url = self.config['stylesheet'].format(\n subreddit=six.text_type(subreddit))\n return self.request_json(url, params=params)['data']", "response": "Return the stylesheet and images for the given subreddit."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef leave_moderator(self, subreddit):\n self.evict(self.config['my_mod_subreddits'])\n return self._leave_status(subreddit, self.config['leavemoderator'])", "response": "Abdicate moderator status in a subreddit. Use with care."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _leave_status(self, subreddit, statusurl):\n if isinstance(subreddit, six.string_types):\n subreddit = self.get_subreddit(subreddit)\n\n data = {'id': subreddit.fullname}\n return self.request_json(statusurl, data=data)", "response": "Abdicate status in a subreddit."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ncopy a multireddit. :param from_redditor: The username or Redditor object for the user who owns the original multireddit :param from_name: The name of the multireddit, belonging to from_redditor :param to_name: The name to copy the multireddit as. If None, uses the name of the original The additional parameters are passed directly into :meth:`~praw.__init__.BaseReddit.request_json`", "response": "def copy_multireddit(self, from_redditor, from_name, to_name=None,\n *args, **kwargs):\n \"\"\"Copy a multireddit.\n\n :param from_redditor: The username or Redditor object for the user\n who owns the original multireddit\n :param from_name: The name of the multireddit, belonging to\n from_redditor\n :param to_name: The name to copy the multireddit as. If None, uses\n the name of the original\n\n The additional parameters are passed directly into\n :meth:`~praw.__init__.BaseReddit.request_json`\n\n \"\"\"\n if to_name is None:\n to_name = from_name\n\n from_multipath = self.MULTI_PATH.format(from_redditor, from_name)\n to_multipath = self.MULTI_PATH.format(self.user.name, to_name)\n data = {'display_name': to_name,\n 'from': from_multipath,\n 'to': to_multipath}\n return self.request_json(self.config['multireddit_copy'], data=data,\n *args, **kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncreates a new Multireddit with the specified parameters.", "response": "def create_multireddit(self, name, description_md=None, icon_name=None,\n key_color=None, subreddits=None, visibility=None,\n weighting_scheme=None, overwrite=False,\n *args, **kwargs): # pylint: disable=W0613\n \"\"\"Create a new multireddit.\n\n :param name: The name of the new multireddit.\n :param description_md: Optional description for the multireddit,\n formatted in markdown.\n :param icon_name: Optional, choose an icon name from this list: ``art\n and design``, ``ask``, ``books``, ``business``, ``cars``,\n ``comics``, ``cute animals``, ``diy``, ``entertainment``, ``food\n and drink``, ``funny``, ``games``, ``grooming``, ``health``, ``life\n advice``, ``military``, ``models pinup``, ``music``, ``news``,\n ``philosophy``, ``pictures and gifs``, ``science``, ``shopping``,\n ``sports``, ``style``, ``tech``, ``travel``, ``unusual stories``,\n ``video``, or ``None``.\n :param key_color: Optional rgb hex color code of the form `#xxxxxx`.\n :param subreddits: Optional list of subreddit names or Subreddit\n objects to initialize the Multireddit with. You can always\n add more later with\n :meth:`~praw.objects.Multireddit.add_subreddit`.\n :param visibility: Choose a privacy setting from this list:\n ``public``, ``private``, ``hidden``. Defaults to private if blank.\n :param weighting_scheme: Choose a weighting scheme from this list:\n ``classic``, ``fresh``. Defaults to classic if blank.\n :param overwrite: Allow for overwriting / updating multireddits.\n If False, and the multi name already exists, throw 409 error.\n If True, and the multi name already exists, use the given\n properties to update that multi.\n If True, and the multi name does not exist, create it normally.\n\n :returns: The newly created Multireddit object.\n\n The additional parameters are passed directly into\n :meth:`~praw.__init__.BaseReddit.request_json`\n\n \"\"\"\n url = self.config['multireddit_about'].format(user=self.user.name,\n multi=name)\n if subreddits:\n subreddits = [{'name': six.text_type(sr)} for sr in subreddits]\n model = {}\n for key in ('description_md', 'icon_name', 'key_color', 'subreddits',\n 'visibility', 'weighting_scheme'):\n value = locals()[key]\n if value:\n model[key] = value\n\n method = 'PUT' if overwrite else 'POST'\n return self.request_json(url, data={'model': json.dumps(model)},\n method=method, *args, **kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef delete_multireddit(self, name, *args, **kwargs):\n url = self.config['multireddit_about'].format(user=self.user.name,\n multi=name)\n\n # The modhash isn't necessary for OAuth requests\n if not self._use_oauth:\n self.http.headers['x-modhash'] = self.modhash\n\n try:\n self.request(url, data={}, method='DELETE', *args, **kwargs)\n finally:\n if not self._use_oauth:\n del self.http.headers['x-modhash']", "response": "Delete a Multireddit.\n\n Any additional parameters are passed directly into\n :meth:`~praw.__init__.BaseReddit.request`"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef edit_multireddit(self, *args, **kwargs):\n return self.create_multireddit(*args, overwrite=True, **kwargs)", "response": "Edit a multireddit or create one if it doesn t already exist."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_multireddit(self, redditor, multi, *args, **kwargs):\n return objects.Multireddit(self, six.text_type(redditor), multi,\n *args, **kwargs)", "response": "Return a Multireddit object for the author and name specified."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning a list of multireddits belonging to a user.", "response": "def get_multireddits(self, redditor, *args, **kwargs):\n \"\"\"Return a list of multireddits belonging to a redditor.\n\n :param redditor: The username or Redditor object to find multireddits\n from.\n :returns: The json response from the server\n\n The additional parameters are passed directly into\n :meth:`~praw.__init__.BaseReddit.request_json`\n\n If the requested redditor is the current user, all multireddits\n are visible. Otherwise, only public multireddits are returned.\n\n \"\"\"\n redditor = six.text_type(redditor)\n url = self.config['multireddit_user'].format(user=redditor)\n return self.request_json(url, *args, **kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef rename_multireddit(self, current_name, new_name, *args, **kwargs):\n current_path = self.MULTI_PATH.format(self.user.name, current_name)\n new_path = self.MULTI_PATH.format(self.user.name, new_name)\n data = {'from': current_path,\n 'to': new_path}\n return self.request_json(self.config['multireddit_rename'], data=data,\n *args, **kwargs)", "response": "Rename a Multireddit.\n\n :param current_name: The name of the multireddit to rename\n :param new_name: The new name to assign to this multireddit\n\n The additional parameters are passed directly into\n :meth:`~praw.__init__.BaseReddit.request_json`"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning a get_content generator of subreddits.", "response": "def get_my_contributions(self, *args, **kwargs):\n \"\"\"Return a get_content generator of subreddits.\n\n The Subreddits generated are those where the session's user is a\n contributor.\n\n The additional parameters are passed directly into\n :meth:`.get_content`. Note: the `url` parameter cannot be altered.\n\n \"\"\"\n return self.get_content(self.config['my_con_subreddits'], *args,\n **kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn a get_content generator of subreddits.", "response": "def get_my_moderation(self, *args, **kwargs):\n \"\"\"Return a get_content generator of subreddits.\n\n The Subreddits generated are those where the session's user is a\n moderator.\n\n The additional parameters are passed directly into\n :meth:`.get_content`. Note: the `url` parameter cannot be altered.\n\n \"\"\"\n return self.get_content(self.config['my_mod_subreddits'], *args,\n **kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_my_subreddits(self, *args, **kwargs):\n return self.get_content(self.config['my_subreddits'], *args, **kwargs)", "response": "Return a get_content generator of subreddits."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _mark_as_read(self, thing_ids, unread=False):\n data = {'id': ','.join(thing_ids)}\n key = 'unread_message' if unread else 'read_message'\n response = self.request_json(self.config[key], data=data)\n self.evict([self.config[x] for x in ['inbox', 'messages',\n 'mod_mail', 'unread']])\n return response", "response": "Mark each of the supplied thing_ids as read."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_comment_replies(self, *args, **kwargs):\n return self.get_content(self.config['comment_replies'],\n *args, **kwargs)", "response": "Return a get_content generator for inboxed comment replies."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_inbox(self, *args, **kwargs):\n return self.get_content(self.config['inbox'], *args, **kwargs)", "response": "Return a get_content generator for inbox messages and comments."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a Message object corresponding to the given ID.", "response": "def get_message(self, message_id, *args, **kwargs):\n \"\"\"Return a Message object corresponding to the given ID.\n\n :param message_id: The ID or Fullname for a Message\n\n The additional parameters are passed directly into\n :meth:`~praw.objects.Message.from_id` of Message, and subsequently into\n :meth:`.request_json`.\n\n \"\"\"\n return objects.Message.from_id(self, message_id, *args, **kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning a get_content generator for inbox.", "response": "def get_messages(self, *args, **kwargs):\n \"\"\"Return a get_content generator for inbox (messages only).\n\n The additional parameters are passed directly into\n :meth:`.get_content`. Note: the `url` parameter cannot be altered.\n\n \"\"\"\n return self.get_content(self.config['messages'], *args, **kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_post_replies(self, *args, **kwargs):\n return self.get_content(self.config['post_replies'], *args, **kwargs)", "response": "Return a get_content generator for inboxed submission replies."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning a get_content generator for sent messages.", "response": "def get_sent(self, *args, **kwargs):\n \"\"\"Return a get_content generator for sent messages.\n\n The additional parameters are passed directly into\n :meth:`.get_content`. Note: the `url` parameter cannot be altered.\n\n \"\"\"\n return self.get_content(self.config['sent'], *args, **kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a generator for unread messages.", "response": "def get_unread(self, unset_has_mail=False, update_user=False, *args,\n **kwargs):\n \"\"\"Return a get_content generator for unread messages.\n\n :param unset_has_mail: When True, clear the has_mail flag (orangered)\n for the user.\n :param update_user: If both `unset_has_mail` and `update user` is True,\n set the `has_mail` attribute of the logged-in user to False.\n\n The additional parameters are passed directly into\n :meth:`.get_content`. Note: the `url` parameter cannot be altered.\n\n \"\"\"\n params = kwargs.setdefault('params', {})\n if unset_has_mail:\n params['mark'] = 'true'\n if update_user: # Update the user object\n # Use setattr to avoid pylint error\n setattr(self.user, 'has_mail', False)\n return self.get_content(self.config['unread'], *args, **kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning a get_content generator for username mentions.", "response": "def get_mentions(self, *args, **kwargs):\n \"\"\"Return a get_content generator for username mentions.\n\n The additional parameters are passed directly into\n :meth:`.get_content`. Note: the `url` parameter cannot be altered.\n\n \"\"\"\n return self.get_content(self.config['mentions'], *args, **kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nsends a message to a reddit or a subreddit.", "response": "def send_message(self, recipient, subject, message, from_sr=None,\n captcha=None, **kwargs):\n \"\"\"Send a message to a redditor or a subreddit's moderators (mod mail).\n\n :param recipient: A Redditor or Subreddit instance to send a message\n to. A string can also be used in which case the string is treated\n as a redditor unless it is prefixed with either '/r/' or '#', in\n which case it will be treated as a subreddit.\n :param subject: The subject of the message to send.\n :param message: The actual message content.\n :param from_sr: A Subreddit instance or string to send the message\n from. When provided, messages are sent from the subreddit rather\n than from the authenticated user. Note that the authenticated user\n must be a moderator of the subreddit and have mail permissions.\n\n :returns: The json response from the server.\n\n This function may result in a captcha challenge. PRAW will\n automatically prompt you for a response. See :ref:`handling-captchas`\n if you want to manually handle captchas.\n\n \"\"\"\n if isinstance(recipient, objects.Subreddit):\n recipient = '/r/{0}'.format(six.text_type(recipient))\n else:\n recipient = six.text_type(recipient)\n\n data = {'text': message,\n 'subject': subject,\n 'to': recipient}\n if from_sr:\n data['from_sr'] = six.text_type(from_sr)\n if captcha:\n data.update(captcha)\n response = self.request_json(self.config['compose'], data=data,\n retry_on_error=False)\n self.evict(self.config['sent'])\n return response"} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nhide one or multiple objects in the context of the logged in user.", "response": "def hide(self, thing_id, _unhide=False):\n \"\"\"Hide one or multiple objects in the context of the logged in user.\n\n :param thing_id: A single fullname or list of fullnames,\n representing objects which will be hidden.\n :param _unhide: If True, unhide the object(s) instead. Use\n :meth:`~praw.__init__.ReportMixin.unhide` rather than setting this\n manually.\n\n :returns: The json response from the server.\n\n \"\"\"\n if isinstance(thing_id, six.string_types):\n thing_id = [thing_id]\n else:\n # Guarantee a subscriptable type.\n thing_id = list(thing_id)\n\n if len(thing_id) == 0:\n raise ValueError('No fullnames provided')\n\n # Will we return a list of server responses, or just one?\n # TODO: In future versions, change the threshold to 1 to get\n # list-in-list-out, single-in-single-out behavior. Threshold of 50\n # is to avoid a breaking change at this time.\n return_list = len(thing_id) > 50\n\n id_chunks = chunk_sequence(thing_id, 50)\n responses = []\n for id_chunk in id_chunks:\n id_chunk = ','.join(id_chunk)\n\n method = 'unhide' if _unhide else 'hide'\n data = {'id': id_chunk,\n 'executed': method}\n\n response = self.request_json(self.config[method], data=data)\n responses.append(response)\n\n if self.user is not None:\n self.evict(urljoin(self.user._url, # pylint: disable=W0212\n 'hidden'))\n if return_list:\n return responses\n else:\n return responses[0]"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nadding a comment on the given thing with the given text.", "response": "def _add_comment(self, thing_id, text):\n \"\"\"Comment on the given thing with the given text.\n\n :returns: A Comment object for the newly created comment.\n\n \"\"\"\n def add_comment_helper(self, thing_id, text):\n data = {'thing_id': thing_id,\n 'text': text}\n retval = self.request_json(self.config['comment'], data=data,\n retry_on_error=False)\n return retval\n\n if thing_id.startswith(self.config.by_object[objects.Message]):\n decorator = decorators.restrict_access(scope='privatemessages')\n else:\n decorator = decorators.restrict_access(scope='submit')\n retval = decorator(add_comment_helper)(self, thing_id, text)\n # REDDIT: reddit's end should only ever return a single comment\n return retval['data']['things'][0]"} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nsubmits a new link to the given subreddit.", "response": "def submit(self, subreddit, title, text=None, url=None, captcha=None,\n save=None, send_replies=None, resubmit=None, **kwargs):\n \"\"\"Submit a new link to the given subreddit.\n\n Accepts either a Subreddit object or a str containing the subreddit's\n display name.\n\n :param resubmit: If True, submit the link even if it has already been\n submitted.\n :param save: If True the new Submission will be saved after creation.\n :param send_replies: If True, inbox replies will be received when\n people comment on the submission. If set to None, the default of\n True for text posts and False for link posts will be used.\n\n :returns: The newly created Submission object if the reddit instance\n can access it. Otherwise, return the url to the submission.\n\n This function may result in a captcha challenge. PRAW will\n automatically prompt you for a response. See :ref:`handling-captchas`\n if you want to manually handle captchas.\n\n \"\"\"\n if isinstance(text, six.string_types) == bool(url):\n raise TypeError('One (and only one) of text or url is required!')\n data = {'sr': six.text_type(subreddit),\n 'title': title}\n if text or text == '':\n data['kind'] = 'self'\n data['text'] = text\n else:\n data['kind'] = 'link'\n data['url'] = url\n if captcha:\n data.update(captcha)\n if resubmit is not None:\n data['resubmit'] = resubmit\n if save is not None:\n data['save'] = save\n if send_replies is not None:\n data['sendreplies'] = send_replies\n result = self.request_json(self.config['submit'], data=data,\n retry_on_error=False)\n url = result['data']['url']\n # Clear the OAuth setting when attempting to fetch the submission\n if self._use_oauth:\n self._use_oauth = False\n if url.startswith(self.config.oauth_url):\n url = self.config.api_url + url[len(self.config.oauth_url):]\n try:\n return self.get_submission(url)\n except errors.Forbidden:\n # While the user may be able to submit to a subreddit,\n # that does not guarantee they have read access.\n return url"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef subscribe(self, subreddit, unsubscribe=False):\n data = {'action': 'unsub' if unsubscribe else 'sub',\n 'sr_name': six.text_type(subreddit)}\n response = self.request_json(self.config['subscribe'], data=data)\n self.evict(self.config['my_subreddits'])\n return response", "response": "Subscribe to the given subreddit."} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nmonkeying patches on top of the python webbrowser module to fix bugs and limitations of the module.", "response": "def patch_webbrowser():\n \"\"\"\n Some custom patches on top of the python webbrowser module to fix\n user reported bugs and limitations of the module.\n \"\"\"\n\n # https://bugs.python.org/issue31014\n # https://github.com/michael-lazar/rtv/issues/588\n def register_patch(name, klass, instance=None, update_tryorder=None, preferred=False):\n \"\"\"\n Wrapper around webbrowser.register() that detects if the function was\n invoked with the legacy function signature. If so, the signature is\n fixed before passing it along to the underlying function.\n\n Examples:\n register(name, klass, instance, -1)\n register(name, klass, instance, update_tryorder=-1)\n register(name, klass, instance, preferred=True)\n \"\"\"\n if update_tryorder is not None:\n preferred = (update_tryorder == -1)\n return webbrowser._register(name, klass, instance, preferred=preferred)\n\n if sys.version_info[:2] >= (3, 7):\n webbrowser._register = webbrowser.register\n webbrowser.register = register_patch\n\n # Add support for browsers that aren't defined in the python standard library\n webbrowser.register('surf', None, webbrowser.BackgroundBrowser('surf'))\n webbrowser.register('vimb', None, webbrowser.BackgroundBrowser('vimb'))\n webbrowser.register('qutebrowser', None, webbrowser.BackgroundBrowser('qutebrowser'))\n\n # Fix the opera browser, see https://github.com/michael-lazar/rtv/issues/476.\n # By default, opera will open a new tab in the current window, which is\n # what we want to do anyway.\n webbrowser.register('opera', None, webbrowser.BackgroundBrowser('opera'))\n\n # https://bugs.python.org/issue31348\n # Use MacOS actionscript when opening the program defined in by $BROWSER\n if sys.platform == 'darwin' and 'BROWSER' in os.environ:\n _userchoices = os.environ[\"BROWSER\"].split(os.pathsep)\n for cmdline in reversed(_userchoices):\n if cmdline in ('safari', 'firefox', 'chrome', 'default'):\n browser = webbrowser.MacOSXOSAScript(cmdline)\n webbrowser.register(cmdline, None, browser, update_tryorder=-1)"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef move(self, direction, n_windows):\n\n assert direction in (-1, 1)\n\n valid, redraw = True, False\n forward = ((direction * self.step) > 0)\n\n if forward:\n if self.page_index < 0:\n if self._is_valid(0):\n # Special case - advance the page index if less than zero\n self.page_index = 0\n self.cursor_index = 0\n redraw = True\n else:\n valid = False\n else:\n self.cursor_index += 1\n if not self._is_valid(self.absolute_index):\n # Move would take us out of bounds\n self.cursor_index -= 1\n valid = False\n elif self.cursor_index >= (n_windows - 1):\n # Flip the orientation and reset the cursor\n self.flip(self.cursor_index)\n self.cursor_index = 0\n self.top_item_height = None\n redraw = True\n else:\n if self.cursor_index > 0:\n self.cursor_index -= 1\n if self.top_item_height and self.cursor_index == 0:\n # Selecting the partially displayed item\n self.top_item_height = None\n redraw = True\n else:\n self.page_index -= self.step\n if self._is_valid(self.absolute_index):\n # We have reached the beginning of the page - move the\n # index\n self.top_item_height = None\n redraw = True\n else:\n self.page_index += self.step\n valid = False # Revert\n\n return valid, redraw", "response": "Moves the cursor up or down by the given increment."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef move_page(self, direction, n_windows):\n\n assert direction in (-1, 1)\n assert n_windows >= 0\n\n # top of subreddit/submission page or only one\n # submission/reply on the screen: act as normal move\n if (self.absolute_index < 0) | (n_windows == 0):\n valid, redraw = self.move(direction, n_windows)\n else:\n # first page\n if self.absolute_index < n_windows and direction < 0:\n self.page_index = -1\n self.cursor_index = 0\n self.inverted = False\n\n # not submission mode: starting index is 0\n if not self._is_valid(self.absolute_index):\n self.page_index = 0\n valid = True\n else:\n # flip to the direction of movement\n if ((direction > 0) & (self.inverted is True)) \\\n | ((direction < 0) & (self.inverted is False)):\n self.page_index += (self.step * (n_windows - 1))\n self.inverted = not self.inverted\n self.cursor_index \\\n = (n_windows - (direction < 0)) - self.cursor_index\n\n valid = False\n adj = 0\n # check if reached the bottom\n while not valid:\n n_move = n_windows - adj\n if n_move == 0:\n break\n\n self.page_index += n_move * direction\n valid = self._is_valid(self.absolute_index)\n if not valid:\n self.page_index -= n_move * direction\n adj += 1\n\n redraw = True\n\n return valid, redraw", "response": "Move the page down or up."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef flip(self, n_windows):\n\n assert n_windows >= 0\n self.page_index += (self.step * n_windows)\n self.cursor_index = n_windows\n self.inverted = not self.inverted\n self.top_item_height = None", "response": "Flip the orientation of the page."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nparsing a string and return its character code.", "response": "def parse(cls, key):\n \"\"\"\n Parse a key represented by a string and return its character code.\n \"\"\"\n\n try:\n if isinstance(key, int):\n return key\n elif re.match('[<]KEY_.*[>]', key):\n # Curses control character\n return getattr(curses, key[1:-1])\n elif re.match('[<].*[>]', key):\n # Ascii control character\n return getattr(curses.ascii, key[1:-1])\n elif key.startswith('0x'):\n # Ascii hex code\n return int(key, 16)\n elif len(key) == 2:\n # Double presses\n return tuple(cls.parse(k) for k in key)\n else:\n # Ascii character\n code = ord(key)\n if 0 <= code <= 255:\n return code\n # Python 3.3 has a curses.get_wch() function that we can use\n # for unicode keys, but Python 2.7 is limited to ascii.\n raise exceptions.ConfigError('Invalid configuration! `%s` is '\n 'not in the ascii range' % key)\n\n except (AttributeError, ValueError, TypeError):\n raise exceptions.ConfigError('Invalid configuration! \"%s\" is not a '\n 'valid key' % key)"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nguess based on the file extension.", "response": "def get_mimetype(url):\n \"\"\"\n Guess based on the file extension.\n\n Args:\n url (text): Web url that was linked to by a reddit submission.\n\n Returns:\n modified_url (text): The url (or filename) that will be used when\n constructing the command to run.\n content_type (text): The mime-type that will be used when\n constructing the command to run. If the mime-type is unknown,\n return None and the program will fallback to using the web\n browser.\n \"\"\"\n filename = url.split('?')[0]\n filename = filename.split('#')[0]\n content_type, _ = mimetypes.guess_type(filename)\n return url, content_type"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nrefreshing the content of the current page.", "response": "def refresh_content(self, order=None, name=None):\n \"\"\"\n Re-download all submissions and reset the page index\n \"\"\"\n order = order or self.content.order\n\n # Preserve the query if staying on the current page\n if name is None:\n query = self.content.query\n else:\n query = None\n\n name = name or self.content.name\n\n # Hack to allow an order specified in the name by prompt_subreddit() to\n # override the current default\n if order == 'ignore':\n order = None\n\n with self.term.loader('Refreshing page'):\n self.content = SubredditContent.from_name(\n self.reddit, name, self.term.loader, order=order, query=query)\n if not self.term.loader.exception:\n self.nav = Navigator(self.content.get)"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nopening a prompt to search the given subreddit.", "response": "def search_subreddit(self, name=None):\n \"\"\"\n Open a prompt to search the given subreddit\n \"\"\"\n name = name or self.content.name\n\n query = self.term.prompt_input('Search {0}: '.format(name))\n if not query:\n return\n\n with self.term.loader('Searching'):\n self.content = SubredditContent.from_name(\n self.reddit, name, self.term.loader, query=query)\n if not self.term.loader.exception:\n self.nav = Navigator(self.content.get)"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef show_frontpage(self):\n\n if self.content.name != '/r/front':\n target = '/r/front'\n self.toggled_subreddit = self.content.name\n else:\n target = self.toggled_subreddit\n\n # target still may be empty string if this command hasn't yet been used\n if target is not None:\n self.refresh_content(order='ignore', name=target)", "response": "If this command is pressed on a subreddit remember it and head back to the front page. If this command is not pressed on a subreddit go back to the last subreddit."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef open_submission(self, url=None):\n if url is None:\n data = self.get_selected_item()\n url = data['permalink']\n if data.get('url_type') == 'selfpost':\n self.config.history.add(data['url_full'])\n\n self.selected_page = self.open_submission_page(url)", "response": "Open the submission page."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef open_link(self):\n\n data = self.get_selected_item()\n if data['url_type'] == 'selfpost':\n self.open_submission()\n elif data['url_type'] == 'x-post subreddit':\n self.refresh_content(order='ignore', name=data['xpost_subreddit'])\n elif data['url_type'] == 'x-post submission':\n self.open_submission(url=data['url_full'])\n self.config.history.add(data['url_full'])\n else:\n self.term.open_link(data['url_full'])\n self.config.history.add(data['url_full'])", "response": "Open a link with the webbrowser."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef post_submission(self):\n # Check that the subreddit can be submitted to\n name = self.content.name\n if '+' in name or name in ('/r/all', '/r/front', '/r/me', '/u/saved'):\n self.term.show_notification(\"Can't post to {0}\".format(name))\n return\n\n submission_info = docs.SUBMISSION_FILE.format(name=name)\n with self.term.open_editor(submission_info) as text:\n if not text:\n self.term.show_notification('Canceled')\n return\n elif '\\n' not in text:\n self.term.show_notification('Missing body')\n return\n\n title, content = text.split('\\n', 1)\n with self.term.loader('Posting', delay=0):\n submission = self.reddit.submit(name, title, text=content,\n raise_captcha_exception=True)\n # Give reddit time to process the submission\n time.sleep(2.0)\n if self.term.loader.exception:\n raise TemporaryFileError()\n\n if not self.term.loader.exception:\n # Open the newly created submission\n self.selected_page = self.open_submission_page(submission=submission)", "response": "Post a new submission to the given subreddit."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef refresh_content(self, order=None, name=None):\n order = order or self.content.order\n url = name or self.content.name\n\n # Hack to allow an order specified in the name by prompt_subreddit() to\n # override the current default\n if order == 'ignore':\n order = None\n\n with self.term.loader('Refreshing page'):\n self.content = SubmissionContent.from_url(\n self.reddit, url, self.term.loader, order=order,\n max_comment_cols=self.config['max_comment_cols'])\n if not self.term.loader.exception:\n self.nav = Navigator(self.content.get, page_index=-1)", "response": "Re - download comments and reset the page index"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ntoggle the selected comment tree between visible and hidden ones.", "response": "def toggle_comment(self):\n \"\"\"\n Toggle the selected comment tree between visible and hidden\n \"\"\"\n current_index = self.nav.absolute_index\n self.content.toggle(current_index)\n\n # This logic handles a display edge case after a comment toggle. We\n # want to make sure that when we re-draw the page, the cursor stays at\n # its current absolute position on the screen. In order to do this,\n # apply a fixed offset if, while inverted, we either try to hide the\n # bottom comment or toggle any of the middle comments.\n if self.nav.inverted:\n data = self.content.get(current_index)\n if data['hidden'] or self.nav.cursor_index != 0:\n window = self._subwindows[-1][0]\n n_rows, _ = window.getmaxyx()\n self.nav.flip(len(self._subwindows) - 1)\n self.nav.top_item_height = n_rows"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef open_link(self):\n data = self.get_selected_item()\n if data['type'] == 'Submission':\n link = self.prompt_and_select_link()\n if link:\n self.config.history.add(link)\n self.term.open_link(link)\n elif data['type'] == 'Comment':\n link = self.prompt_and_select_link()\n if link:\n self.term.open_link(link)\n else:\n self.term.flash()", "response": "Open the link contained in the selected item."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef open_pager(self):\n n_rows, n_cols = self.term.stdscr.getmaxyx()\n\n if self.config['max_pager_cols'] is not None:\n n_cols = min(n_cols, self.config['max_pager_cols'])\n\n data = self.get_selected_item()\n if data['type'] == 'Submission':\n text = '\\n\\n'.join((data['permalink'], data['text']))\n self.term.open_pager(text, wrap=n_cols)\n elif data['type'] == 'Comment':\n text = '\\n\\n'.join((data['permalink'], data['body']))\n self.term.open_pager(text, wrap=n_cols)\n else:\n self.term.flash()", "response": "Open the selected item with the system s pager"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef comment_urlview(self):\n data = self.get_selected_item()\n comment = data.get('body') or data.get('text') or data.get('url_full')\n if comment:\n self.term.open_urlview(comment)\n else:\n self.term.flash()", "response": "Open the selected comment with the URL viewer"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nmove the cursor up to the comment s parent.", "response": "def move_parent_up(self):\n \"\"\"\n Move the cursor up to the comment's parent. If the comment is\n top-level, jump to the previous top-level comment.\n \"\"\"\n cursor = self.nav.absolute_index\n if cursor > 0:\n level = max(self.content.get(cursor)['level'], 1)\n while self.content.get(cursor - 1)['level'] >= level:\n self._move_cursor(-1)\n cursor -= 1\n self._move_cursor(-1)\n else:\n self.term.flash()\n\n self.clear_input_queue()"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nmove the next sibling comment to the next comment that s at the same level as the selected comment.", "response": "def move_sibling_next(self):\n \"\"\"\n Jump to the next comment that's at the same level as the selected\n comment and shares the same parent.\n \"\"\"\n cursor = self.nav.absolute_index\n if cursor >= 0:\n level = self.content.get(cursor)['level']\n try:\n move = 1\n while self.content.get(cursor + move)['level'] > level:\n move += 1\n except IndexError:\n self.term.flash()\n else:\n if self.content.get(cursor + move)['level'] == level:\n for _ in range(move):\n self._move_cursor(1)\n else:\n self.term.flash()\n else:\n self.term.flash()\n\n self.clear_input_queue()"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _get_redditor_listing(subpath=''):\n def _listing(self, sort='new', time='all', *args, **kwargs):\n \"\"\"Return a get_content generator for some RedditContentObject type.\n\n :param sort: Specify the sort order of the results if applicable\n (one of ``'hot'``, ``'new'``, ``'top'``, ``'controversial'``).\n :param time: Specify the time-period to return submissions if\n applicable (one of ``'hour'``, ``'day'``, ``'week'``,\n ``'month'``, ``'year'``, ``'all'``).\n\n The additional parameters are passed directly into\n :meth:`.get_content`. Note: the `url` parameter cannot be altered.\n\n \"\"\"\n kwargs.setdefault('params', {})\n kwargs['params'].setdefault('sort', sort)\n kwargs['params'].setdefault('t', time)\n url = urljoin(self._url, subpath) # pylint: disable=W0212\n return self.reddit_session.get_content(url, *args, **kwargs)\n return _listing", "response": "Return a function to generate Redditor listings."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _get_sorter(subpath='', **defaults):\n @restrict_access(scope='read')\n def _sorted(self, *args, **kwargs):\n \"\"\"Return a get_content generator for some RedditContentObject type.\n\n The additional parameters are passed directly into\n :meth:`.get_content`. Note: the `url` parameter cannot be altered.\n\n \"\"\"\n if not kwargs.get('params'):\n kwargs['params'] = {}\n for key, value in six.iteritems(defaults):\n kwargs['params'].setdefault(key, value)\n url = urljoin(self._url, subpath) # pylint: disable=W0212\n return self.reddit_session.get_content(url, *args, **kwargs)\n return _sorted", "response": "Return a function to generate specific subreddit Submission listings."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a function that can be used to modify a relationship.", "response": "def _modify_relationship(relationship, unlink=False, is_sub=False):\n \"\"\"Return a function for relationship modification.\n\n Used to support friending (user-to-user), as well as moderating,\n contributor creating, and banning (user-to-subreddit).\n\n \"\"\"\n # The API uses friend and unfriend to manage all of these relationships.\n url_key = 'unfriend' if unlink else 'friend'\n\n if relationship == 'friend':\n access = {'scope': None, 'login': True}\n elif relationship == 'moderator':\n access = {'scope': 'modothers'}\n elif relationship in ['banned', 'contributor', 'muted']:\n access = {'scope': 'modcontributors'}\n elif relationship in ['wikibanned', 'wikicontributor']:\n access = {'scope': ['modcontributors', 'modwiki']}\n else:\n access = {'scope': None, 'mod': True}\n\n @restrict_access(**access)\n def do_relationship(thing, user, **kwargs):\n data = {'name': six.text_type(user),\n 'type': relationship}\n data.update(kwargs)\n if is_sub:\n data['r'] = six.text_type(thing)\n else:\n data['container'] = thing.fullname\n\n session = thing.reddit_session\n if relationship == 'moderator':\n session.evict(session.config['moderators'].format(\n subreddit=six.text_type(thing)))\n url = session.config[url_key]\n return session.request_json(url, data=data)\n return do_relationship"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _prepare_request(reddit_session, url, params, data, auth, files,\n method=None):\n \"\"\"Return a requests Request object that can be \"prepared\".\"\"\"\n # Requests using OAuth for authorization must switch to using the oauth\n # domain.\n if getattr(reddit_session, '_use_oauth', False):\n bearer = 'bearer {0}'.format(reddit_session.access_token)\n headers = {'Authorization': bearer}\n config = reddit_session.config\n for prefix in (config.api_url, config.permalink_url):\n if url.startswith(prefix):\n if config.log_requests >= 1:\n msg = 'substituting {0} for {1} in url\\n'.format(\n config.oauth_url, prefix)\n sys.stderr.write(msg)\n url = config.oauth_url + url[len(prefix):]\n break\n else:\n headers = {}\n headers.update(reddit_session.http.headers)\n\n if method:\n pass\n elif data or files:\n method = 'POST'\n else:\n method = 'GET'\n\n # Log the request if logging is enabled\n if reddit_session.config.log_requests >= 1:\n sys.stderr.write('{0}: {1}\\n'.format(method, url))\n if reddit_session.config.log_requests >= 2:\n if params:\n sys.stderr.write('params: {0}\\n'.format(params))\n if data:\n sys.stderr.write('data: {0}\\n'.format(data))\n if auth:\n sys.stderr.write('auth: {0}\\n'.format(auth))\n # Prepare request\n request = Request(method=method, url=url, headers=headers, params=params,\n auth=auth, cookies=reddit_session.http.cookies)\n if method == 'GET':\n return request\n # Most POST requests require adding `api_type` and `uh` to the data.\n if data is True:\n data = {}\n\n if isinstance(data, dict):\n if not auth:\n data.setdefault('api_type', 'json')\n if reddit_session.modhash:\n data.setdefault('uh', reddit_session.modhash)\n else:\n request.headers.setdefault('Content-Type', 'application/json')\n\n request.data = data\n request.files = files\n return request", "response": "Prepare a request object that can be prepared."} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nraises exceptions if appropriate.", "response": "def _raise_redirect_exceptions(response):\n \"\"\"Return the new url or None if there are no redirects.\n\n Raise exceptions if appropriate.\n\n \"\"\"\n if response.status_code not in [301, 302, 307]:\n return None\n new_url = urljoin(response.url, response.headers['location'])\n if 'reddits/search' in new_url: # Handle non-existent subreddit\n subreddit = new_url.rsplit('=', 1)[1]\n raise InvalidSubreddit('`{0}` is not a valid subreddit'\n .format(subreddit))\n elif not RE_REDIRECT.search(response.url):\n raise RedirectException(response.url, new_url)\n return new_url"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nraising specific errors on some status codes.", "response": "def _raise_response_exceptions(response):\n \"\"\"Raise specific errors on some status codes.\"\"\"\n if not response.ok and 'www-authenticate' in response.headers:\n msg = response.headers['www-authenticate']\n if 'insufficient_scope' in msg:\n raise OAuthInsufficientScope('insufficient_scope', response.url)\n elif 'invalid_token' in msg:\n raise OAuthInvalidToken('invalid_token', response.url)\n else:\n raise OAuthException(msg, response.url)\n\n if response.status_code == codes.forbidden: # pylint: disable=E1101\n raise Forbidden(_raw=response)\n elif response.status_code == codes.not_found: # pylint: disable=E1101\n raise NotFound(_raw=response)\n else:\n try:\n response.raise_for_status() # These should all be directly mapped\n except exceptions.HTTPError as exc:\n raise HTTPException(_raw=exc.response)"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning an argument converted to a reddit - formatted list.", "response": "def _to_reddit_list(arg):\n \"\"\"Return an argument converted to a reddit-formatted list.\n\n The returned format is a comma deliminated list. Each element is a string\n representation of an object. Either given as a string or as an object that\n is then converted to its string representation.\n \"\"\"\n if (isinstance(arg, six.string_types) or not (\n hasattr(arg, \"__getitem__\") or hasattr(arg, \"__iter__\"))):\n return six.text_type(arg)\n else:\n return ','.join(six.text_type(a) for a in arg)"} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nprompts user for captcha solution and return a prepared result.", "response": "def _get_captcha(reddit_session, captcha_id):\n \"\"\"Prompt user for captcha solution and return a prepared result.\"\"\"\n url = urljoin(reddit_session.config['captcha'],\n captcha_id + '.png')\n sys.stdout.write('Captcha URL: {0}\\nCaptcha: '.format(url))\n sys.stdout.flush()\n raw = sys.stdin.readline()\n if not raw: # stdin has reached the end of file\n # Trigger exception raising next time through. The request is\n # cached so this will not require and extra request and delay.\n sys.stdin.close()\n return None\n return {'iden': captcha_id, 'captcha': raw.strip()}"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef rate_limit(function):\n @wraps(function)\n def wrapped(cls, _rate_domain, _rate_delay, **kwargs):\n cls.rl_lock.acquire()\n lock_last = cls.last_call.setdefault(_rate_domain, [Lock(), 0])\n with lock_last[0]: # Obtain the domain specific lock\n cls.rl_lock.release()\n # Sleep if necessary, then perform the request\n now = timer()\n delay = lock_last[1] + _rate_delay - now\n if delay > 0:\n now += delay\n time.sleep(delay)\n lock_last[1] = now\n return function(cls, **kwargs)\n return wrapped", "response": "Decorator that enforces API request limit guidelines."} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ndispatch the request and returns the response.", "response": "def request(self, request, proxies, timeout, verify, **_):\n \"\"\"Responsible for dispatching the request and returning the result.\n\n Network level exceptions should be raised and only\n ``requests.Response`` should be returned.\n\n :param request: A ``requests.PreparedRequest`` object containing all\n the data necessary to perform the request.\n :param proxies: A dictionary of proxy settings to be utilized for the\n request.\n :param timeout: Specifies the maximum time that the actual HTTP request\n can take.\n :param verify: Specifies if SSL certificates should be validated.\n\n ``**_`` should be added to the method call to ignore the extra\n arguments intended for the cache handler.\n\n \"\"\"\n settings = self.http.merge_environment_settings(\n request.url, proxies, False, verify, None\n )\n return self.http.send(request, timeout=timeout, allow_redirects=False,\n **settings)"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef with_cache(function):\n @wraps(function)\n def wrapped(cls, _cache_key, _cache_ignore, _cache_timeout, **kwargs):\n def clear_timeouts():\n \"\"\"Clear the cache of timed out results.\"\"\"\n for key in list(cls.timeouts):\n if timer() - cls.timeouts[key] > _cache_timeout:\n del cls.timeouts[key]\n del cls.cache[key]\n\n if _cache_ignore:\n return function(cls, **kwargs)\n with cls.ca_lock:\n clear_timeouts()\n if _cache_key in cls.cache:\n if cls.cache_hit_callback:\n cls.cache_hit_callback(_cache_key)\n return cls.cache[_cache_key]\n # Releasing the lock before actually making the request allows for\n # the possibility of more than one thread making the same request\n # to get through. Without having domain-specific caching (under the\n # assumption only one request to a domain can be made at a\n # time), there isn't a better way to handle this.\n result = function(cls, **kwargs)\n # The handlers don't call `raise_for_status` so we need to ignore\n # status codes that will result in an exception that should not be\n # cached.\n if result.status_code not in (200, 302):\n return result\n with cls.ca_lock:\n cls.timeouts[_cache_key] = timer()\n cls.cache[_cache_key] = result\n return result\n return wrapped", "response": "Returns a decorator that interacts with a handler s cache."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef evict(cls, urls):\n if isinstance(urls, text_type):\n urls = [urls]\n urls = set(normalize_url(url) for url in urls)\n retval = 0\n with cls.ca_lock:\n for key in list(cls.cache):\n if key[0] in urls:\n retval += 1\n del cls.cache[key]\n del cls.timeouts[key]\n return retval", "response": "Remove items from cache matching URLs. Returns the number of items removed."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _relay(self, **kwargs):\n retval = None\n delay_time = 2 # For connection retries\n read_attempts = 0 # For reading from socket\n while retval is None: # Evict can return False\n sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)\n sock_fp = sock.makefile('rwb') # Used for pickle\n try:\n sock.connect((self.host, self.port))\n cPickle.dump(kwargs, sock_fp, cPickle.HIGHEST_PROTOCOL)\n sock_fp.flush()\n retval = cPickle.load(sock_fp)\n except: # pylint: disable=W0702\n exc_type, exc, _ = sys.exc_info()\n socket_error = exc_type is socket.error\n if socket_error and exc.errno == 111: # Connection refused\n sys.stderr.write('Cannot connect to multiprocess server. I'\n 's it running? Retrying in {0} seconds.\\n'\n .format(delay_time))\n time.sleep(delay_time)\n delay_time = min(64, delay_time * 2)\n elif exc_type is EOFError or socket_error and exc.errno == 104:\n # Failure during socket READ\n if read_attempts >= 3:\n raise ClientException('Successive failures reading '\n 'from the multiprocess server.')\n sys.stderr.write('Lost connection with multiprocess server'\n ' during read. Trying again.\\n')\n read_attempts += 1\n else:\n raise\n finally:\n sock_fp.close()\n sock.close()\n if isinstance(retval, Exception):\n raise retval # pylint: disable=E0702\n return retval", "response": "Send the request through the server and return the HTTP response."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _load_configuration():\n config = configparser.RawConfigParser()\n module_dir = os.path.dirname(sys.modules[__name__].__file__)\n if 'APPDATA' in os.environ: # Windows\n os_config_path = os.environ['APPDATA']\n elif 'XDG_CONFIG_HOME' in os.environ: # Modern Linux\n os_config_path = os.environ['XDG_CONFIG_HOME']\n elif 'HOME' in os.environ: # Legacy Linux\n os_config_path = os.path.join(os.environ['HOME'], '.config')\n else:\n os_config_path = None\n locations = [os.path.join(module_dir, 'praw.ini'), 'praw.ini']\n if os_config_path is not None:\n locations.insert(1, os.path.join(os_config_path, 'praw.ini'))\n if not config.read(locations):\n raise Exception('Could not find config file in any of: {0}'\n .format(locations))\n return config", "response": "Attempt to load settings from various praw. ini files."} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nbinding the theme s colors to curses s internal color pair map.", "response": "def bind_curses(self):\n \"\"\"\n Bind the theme's colors to curses's internal color pair map.\n\n This method must be called once (after curses has been initialized)\n before any element attributes can be accessed. Color codes and other\n special attributes will be mixed bitwise into a single value that\n can be passed into curses draw functions.\n \"\"\"\n self._color_pair_map = {}\n self._attribute_map = {}\n\n for element, item in self.elements.items():\n fg, bg, attrs = item\n\n color_pair = (fg, bg)\n if self.use_color and color_pair != (-1, -1):\n # Curses limits the number of available color pairs, so we\n # need to reuse them if there are multiple elements with the\n # same foreground and background.\n if color_pair not in self._color_pair_map:\n # Index 0 is reserved by curses for the default color\n index = len(self._color_pair_map) + 1\n curses.init_pair(index, color_pair[0], color_pair[1])\n self._color_pair_map[color_pair] = curses.color_pair(index)\n attrs |= self._color_pair_map[color_pair]\n\n self._attribute_map[element] = attrs"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn the curses attribute code for the given element.", "response": "def get(self, element, selected=False):\n \"\"\"\n Returns the curses attribute code for the given element.\n \"\"\"\n if self._attribute_map is None:\n raise RuntimeError('Attempted to access theme attribute before '\n 'calling initialize_curses_theme()')\n\n if selected or self._selected:\n element = '@{0}'.format(element)\n\n return self._attribute_map[element]"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nlist all of the themes in the given search path.", "response": "def list_themes(cls, path=THEMES):\n \"\"\"\n Compile all of the themes configuration files in the search path.\n \"\"\"\n themes, errors = [], OrderedDict()\n\n def load_themes(path, source):\n \"\"\"\n Load all themes in the given path.\n \"\"\"\n if os.path.isdir(path):\n for filename in sorted(os.listdir(path)):\n if not filename.endswith('.cfg'):\n continue\n\n filepath = os.path.join(path, filename)\n name = filename[:-4]\n try:\n # Make sure the theme is valid\n theme = cls.from_file(filepath, source)\n except Exception as e:\n errors[(source, name)] = e\n else:\n themes.append(theme)\n\n themes.extend([Theme(use_color=True), Theme(use_color=False)])\n load_themes(DEFAULT_THEMES, 'preset')\n load_themes(path, 'installed')\n\n return themes, errors"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nprint a human - readable summary of the installed themes to stdout.", "response": "def print_themes(cls, path=THEMES):\n \"\"\"\n Prints a human-readable summary of the installed themes to stdout.\n\n This is intended to be used as a command-line utility, outside of the\n main curses display loop.\n \"\"\"\n themes, errors = cls.list_themes(path=path + '/')\n\n print('\\nInstalled ({0}):'.format(path))\n installed = [t for t in themes if t.source == 'installed']\n if installed:\n for theme in installed:\n line = ' {0:<20}[requires {1} colors]'\n print(line.format(theme.name, theme.required_colors))\n else:\n print(' (empty)')\n\n print('\\nPresets:')\n preset = [t for t in themes if t.source == 'preset']\n for theme in preset:\n line = ' {0:<20}[requires {1} colors]'\n print(line.format(theme.name, theme.required_colors))\n\n print('\\nBuilt-in:')\n built_in = [t for t in themes if t.source == 'built-in']\n for theme in built_in:\n line = ' {0:<20}[requires {1} colors]'\n print(line.format(theme.name, theme.required_colors))\n\n if errors:\n print('\\nWARNING: Some files encountered errors:')\n for (source, name), error in errors.items():\n theme_info = '({0}) {1}'.format(source, name)\n # Align multi-line error messages with the right column\n err_message = six.text_type(error).replace('\\n', '\\n' + ' ' * 20)\n print(' {0:<20}{1}'.format(theme_info, err_message))\n\n print('')"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef from_name(cls, name, path=THEMES):\n\n if os.path.isfile(name):\n return cls.from_file(name, 'custom')\n\n filename = os.path.join(path, '{0}.cfg'.format(name))\n if os.path.isfile(filename):\n return cls.from_file(filename, 'installed')\n\n filename = os.path.join(DEFAULT_THEMES, '{0}.cfg'.format(name))\n if os.path.isfile(filename):\n return cls.from_file(filename, 'preset')\n\n raise ConfigError('Could not find theme named \"{0}\"'.format(name))", "response": "Load a theme from a file path."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef from_file(cls, filename, source):\n _logger.info('Loading theme %s', filename)\n\n try:\n config = configparser.ConfigParser()\n config.optionxform = six.text_type # Preserve case\n with codecs.open(filename, encoding='utf-8') as fp:\n config.readfp(fp)\n except configparser.ParsingError as e:\n raise ConfigError(e.message)\n\n if not config.has_section('theme'):\n raise ConfigError(\n 'Error loading {0}:\\n'\n ' missing [theme] section'.format(filename))\n\n theme_name = os.path.basename(filename)\n theme_name, _ = os.path.splitext(theme_name)\n\n elements = {}\n for element, line in config.items('theme'):\n if element not in cls.DEFAULT_ELEMENTS:\n # Could happen if using a new config with an older version\n # of the software\n _logger.info('Skipping element %s', element)\n continue\n elements[element] = cls._parse_line(element, line, filename)\n\n return cls(name=theme_name, source=source, elements=elements)", "response": "Load a theme from the specified configuration file."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _parse_line(cls, element, line, filename=None):\n\n items = line.split()\n if len(items) == 2:\n fg, bg, attrs = items[0], items[1], ''\n elif len(items) == 3:\n fg, bg, attrs = items\n else:\n raise ConfigError(\n 'Error loading {0}, invalid line:\\n'\n ' {1} = {2}'.format(filename, element, line))\n\n if fg.startswith('#'):\n fg = cls.rgb_to_ansi(fg)\n if bg.startswith('#'):\n bg = cls.rgb_to_ansi(bg)\n\n if fg not in cls.COLOR_CODES:\n raise ConfigError(\n 'Error loading {0}, invalid :\\n'\n ' {1} = {2}'.format(filename, element, line))\n fg_code = cls.COLOR_CODES[fg]\n\n if bg not in cls.COLOR_CODES:\n raise ConfigError(\n 'Error loading {0}, invalid :\\n'\n ' {1} = {2}'.format(filename, element, line))\n bg_code = cls.COLOR_CODES[bg]\n\n attrs_code = curses.A_NORMAL\n for attr in attrs.split('+'):\n if attr not in cls.ATTRIBUTE_CODES:\n raise ConfigError(\n 'Error loading {0}, invalid :\\n'\n ' {1} = {2}'.format(filename, element, line))\n attr_code = cls.ATTRIBUTE_CODES[attr]\n if attr_code is None:\n attrs_code = None\n break\n else:\n attrs_code |= attr_code\n\n return fg_code, bg_code, attrs_code", "response": "Parse a single line from a theme file."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef rgb_to_ansi(color):\n\n if color[0] != '#' or len(color) != 7:\n return None\n\n try:\n r = round(int(color[1:3], 16) / 51.0) # Normalize between 0-5\n g = round(int(color[3:5], 16) / 51.0)\n b = round(int(color[5:7], 16) / 51.0)\n n = int(36 * r + 6 * g + b + 16)\n return 'ansi_{0:d}'.format(n)\n except ValueError:\n return None", "response": "Convert hex RGB to the 6x6x6 xterm color space"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ntraversing the list in the given direction and return the next theme", "response": "def _step(self, theme, direction):\n \"\"\"\n Traverse the list in the given direction and return the next theme\n \"\"\"\n if not self.themes:\n self.reload()\n\n # Try to find the starting index\n key = (theme.source, theme.name)\n for i, val in enumerate(self.themes):\n if (val.source, val.name) == key:\n index = i\n break\n else:\n # If the theme was set from a custom source it might\n # not be a part of the list returned by list_themes().\n self.themes.insert(0, theme)\n index = 0\n\n index = (index + direction) % len(self.themes)\n new_theme = self.themes[index]\n return new_theme"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef run():\n parser = OptionParser(version='%prog {0}'.format(__version__))\n parser.add_option('-a', '--addr', default='localhost',\n help=('The address or host to listen on. Specify -a '\n '0.0.0.0 to listen on all addresses. '\n 'Default: localhost'))\n parser.add_option('-p', '--port', type='int', default='10101',\n help=('The port to listen for requests on. '\n 'Default: 10101'))\n options, _ = parser.parse_args()\n try:\n server = ThreadingTCPServer((options.addr, options.port),\n RequestHandler)\n except (socket.error, socket.gaierror) as exc: # Handle bind errors\n print(exc)\n sys.exit(1)\n print('Listening on {0} port {1}'.format(options.addr, options.port))\n try:\n server.serve_forever() # pylint: disable=E1101\n except KeyboardInterrupt:\n server.socket.close() # pylint: disable=E1101\n RequestHandler.http.close()\n print('Goodbye!')", "response": "The entry point from the praw - multiprocess utility."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef handle_error(_, client_addr):\n exc_type, exc_value, _ = sys.exc_info()\n if exc_type is socket.error and exc_value[0] == 32:\n pass\n elif exc_type is cPickle.UnpicklingError:\n sys.stderr.write('Invalid connection from {0}\\n'\n .format(client_addr[0]))\n else:\n raise", "response": "Mute tracebacks of common errors."} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ndispatches the actual request and return the result.", "response": "def do_request(self, request, proxies, timeout, **_):\n \"\"\"Dispatch the actual request and return the result.\"\"\"\n print('{0} {1}'.format(request.method, request.url))\n response = self.http.send(request, proxies=proxies, timeout=timeout,\n allow_redirects=False)\n response.raw = None # Make pickleable\n return response"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef handle(self):\n data = cPickle.load(self.rfile) # pylint: disable=E1101\n method = data.pop('method')\n try:\n retval = getattr(self, 'do_{0}'.format(method))(**data)\n except Exception as e:\n # All exceptions should be passed to the client\n retval = e\n cPickle.dump(retval, self.wfile, # pylint: disable=E1101\n cPickle.HIGHEST_PROTOCOL)", "response": "Parse the RPC make the call and pickle up the return value."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncopy a file from the repo to the user s home directory.", "response": "def _copy_settings_file(source, destination, name):\n \"\"\"\n Copy a file from the repo to the user's home directory.\n \"\"\"\n\n if os.path.exists(destination):\n try:\n ch = six.moves.input(\n 'File %s already exists, overwrite? y/[n]):' % destination)\n if ch not in ('Y', 'y'):\n return\n except KeyboardInterrupt:\n return\n\n filepath = os.path.dirname(destination)\n if not os.path.exists(filepath):\n os.makedirs(filepath)\n\n print('Copying default %s to %s' % (name, destination))\n shutil.copy(source, destination)\n os.chmod(destination, 0o664)"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get_args():\n\n parser = build_parser()\n args = vars(parser.parse_args())\n\n # Overwrite the deprecated \"-l\" option into the link variable\n if args['link_deprecated'] and args['link'] is None:\n args['link'] = args['link_deprecated']\n args.pop('link_deprecated', None)\n\n # Filter out argument values that weren't supplied\n return {key: val for key, val in args.items() if val is not None}", "response": "Load settings from the command line."} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nload settings from an rtv configuration file.", "response": "def get_file(cls, filename=None):\n \"\"\"\n Load settings from an rtv configuration file.\n \"\"\"\n\n if filename is None:\n filename = CONFIG\n\n config = configparser.ConfigParser()\n if os.path.exists(filename):\n with codecs.open(filename, encoding='utf-8') as fp:\n config.readfp(fp)\n\n return cls._parse_rtv_file(config)"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nensuring that the directory exists before trying to write to the file.", "response": "def _ensure_filepath(filename):\n \"\"\"\n Ensure that the directory exists before trying to write to the file.\n \"\"\"\n\n filepath = os.path.dirname(filename)\n if not os.path.exists(filepath):\n os.makedirs(filepath)"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nhandles the selected page.", "response": "def handle_selected_page(self):\n \"\"\"\n Open the subscription and submission pages subwindows, but close the\n current page if any other type of page is selected.\n \"\"\"\n if not self.selected_page:\n pass\n if self.selected_page.name in ('subscription', 'submission'):\n # Launch page in a subwindow\n self.selected_page = self.selected_page.loop()\n elif self.selected_page.name in ('subreddit', 'inbox'):\n # Replace the current page\n self.active = False\n else:\n raise RuntimeError(self.selected_page.name)"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nmarking the selected message or comment as seen.", "response": "def mark_seen(self):\n \"\"\"\n Mark the selected message or comment as seen.\n \"\"\"\n data = self.get_selected_item()\n if data['is_new']:\n with self.term.loader('Marking as read'):\n data['object'].mark_as_read()\n if not self.term.loader.exception:\n data['is_new'] = False\n else:\n with self.term.loader('Marking as unread'):\n data['object'].mark_as_unread()\n if not self.term.loader.exception:\n data['is_new'] = True"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nviews the context surrounding the selected comment.", "response": "def view_context(self):\n \"\"\"\n View the context surrounding the selected comment.\n \"\"\"\n url = self.get_selected_item().get('context')\n if url:\n self.selected_page = self.open_submission_page(url)"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nopening the full submission and comment tree for the selected comment.", "response": "def open_submission(self):\n \"\"\"\n Open the full submission and comment tree for the selected comment.\n \"\"\"\n url = self.get_selected_item().get('submission_permalink')\n if url:\n self.selected_page = self.open_submission_page(url)"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef logged_in(f):\n\n @wraps(f)\n def wrapped_method(self, *args, **kwargs):\n if not self.reddit.is_oauth_session():\n self.term.show_notification('Not logged in')\n return None\n return f(self, *args, **kwargs)\n return wrapped_method", "response": "Decorator for Page methods that require the user to be authenticated."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef loop(self):\n self.active = True\n\n # This needs to be called once before the main loop, in case a subpage\n # was pre-selected before the loop started. This happens in __main__.py\n # with ``page.open_submission(url=url)``\n while self.selected_page and self.active:\n self.handle_selected_page()\n\n while self.active:\n self.draw()\n ch = self.term.stdscr.getch()\n self.controller.trigger(ch)\n\n while self.selected_page and self.active:\n self.handle_selected_page()\n\n return self.selected_page", "response": "Main control loop for the main control loop."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef previous_theme(self):\n theme = self.term.theme_list.previous(self.term.theme)\n while not self.term.check_theme(theme):\n theme = self.term.theme_list.previous(theme)\n\n self.term.set_theme(theme)\n self.draw()\n message = self.term.theme.display_string\n self.term.show_notification(message, timeout=1)", "response": "Cycle to preview the previous theme from the internal list of themes."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncycles to preview the next theme from the internal list of themes.", "response": "def next_theme(self):\n \"\"\"\n Cycle to preview the next theme from the internal list of themes.\n \"\"\"\n theme = self.term.theme_list.next(self.term.theme)\n while not self.term.check_theme(theme):\n theme = self.term.theme_list.next(theme)\n\n self.term.set_theme(theme)\n self.draw()\n message = self.term.theme.display_string\n self.term.show_notification(message, timeout=1)"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef move_page_top(self):\n self.nav.page_index = self.content.range[0]\n self.nav.cursor_index = 0\n self.nav.inverted = False", "response": "Move the cursor to the first item on the page."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nmoving the cursor to the last item on the page.", "response": "def move_page_bottom(self):\n \"\"\"\n Move the cursor to the last item on the page.\n \"\"\"\n self.nav.page_index = self.content.range[1]\n self.nav.cursor_index = 0\n self.nav.inverted = True"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef downvote(self):\n data = self.get_selected_item()\n if 'likes' not in data:\n self.term.flash()\n elif getattr(data['object'], 'archived'):\n self.term.show_notification(\"Voting disabled for archived post\", style='Error')\n elif data['likes'] or data['likes'] is None:\n with self.term.loader('Voting'):\n data['object'].downvote()\n if not self.term.loader.exception:\n data['likes'] = False\n else:\n with self.term.loader('Clearing vote'):\n data['object'].clear_vote()\n if not self.term.loader.exception:\n data['likes'] = None", "response": "Downvote the currently selected item."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef save(self):\n data = self.get_selected_item()\n if 'saved' not in data:\n self.term.flash()\n elif not data['saved']:\n with self.term.loader('Saving'):\n data['object'].save()\n if not self.term.loader.exception:\n data['saved'] = True\n else:\n with self.term.loader('Unsaving'):\n data['object'].unsave()\n if not self.term.loader.exception:\n data['saved'] = False", "response": "Save the currently selected item through the reddit API."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nprompting to log into the user s account or log out of the current user s account.", "response": "def login(self):\n \"\"\"\n Prompt to log into the user's account, or log out of the current\n account.\n \"\"\"\n if self.reddit.is_oauth_session():\n ch = self.term.show_notification('Log out? (y/n)')\n if ch in (ord('y'), ord('Y')):\n self.oauth.clear_oauth_data()\n self.term.show_notification('Logged out')\n else:\n self.oauth.authorize()"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef reply(self):\n data = self.get_selected_item()\n\n if data['type'] == 'Submission':\n body = data['text']\n description = 'submission'\n reply = data['object'].add_comment\n elif data['type'] in ('Comment', 'InboxComment'):\n body = data['body']\n description = 'comment'\n reply = data['object'].reply\n elif data['type'] == 'Message':\n body = data['body']\n description = 'private message'\n reply = data['object'].reply\n else:\n self.term.flash()\n return\n\n # Construct the text that will be displayed in the editor file.\n # The post body will be commented out and added for reference\n lines = [' |' + line for line in body.split('\\n')]\n content = '\\n'.join(lines)\n comment_info = docs.REPLY_FILE.format(\n author=data['author'],\n type=description,\n content=content)\n\n with self.term.open_editor(comment_info) as comment:\n if not comment:\n self.term.show_notification('Canceled')\n return\n\n with self.term.loader('Posting {}'.format(description), delay=0):\n reply(comment)\n # Give reddit time to process the submission\n time.sleep(2.0)\n\n if self.term.loader.exception is None:\n self.reload_page()\n else:\n raise TemporaryFileError()", "response": "Reply to the selected item."} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ndeletes a submission or comment.", "response": "def delete_item(self):\n \"\"\"\n Delete a submission or comment.\n \"\"\"\n data = self.get_selected_item()\n if data.get('author') != self.reddit.user.name:\n self.term.flash()\n return\n\n prompt = 'Are you sure you want to delete this? (y/n): '\n if not self.term.prompt_y_or_n(prompt):\n self.term.show_notification('Canceled')\n return\n\n with self.term.loader('Deleting', delay=0):\n data['object'].delete()\n # Give reddit time to process the request\n time.sleep(2.0)\n\n if self.term.loader.exception is None:\n self.reload_page()"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef edit(self):\n data = self.get_selected_item()\n if data.get('author') != self.reddit.user.name:\n self.term.flash()\n return\n\n if data['type'] == 'Submission':\n content = data['text']\n info = docs.SUBMISSION_EDIT_FILE.format(\n content=content, id=data['object'].id)\n elif data['type'] == 'Comment':\n content = data['body']\n info = docs.COMMENT_EDIT_FILE.format(\n content=content, id=data['object'].id)\n else:\n self.term.flash()\n return\n\n with self.term.open_editor(info) as text:\n if not text or text == content:\n self.term.show_notification('Canceled')\n return\n\n with self.term.loader('Editing', delay=0):\n data['object'].edit(text)\n time.sleep(2.0)\n\n if self.term.loader.exception is None:\n self.reload_page()\n else:\n raise TemporaryFileError()", "response": "Edit a submission or comment."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef send_private_message(self):\n message_info = docs.MESSAGE_FILE\n with self.term.open_editor(message_info) as text:\n if not text:\n self.term.show_notification('Canceled')\n return\n\n parts = text.split('\\n', 2)\n if len(parts) == 1:\n self.term.show_notification('Missing message subject')\n return\n elif len(parts) == 2:\n self.term.show_notification('Missing message body')\n return\n\n recipient, subject, message = parts\n recipient = recipient.strip()\n subject = subject.strip()\n message = message.rstrip()\n\n if not recipient:\n self.term.show_notification('Missing recipient')\n return\n elif not subject:\n self.term.show_notification('Missing message subject')\n return\n elif not message:\n self.term.show_notification('Missing message body')\n return\n\n with self.term.loader('Sending message', delay=0):\n self.reddit.send_message(\n recipient, subject, message, raise_captcha_exception=True)\n # Give reddit time to process the message\n time.sleep(2.0)\n\n if self.term.loader.exception:\n raise TemporaryFileError()\n else:\n self.term.show_notification('Message sent!')\n self.selected_page = self.open_inbox_page('sent')", "response": "Send a new private message to another user."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef prompt_and_select_link(self):\n data = self.get_selected_item()\n url_full = data.get('url_full')\n permalink = data.get('permalink')\n\n if url_full and url_full != permalink:\n # The item is a link-only submission that won't contain text\n link = url_full\n else:\n html = data.get('html')\n if html:\n extracted_links = self.content.extract_links(html)\n if not extracted_links:\n # Only one selection to choose from, so just pick it\n link = permalink\n else:\n # Let the user decide which link to open\n links = []\n if permalink:\n links += [{'text': 'Permalink', 'href': permalink}]\n links += extracted_links\n link = self.term.prompt_user_to_select_link(links)\n else:\n # Some items like hidden comments don't have any HTML to parse\n link = permalink\n\n return link", "response": "Prompt the user to select a link from a list to open."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef copy_to_clipboard(self, url):\n if url is None:\n self.term.flash()\n return\n\n try:\n clipboard_copy(url)\n except (ProgramError, OSError) as e:\n _logger.exception(e)\n self.term.show_notification(\n 'Failed to copy url: {0}'.format(e))\n else:\n self.term.show_notification(\n ['Copied to clipboard:', url], timeout=1)", "response": "Copy the selected URL to the user s clipboard."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nopening a prompt to navigate to a different subreddit or comment", "response": "def prompt(self):\n \"\"\"\n Open a prompt to navigate to a different subreddit or comment\"\n \"\"\"\n name = self.term.prompt_input('Enter page: /')\n if name:\n # Check if opening a submission url or a subreddit url\n # Example patterns for submissions:\n # comments/571dw3\n # /comments/571dw3\n # /r/pics/comments/571dw3/\n # https://www.reddit.com/r/pics/comments/571dw3/at_disneyland\n submission_pattern = re.compile(r'(^|/)comments/(?P.+?)($|/)')\n\n match = submission_pattern.search(name)\n if match:\n url = 'https://www.reddit.com/comments/{0}'.format(match.group('id'))\n self.selected_page = self.open_submission_page(url)\n else:\n self.selected_page = self.open_subreddit_page(name)"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef open_inbox_page(self, content_type):\n from .inbox_page import InboxPage\n\n with self.term.loader('Loading inbox'):\n page = InboxPage(self.reddit, self.term, self.config, self.oauth,\n content_type=content_type)\n if not self.term.loader.exception:\n return page", "response": "Open an instance of the InboxPage class for the logged in user."} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nopens an instance of the subscriptions page with the selected content.", "response": "def open_subscription_page(self, content_type):\n \"\"\"\n Open an instance of the subscriptions page with the selected content.\n \"\"\"\n from .subscription_page import SubscriptionPage\n\n with self.term.loader('Loading {0}s'.format(content_type)):\n page = SubscriptionPage(self.reddit, self.term, self.config,\n self.oauth, content_type=content_type)\n if not self.term.loader.exception:\n return page"} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nopens an instance of the submission page for the given submission URL.", "response": "def open_submission_page(self, url=None, submission=None):\n \"\"\"\n Open an instance of the submission page for the given submission URL.\n \"\"\"\n from .submission_page import SubmissionPage\n\n with self.term.loader('Loading submission'):\n page = SubmissionPage(self.reddit, self.term, self.config,\n self.oauth, url=url, submission=submission)\n if not self.term.loader.exception:\n return page"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef open_subreddit_page(self, name):\n from .subreddit_page import SubredditPage\n\n with self.term.loader('Loading subreddit'):\n page = SubredditPage(self.reddit, self.term, self.config,\n self.oauth, name)\n if not self.term.loader.exception:\n return page", "response": "Open an instance of the subreddit page for the given subreddit name."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nclear the terminal screen and redraw all of the sub - windows", "response": "def draw(self):\n \"\"\"\n Clear the terminal screen and redraw all of the sub-windows\n \"\"\"\n n_rows, n_cols = self.term.stdscr.getmaxyx()\n if n_rows < self.term.MIN_HEIGHT or n_cols < self.term.MIN_WIDTH:\n # TODO: Will crash when you try to navigate if the terminal is too\n # small at startup because self._subwindows will never be populated\n return\n\n self._row = 0\n self._draw_header()\n self._draw_banner()\n self._draw_content()\n self._draw_footer()\n self.term.clear_screen()\n self.term.stdscr.refresh()"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _draw_header(self):\n n_rows, n_cols = self.term.stdscr.getmaxyx()\n\n # Note: 2 argument form of derwin breaks PDcurses on Windows 7!\n window = self.term.stdscr.derwin(1, n_cols, self._row, 0)\n window.erase()\n # curses.bkgd expects bytes in py2 and unicode in py3\n window.bkgd(str(' '), self.term.attr('TitleBar'))\n\n sub_name = self.content.name\n sub_name = sub_name.replace('/r/front', 'Front Page')\n\n parts = sub_name.split('/')\n if len(parts) == 1:\n pass\n elif '/m/' in sub_name:\n _, _, user, _, multi = parts\n sub_name = '{} Curated by {}'.format(multi, user)\n elif parts[1] == 'u':\n noun = 'My' if parts[2] == 'me' else parts[2] + \"'s\"\n user_room = parts[3] if len(parts) == 4 else 'overview'\n title_lookup = {\n 'overview': 'Overview',\n 'submitted': 'Submissions',\n 'comments': 'Comments',\n 'saved': 'Saved Content',\n 'hidden': 'Hidden Content',\n 'upvoted': 'Upvoted Content',\n 'downvoted': 'Downvoted Content'\n }\n sub_name = \"{} {}\".format(noun, title_lookup[user_room])\n\n query = self.content.query\n if query:\n sub_name = 'Searching {0}: {1}'.format(sub_name, query)\n self.term.add_line(window, sub_name, 0, 0)\n\n # Set the terminal title\n if len(sub_name) > 50:\n title = sub_name.strip('/')\n title = title.replace('_', ' ')\n try:\n title = title.rsplit('/', 1)[1]\n except IndexError:\n pass\n else:\n title = sub_name\n\n # Setting the terminal title will break emacs or systems without\n # X window.\n if os.getenv('DISPLAY') and not os.getenv('INSIDE_EMACS'):\n title += ' - rtv {0}'.format(__version__)\n title = self.term.clean(title)\n if six.PY3:\n # In py3 you can't write bytes to stdout\n title = title.decode('utf-8')\n title = '\\x1b]2;{0}\\x07'.format(title)\n else:\n title = b'\\x1b]2;{0}\\x07'.format(title)\n sys.stdout.write(title)\n sys.stdout.flush()\n\n if self.reddit and self.reddit.user is not None:\n # The starting position of the name depends on if we're converting\n # to ascii or not\n width = len if self.config['ascii'] else textual_width\n\n if self.config['hide_username']:\n username = \"Logged in\"\n else:\n username = self.reddit.user.name\n s_col = (n_cols - width(username) - 1)\n # Only print username if it fits in the empty space on the right\n if (s_col - 1) >= width(sub_name):\n self.term.add_line(window, username, 0, s_col)\n\n self._row += 1", "response": "Draw the header at the top of the screen"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ndraw the banner at the top of the page", "response": "def _draw_banner(self):\n \"\"\"\n Draw the banner with sorting options at the top of the page\n \"\"\"\n n_rows, n_cols = self.term.stdscr.getmaxyx()\n window = self.term.stdscr.derwin(1, n_cols, self._row, 0)\n window.erase()\n window.bkgd(str(' '), self.term.attr('OrderBar'))\n\n banner = docs.BANNER_SEARCH if self.content.query else self.BANNER\n items = banner.strip().split(' ')\n\n distance = (n_cols - sum(len(t) for t in items) - 1) / (len(items) - 1)\n spacing = max(1, int(distance)) * ' '\n text = spacing.join(items)\n self.term.add_line(window, text, 0, 0)\n if self.content.order is not None:\n order = self.content.order.split('-')[0]\n col = text.find(order) - 3\n attr = self.term.attr('OrderBarHighlight')\n window.chgat(0, col, 3, attr)\n\n self._row += 1"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _draw_content(self):\n n_rows, n_cols = self.term.stdscr.getmaxyx()\n window = self.term.stdscr.derwin(n_rows - self._row - 1, n_cols, self._row, 0)\n window.erase()\n win_n_rows, win_n_cols = window.getmaxyx()\n\n self._subwindows = []\n page_index, cursor_index, inverted = self.nav.position\n step = self.nav.step\n\n # If not inverted, align the first submission with the top and draw\n # downwards. If inverted, align the first submission with the bottom\n # and draw upwards.\n cancel_inverted = True\n current_row = (win_n_rows - 1) if inverted else 0\n available_rows = win_n_rows\n top_item_height = None if inverted else self.nav.top_item_height\n for data in self.content.iterate(page_index, step, win_n_cols - 2):\n subwin_n_rows = min(available_rows, data['n_rows'])\n subwin_inverted = inverted\n if top_item_height is not None:\n # Special case: draw the page as non-inverted, except for the\n # top element. This element will be drawn as inverted with a\n # restricted height\n subwin_n_rows = min(subwin_n_rows, top_item_height)\n subwin_inverted = True\n top_item_height = None\n subwin_n_cols = win_n_cols - data['h_offset']\n start = current_row - subwin_n_rows + 1 if inverted else current_row\n subwindow = window.derwin(subwin_n_rows, subwin_n_cols, start, data['h_offset'])\n self._subwindows.append((subwindow, data, subwin_inverted))\n available_rows -= (subwin_n_rows + 1) # Add one for the blank line\n current_row += step * (subwin_n_rows + 1)\n if available_rows <= 0:\n # Indicate the page is full and we can keep the inverted screen.\n cancel_inverted = False\n break\n\n if len(self._subwindows) == 1:\n # Never draw inverted if only one subwindow. The top of the\n # subwindow should always be aligned with the top of the screen.\n cancel_inverted = True\n\n if cancel_inverted and self.nav.inverted:\n # In some cases we need to make sure that the screen is NOT\n # inverted. Unfortunately, this currently means drawing the whole\n # page over again. Could not think of a better way to pre-determine\n # if the content will fill up the page, given that it is dependent\n # on the size of the terminal.\n self.nav.flip((len(self._subwindows) - 1))\n self._draw_content()\n return\n\n if self.nav.cursor_index >= len(self._subwindows):\n # Don't allow the cursor to go over the number of subwindows\n # This could happen if the window is resized and the cursor index is\n # pushed out of bounds\n self.nav.cursor_index = len(self._subwindows) - 1\n\n # Now that the windows are setup, we can take a second pass through\n # to draw the text onto each subwindow\n for index, (win, data, inverted) in enumerate(self._subwindows):\n if self.nav.absolute_index >= 0 and index == self.nav.cursor_index:\n win.bkgd(str(' '), self.term.attr('Selected'))\n with self.term.theme.turn_on_selected():\n self._draw_item(win, data, inverted)\n else:\n win.bkgd(str(' '), self.term.attr('Normal'))\n self._draw_item(win, data, inverted)\n\n self._row += win_n_rows", "response": "Draw the content page."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _draw_footer(self):\n n_rows, n_cols = self.term.stdscr.getmaxyx()\n window = self.term.stdscr.derwin(1, n_cols, self._row, 0)\n window.erase()\n window.bkgd(str(' '), self.term.attr('HelpBar'))\n\n text = self.FOOTER.strip()\n self.term.add_line(window, text, 0, 0)\n self._row += 1", "response": "Draw the footer of the key binds help bar at the bottom of the screen"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nhandle GET requests to the server and stores the response body and headers.", "response": "def do_GET(self):\n \"\"\"\n Accepts GET requests to http://localhost:6500/, and stores the query\n params in the global dict. If shutdown_on_request is true, stop the\n server after the first successful request.\n\n The http request may contain the following query params:\n - state : unique identifier, should match what we passed to reddit\n - code : code that can be exchanged for a refresh token\n - error : if provided, the OAuth error that occurred\n \"\"\"\n\n parsed_path = urlparse(self.path)\n if parsed_path.path != '/':\n self.send_error(404)\n\n qs = parse_qs(parsed_path.query)\n self.params['state'] = qs['state'][0] if 'state' in qs else None\n self.params['code'] = qs['code'][0] if 'code' in qs else None\n self.params['error'] = qs['error'][0] if 'error' in qs else None\n\n body = self.build_body()\n\n # send_response also sets the Server and Date headers\n self.send_response(200)\n self.send_header('Content-Type', 'text/html; charset=UTF-8')\n self.send_header('Content-Length', len(body))\n self.end_headers()\n\n self.wfile.write(body)\n\n if self.shutdown_on_request:\n # Shutdown the server after serving the request\n # http://stackoverflow.com/a/22533929\n thread = threading.Thread(target=self.server.shutdown)\n thread.daemon = True\n thread.start()"} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nbuilds the body of the response from the template file.", "response": "def build_body(self, template_file=INDEX):\n \"\"\"\n Params:\n template_file (text): Path to an index.html template\n\n Returns:\n body (bytes): THe utf-8 encoded document body\n \"\"\"\n\n if self.params['error'] == 'access_denied':\n message = docs.OAUTH_ACCESS_DENIED\n elif self.params['error'] is not None:\n message = docs.OAUTH_ERROR.format(error=self.params['error'])\n elif self.params['state'] is None or self.params['code'] is None:\n message = docs.OAUTH_INVALID\n else:\n message = docs.OAUTH_SUCCESS\n\n with codecs.open(template_file, 'r', 'utf-8') as fp:\n index_text = fp.read()\n\n body = string.Template(index_text).substitute(message=message)\n body = codecs.encode(body, 'utf-8')\n return body"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ndetermines if the webbrowser will open in the terminal or not.", "response": "def display(self):\n \"\"\"\n Use a number of methods to guess if the default webbrowser will open in\n the background as opposed to opening directly in the terminal.\n \"\"\"\n\n if self._display is None:\n if sys.platform == 'darwin':\n # OS X won't set $DISPLAY unless xQuartz is installed.\n # If you're using OS X and you want to access a terminal\n # browser, you need to set it manually via $BROWSER.\n # See issue #166\n display = True\n else:\n display = bool(os.environ.get(\"DISPLAY\"))\n\n # Use the convention defined here to parse $BROWSER\n # https://docs.python.org/2/library/webbrowser.html\n console_browsers = ['www-browser', 'links', 'links2', 'elinks',\n 'lynx', 'w3m']\n if \"BROWSER\" in os.environ:\n user_browser = os.environ[\"BROWSER\"].split(os.pathsep)[0]\n if user_browser in console_browsers:\n display = False\n if webbrowser._tryorder:\n if webbrowser._tryorder[0] in console_browsers:\n display = False\n self._display = display\n return self._display"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef addch(window, y, x, ch, attr):\n\n if sys.version_info[:3] == (3, 4, 0):\n y, x = x, y\n\n window.addch(y, x, ch, attr)", "response": "addch - Add a character to the current color"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns the arrow and the attributes of the current object.", "response": "def get_arrow(self, likes):\n \"\"\"\n Curses does define constants for symbols (e.g. curses.ACS_BULLET).\n However, they rely on using the curses.addch() function, which has been\n found to be buggy and a general PITA to work with. By defining them as\n unicode points they can be added via the more reliable curses.addstr().\n http://bugs.python.org/issue21088\n \"\"\"\n\n if likes is None:\n return self.neutral_arrow, self.attr('NeutralVote')\n elif likes:\n return self.up_arrow, self.attr('Upvote')\n else:\n return self.down_arrow, self.attr('Downvote')"} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncleans a string of unicode characters into a set of n_cols columns.", "response": "def clean(self, string, n_cols=None):\n \"\"\"\n Required reading!\n http://nedbatchelder.com/text/unipain.html\n\n Python 2 input string will be a unicode type (unicode code points).\n Curses will accept unicode if all of the points are in the ascii range.\n However, if any of the code points are not valid ascii curses will\n throw a UnicodeEncodeError: 'ascii' codec can't encode character,\n ordinal not in range(128). If we encode the unicode to a utf-8 byte\n string and pass that to curses, it will render correctly.\n\n Python 3 input string will be a string type (unicode code points).\n Curses will accept that in all cases. However, the n character count in\n addnstr will not be correct. If code points are passed to addnstr,\n curses will treat each code point as one character and will not account\n for wide characters. If utf-8 is passed in, addnstr will treat each\n 'byte' as a single character.\n\n Reddit's api sometimes chokes and double-encodes some html characters\n Praw handles the initial decoding, but we need to do a second pass\n just to make sure. See https://github.com/michael-lazar/rtv/issues/96\n\n Example:\n &amp; -> returned directly from reddit's api\n & -> returned after PRAW decodes the html characters\n & -> returned after our second pass, this is the true value\n \"\"\"\n\n if n_cols is not None and n_cols <= 0:\n return ''\n\n if isinstance(string, six.text_type):\n string = unescape(string)\n\n if self.config['ascii']:\n if isinstance(string, six.binary_type):\n string = string.decode('utf-8')\n string = string.encode('ascii', 'replace')\n return string[:n_cols] if n_cols else string\n else:\n if n_cols:\n string = textual_width_chop(string, n_cols)\n if isinstance(string, six.text_type):\n string = string.encode('utf-8')\n return string"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef add_line(self, window, text, row=None, col=None, attr=None):\n\n # The following arg combos must be supported to conform with addnstr\n # (window, text)\n # (window, text, attr)\n # (window, text, row, col)\n # (window, text, row, col, attr)\n cursor_row, cursor_col = window.getyx()\n row = row if row is not None else cursor_row\n col = col if col is not None else cursor_col\n\n max_rows, max_cols = window.getmaxyx()\n n_cols = max_cols - col - 1\n if n_cols <= 0:\n # Trying to draw outside of the screen bounds\n return\n\n try:\n text = self.clean(text, n_cols)\n params = [] if attr is None else [attr]\n window.addstr(row, col, text, *params)\n except (curses.error, ValueError, TypeError) as e:\n # Curses handling of strings with invalid null bytes (b'\\00')\n # python 2: TypeError: \"int,int,str\"\n # python 3: ValueError: \"embedded null byte\"\n _logger.warning('add_line raised an exception')\n _logger.exception(str(e))", "response": "Unicode aware version of curses s built - in addnstr method."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef show_notification(self, message, timeout=None, style='Info'):\n\n assert style in ('Info', 'Warning', 'Error', 'Success')\n\n if isinstance(message, six.string_types):\n message = message.splitlines()\n\n n_rows, n_cols = self.stdscr.getmaxyx()\n v_offset, h_offset = self.stdscr.getbegyx()\n\n box_width = max(len(m) for m in message) + 2\n box_height = len(message) + 2\n\n # Cut off the lines of the message that don't fit on the screen\n box_width = min(box_width, n_cols)\n box_height = min(box_height, n_rows)\n message = message[:box_height - 2]\n\n s_row = (n_rows - box_height) // 2 + v_offset\n s_col = (n_cols - box_width) // 2 + h_offset\n\n window = curses.newwin(box_height, box_width, s_row, s_col)\n window.bkgd(str(' '), self.attr('Notice{0}'.format(style)))\n window.erase()\n window.border()\n\n for index, line in enumerate(message, start=1):\n self.add_line(window, line, index, 1)\n window.refresh()\n\n ch, start = -1, time.time()\n with self.no_delay():\n while timeout is None or time.time() - start < timeout:\n ch = self.getch()\n if ch != -1:\n break\n time.sleep(0.01)\n\n window.clear()\n del window\n self.stdscr.touchwin()\n self.stdscr.refresh()\n\n return ch", "response": "Overlays a message box on the center of the screen and waits for input."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef prompt_user_to_select_link(self, links):\n link_pages = self.get_link_pages(links)\n n = 0\n while n in range(len(link_pages)):\n link_page = link_pages[n]\n text = 'Select a link to open (page {} of {}):\\n\\n'\n text = text.format(n+1, len(link_pages))\n text += self.get_link_page_text(link_page)\n if link_page is not link_pages[-1]:\n text += '[j] next page...'\n if link_page is not link_pages[0]:\n if link_page is not link_pages[-1]:\n text += '\\n'\n text += '[k] ...previous page'\n\n try:\n choice = chr(self.show_notification(text))\n try:\n choice = int(choice)\n except ValueError:\n pass\n except ValueError:\n return None\n if choice == 'j':\n if link_page is not link_pages[-1]:\n n += 1\n continue\n elif choice == 'k':\n if link_page is not link_pages[0]:\n n -= 1\n continue\n elif choice not in range(len(link_page)):\n return None\n return link_page[choice]['href']", "response": "Prompt the user to select a link from a list to open."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ngive a list of links return a list of pages that can be displayed by the user and navigated using the 1 - 9 and 0 number keys.", "response": "def get_link_pages(links):\n \"\"\"\n Given a list of links, separate them into pages that can be displayed\n to the user and navigated using the 1-9 and 0 number keys.\n \"\"\"\n link_pages = []\n i = 0\n while i < len(links):\n link_page = []\n while i < len(links) and len(link_page) < 10:\n link_page.append(links[i])\n i += 1\n link_pages.append(link_page)\n return link_pages"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nconstruct the dialog box to display a list of links to the user.", "response": "def get_link_page_text(link_page):\n \"\"\"\n Construct the dialog box to display a list of links to the user.\n \"\"\"\n text = ''\n for i, link in enumerate(link_page):\n capped_link_text = (link['text'] if len(link['text']) <= 20\n else link['text'][:19] + '\u2026')\n text += '[{}] [{}]({})\\n'.format(i, capped_link_text, link['href'])\n return text"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef open_link(self, url):\n\n if not self.config['enable_media']:\n self.open_browser(url)\n return\n\n try:\n with self.loader('Checking link', catch_exception=False):\n command, entry = self.get_mailcap_entry(url)\n except exceptions.MailcapEntryNotFound:\n self.open_browser(url)\n return\n\n _logger.info('Executing command: %s', command)\n needs_terminal = 'needsterminal' in entry\n copious_output = 'copiousoutput' in entry\n\n if needs_terminal or copious_output:\n # Blocking, pause rtv until the process returns\n with self.suspend():\n os.system('clear')\n p = subprocess.Popen(\n [command], stderr=subprocess.PIPE,\n universal_newlines=True, shell=True)\n _, stderr = p.communicate()\n if copious_output:\n six.moves.input('Press any key to continue')\n code = p.poll()\n if code != 0:\n _logger.warning(stderr)\n self.show_notification(\n 'Program exited with status={0}\\n{1}'.format(\n code, stderr.strip()), style='Error')\n\n else:\n # Non-blocking, open a background process\n with self.loader('Opening page', delay=0):\n p = subprocess.Popen(\n [command], shell=True, universal_newlines=True,\n stdout=subprocess.PIPE, stderr=subprocess.PIPE)\n # Wait a little while to make sure that the command doesn't\n # exit with an error. This isn't perfect, but it should be good\n # enough to catch invalid commands.\n time.sleep(1.0)\n code = p.poll()\n if code is not None and code != 0:\n _, stderr = p.communicate()\n raise exceptions.BrowserError(\n 'Program exited with status={0}\\n{1}'.format(\n code, stderr.strip()))\n\n # Spin off a thread with p.communicate() to avoid subprocess\n # hang when the stodout/stderr PIPE gets filled up. This\n # behavior was discovered when opening long gifs with mpv\n # because mpv sends a progress bar to stderr.\n # https://thraxil.org/users/anders/posts/2008/03/13/\n threading.Thread(target=p.communicate).start()", "response": "Open a media link using the mailcap file."} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nsearches through the mime handlers list and attempts to find the appropriate command to open the provided url with.", "response": "def get_mailcap_entry(self, url):\n \"\"\"\n Search through the mime handlers list and attempt to find the\n appropriate command to open the provided url with.\n\n Will raise a MailcapEntryNotFound exception if no valid command exists.\n\n Params:\n url (text): URL that will be checked\n\n Returns:\n command (text): The string of the command that should be executed\n in a subprocess to open the resource.\n entry (dict): The full mailcap entry for the corresponding command\n \"\"\"\n\n for parser in mime_parsers.parsers:\n if parser.pattern.match(url):\n # modified_url may be the same as the original url, but it\n # could also be updated to point to a different page, or it\n # could refer to the location of a temporary file with the\n # page's downloaded content.\n try:\n modified_url, content_type = parser.get_mimetype(url)\n except Exception as e:\n # If Imgur decides to change its html layout, let it fail\n # silently in the background instead of crashing.\n _logger.warning('parser %s raised an exception', parser)\n _logger.exception(e)\n raise exceptions.MailcapEntryNotFound()\n if not content_type:\n _logger.info('Content type could not be determined')\n raise exceptions.MailcapEntryNotFound()\n elif content_type == 'text/html':\n _logger.info('Content type text/html, deferring to browser')\n raise exceptions.MailcapEntryNotFound()\n\n command, entry = mailcap.findmatch(\n self._mailcap_dict, content_type, filename=modified_url)\n if not entry:\n _logger.info('Could not find a valid mailcap entry')\n raise exceptions.MailcapEntryNotFound()\n\n return command, entry\n\n # No parsers matched the url\n raise exceptions.MailcapEntryNotFound()"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nopen a URL using the default webbrowser.", "response": "def open_browser(self, url):\n \"\"\"\n Open the given url using the default webbrowser. The preferred browser\n can specified with the $BROWSER environment variable. If not specified,\n python webbrowser will try to determine the default to use based on\n your system.\n\n For browsers requiring an X display, we open a new subprocess and\n redirect stdout/stderr to devnull. This is a workaround to stop\n BackgroundBrowsers (e.g. xdg-open, any BROWSER command ending in \"&\"),\n from spewing warning messages to the console. See\n http://bugs.python.org/issue22277 for a better description of the\n problem.\n\n For console browsers (e.g. w3m), RTV will suspend and display the\n browser window within the same terminal. This mode is triggered either\n when\n\n 1. $BROWSER is set to a known console browser, or\n 2. $DISPLAY is undefined, indicating that the terminal is running\n headless\n\n There may be other cases where console browsers are opened (xdg-open?)\n but are not detected here. These cases are still unhandled and will\n probably be broken if we incorrectly assume that self.display=True.\n \"\"\"\n\n if self.display:\n with self.loader('Opening page in a new window'):\n\n def open_url_silent(url):\n # This used to be done using subprocess.Popen().\n # It was switched to multiprocessing.Process so that we\n # can re-use the webbrowser instance that has been patched\n # by RTV. It's also safer because it doesn't inject\n # python code through the command line.\n\n # Suppress stdout/stderr from the browser, see\n # https://stackoverflow.com/questions/2323080. We can't\n # depend on replacing sys.stdout & sys.stderr because\n # webbrowser uses Popen().\n stdout, stderr = os.dup(1), os.dup(2)\n null = os.open(os.devnull, os.O_RDWR)\n try:\n os.dup2(null, 1)\n os.dup2(null, 2)\n if self.config['force_new_browser_window']:\n webbrowser.open_new(url)\n else:\n webbrowser.open_new_tab(url)\n finally:\n try:\n os.close(null)\n except OSError:\n pass\n os.dup2(stdout, 1)\n os.dup2(stderr, 2)\n\n p = Process(target=open_url_silent, args=(url,))\n p.start()\n # Give the browser 7 seconds to open a new tab. Because the\n # display is set, calling webbrowser should be non-blocking.\n # If it blocks or returns an error, something went wrong.\n try:\n p.join(7)\n if p.is_alive():\n raise exceptions.BrowserError(\n 'Timeout waiting for browser to open')\n finally:\n # This will be hit on the browser timeout, but also if the\n # user presses the ESC key. We always want to kill the\n # webbrowser process if it hasn't opened the tab and\n # terminated by now.\n try:\n p.terminate()\n except OSError:\n pass\n else:\n with self.suspend():\n if self.config['force_new_browser_window']:\n webbrowser.open_new(url)\n else:\n webbrowser.open_new_tab(url)"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef open_pager(self, data, wrap=None):\n\n pager = os.getenv('RTV_PAGER')\n if pager is None:\n pager = os.getenv('PAGER') or 'less'\n command = shlex.split(pager)\n\n if wrap:\n data_lines = content.Content.wrap_text(data, wrap)\n data = '\\n'.join(data_lines)\n\n try:\n with self.suspend():\n _logger.debug('Running command: %s', command)\n p = subprocess.Popen(command, stdin=subprocess.PIPE)\n try:\n p.communicate(data.encode('utf-8'))\n except KeyboardInterrupt:\n p.terminate()\n except OSError as e:\n _logger.exception(e)\n self.show_notification('Could not open pager %s' % pager)", "response": "Open a pager file with the specified data."} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nopens a file for editing using the system s default editor.", "response": "def open_editor(self, data=''):\n \"\"\"\n Open a file for editing using the system's default editor.\n\n After the file has been altered, the text will be read back and the\n HTML comment tag will be stripped. If an error\n occurs inside of the context manager, the file will be preserved so\n users can recover their data. Otherwise, the file will be deleted when\n the context manager closes.\n\n Params:\n data (str): If provided, text will be written to the file before\n opening it with the editor.\n\n Returns:\n text (str): The text that the user entered into the editor.\n \"\"\"\n\n with NamedTemporaryFile(prefix='rtv_', suffix='.txt', delete=False) as fp:\n # Create a tempory file and grab the name, but close immediately so\n # we can re-open using the right encoding\n filepath = fp.name\n\n with codecs.open(filepath, 'w', 'utf-8') as fp:\n fp.write(data)\n _logger.info('File created: %s', filepath)\n\n editor = (os.getenv('RTV_EDITOR') or\n os.getenv('VISUAL') or\n os.getenv('EDITOR') or\n 'nano')\n command = shlex.split(editor) + [filepath]\n try:\n with self.suspend():\n _logger.debug('Running command: %s', command)\n p = subprocess.Popen(command)\n try:\n p.communicate()\n except KeyboardInterrupt:\n p.terminate()\n except OSError as e:\n _logger.exception(e)\n self.show_notification('Could not open file with %s' % editor)\n\n with codecs.open(filepath, 'r', 'utf-8') as fp:\n text = fp.read()\n text = self.strip_instructions(text)\n\n try:\n yield text\n except exceptions.TemporaryFileError:\n # All exceptions will cause the file to *not* be removed, but these\n # ones should also be swallowed\n _logger.info('Caught TemporaryFileError')\n self.show_notification('Post saved as: %s' % filepath)\n else:\n # If no errors occurred, try to remove the file\n try:\n os.remove(filepath)\n except OSError:\n _logger.warning('Could not delete: %s', filepath)\n else:\n _logger.info('File deleted: %s', filepath)"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef open_urlview(self, data):\n\n urlview = os.getenv('RTV_URLVIEWER') or 'urlview'\n command = shlex.split(urlview)\n try:\n with self.suspend():\n _logger.debug('Running command: %s', command)\n p = subprocess.Popen(command, stdin=subprocess.PIPE)\n try:\n p.communicate(input=data.encode('utf-8'))\n except KeyboardInterrupt:\n p.terminate()\n\n code = p.poll()\n if code == 1:\n # Clear the \"No URLs found.\" message from stdout\n sys.stdout.write(\"\\033[F\")\n sys.stdout.flush()\n\n if code == 1:\n self.show_notification('No URLs found')\n\n except OSError as e:\n _logger.exception(e)\n self.show_notification(\n 'Failed to open {0}'.format(urlview))", "response": "Open a urlview with the given text."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef text_input(self, window, allow_resize=False):\n\n window.clear()\n\n # Set cursor mode to 1 because 2 doesn't display on some terminals\n self.curs_set(1)\n\n # Keep insert_mode off to avoid the recursion error described here\n # http://bugs.python.org/issue13051\n textbox = textpad.Textbox(window)\n textbox.stripspaces = 0\n\n def validate(ch):\n \"Filters characters for special key sequences\"\n if ch == self.ESCAPE:\n raise exceptions.EscapeInterrupt()\n if (not allow_resize) and (ch == curses.KEY_RESIZE):\n raise exceptions.EscapeInterrupt()\n # Fix backspace for iterm\n if ch == curses.ascii.DEL:\n ch = curses.KEY_BACKSPACE\n return ch\n\n # Wrapping in an exception block so that we can distinguish when the\n # user hits the return character from when the user tries to back out\n # of the input.\n try:\n out = textbox.edit(validate=validate)\n if isinstance(out, six.binary_type):\n out = out.decode('utf-8')\n except exceptions.EscapeInterrupt:\n out = None\n\n self.curs_set(0)\n return self.strip_textpad(out)", "response": "Transform a window into a text box that will accept user input and loop\n until a escape sequence is entered."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ndisplay a text prompt at the bottom of the screen.", "response": "def prompt_input(self, prompt, key=False):\n \"\"\"\n Display a text prompt at the bottom of the screen.\n\n Params:\n prompt (string): Text prompt that will be displayed\n key (bool): If true, grab a single keystroke instead of a full\n string. This can be faster than pressing enter for\n single key prompts (e.g. y/n?)\n \"\"\"\n\n n_rows, n_cols = self.stdscr.getmaxyx()\n v_offset, h_offset = self.stdscr.getbegyx()\n ch, attr = str(' '), self.attr('Prompt')\n prompt = self.clean(prompt, n_cols - 1)\n\n # Create a new window to draw the text at the bottom of the screen,\n # so we can erase it when we're done.\n s_row = v_offset + n_rows - 1\n s_col = h_offset\n prompt_win = curses.newwin(1, len(prompt) + 1, s_row, s_col)\n prompt_win.bkgd(ch, attr)\n self.add_line(prompt_win, prompt)\n prompt_win.refresh()\n\n # Create a separate window for text input\n s_col = h_offset + len(prompt)\n input_win = curses.newwin(1, n_cols - len(prompt), s_row, s_col)\n input_win.bkgd(ch, attr)\n input_win.refresh()\n\n if key:\n self.curs_set(1)\n ch = self.getch()\n # We can't convert the character to unicode, because it may return\n # Invalid values for keys that don't map to unicode characters,\n # e.g. F1\n text = ch if ch != self.ESCAPE else None\n self.curs_set(0)\n else:\n text = self.text_input(input_win)\n\n prompt_win.clear()\n input_win.clear()\n del prompt_win\n del input_win\n self.stdscr.touchwin()\n self.stdscr.refresh()\n\n return text"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nwrapping around prompt_input for simple yes or no queries.", "response": "def prompt_y_or_n(self, prompt):\n \"\"\"\n Wrapper around prompt_input for simple yes/no queries.\n \"\"\"\n\n ch = self.prompt_input(prompt, key=True)\n if ch in (ord('Y'), ord('y')):\n return True\n elif ch in (ord('N'), ord('n'), None):\n return False\n else:\n self.flash()\n return False"} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nstrips excess whitespace from the output of curses textpad.", "response": "def strip_textpad(text):\n \"\"\"\n Attempt to intelligently strip excess whitespace from the output of a\n curses textpad.\n \"\"\"\n\n if text is None:\n return text\n\n # Trivial case where the textbox is only one line long.\n if '\\n' not in text:\n return text.rstrip()\n\n # Allow one space at the end of the line. If there is more than one\n # space, assume that a newline operation was intended by the user\n stack, current_line = [], ''\n for line in text.split('\\n'):\n if line.endswith(' ') or not line:\n stack.append(current_line + line.rstrip())\n current_line = ''\n else:\n current_line += line\n stack.append(current_line)\n\n # Prune empty lines at the bottom of the textbox.\n for item in stack[::-1]:\n if not item:\n stack.pop()\n else:\n break\n\n out = '\\n'.join(stack)\n return out"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef strip_instructions(text):\n # Pattern can span multiple lines, allows dot to match newline chars\n flags = re.MULTILINE | re.DOTALL\n pattern = ''.format(token=TOKEN)\n text = re.sub(pattern, '', text, flags=flags)\n return re.sub(r'\\A[\\s\\n]*\\n', '', text, flags=flags).rstrip()", "response": "Remove instructional HTML comment tags inserted by RTV."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef clear_screen(self):\n\n if self._term != 'xterm-256color':\n self.stdscr.touchwin()\n else:\n self.stdscr.clearok(True)", "response": "Clear the screen for the current term."} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncheck if the given theme is compatible with the terminal color.", "response": "def check_theme(theme):\n \"\"\"\n Check if the given theme is compatible with the terminal\n \"\"\"\n terminal_colors = curses.COLORS if curses.has_colors() else 0\n\n if theme.required_colors > terminal_colors:\n return False\n elif theme.required_color_pairs > curses.COLOR_PAIRS:\n return False\n else:\n return True"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef set_theme(self, theme=None):\n\n terminal_colors = curses.COLORS if curses.has_colors() else 0\n default_theme = Theme(use_color=bool(terminal_colors))\n\n if theme is None:\n theme = default_theme\n\n elif theme.required_color_pairs > curses.COLOR_PAIRS:\n _logger.warning(\n 'Theme `%s` requires %s color pairs, but $TERM=%s only '\n 'supports %s color pairs, switching to default theme',\n theme.name, theme.required_color_pairs, self._term,\n curses.COLOR_PAIRS)\n theme = default_theme\n\n elif theme.required_colors > terminal_colors:\n _logger.warning(\n 'Theme `%s` requires %s colors, but $TERM=%s only '\n 'supports %s colors, switching to default theme',\n theme.name, theme.required_colors, self._term,\n curses.COLORS)\n theme = default_theme\n\n theme.bind_curses()\n self.theme = theme\n\n # Apply the default color to the whole screen\n self.stdscr.bkgd(str(' '), self.attr('Normal'))", "response": "Set the terminal theme."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns the object s fullname.", "response": "def fullname(self):\n \"\"\"Return the object's fullname.\n\n A fullname is an object's kind mapping like `t3` followed by an\n underscore and the object's base36 id, e.g., `t1_c5s96e0`.\n\n \"\"\"\n by_object = self.reddit_session.config.by_object\n return '{0}_{1}'.format(by_object[self.__class__], self.id)"} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\napproves object. This reverts a removal, resets the report counter, marks it with a green check mark (only visible to other moderators) on the website view and sets the approved_by attribute to the logged in user. :returns: The json response from the server.", "response": "def approve(self):\n \"\"\"Approve object.\n\n This reverts a removal, resets the report counter, marks it with a\n green check mark (only visible to other moderators) on the website view\n and sets the approved_by attribute to the logged in user.\n\n :returns: The json response from the server.\n\n \"\"\"\n url = self.reddit_session.config['approve']\n data = {'id': self.fullname}\n response = self.reddit_session.request_json(url, data=data)\n urls = [self.reddit_session.config[x] for x in ['modqueue', 'spam']]\n if isinstance(self, Submission):\n urls += self.subreddit._listing_urls # pylint: disable=W0212\n self.reddit_session.evict(urls)\n return response"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning a json response of the distinguished object.", "response": "def distinguish(self, as_made_by='mod', sticky=False):\n \"\"\"Distinguish object as made by mod, admin or special.\n\n Distinguished objects have a different author color. With Reddit\n Enhancement Suite it is the background color that changes.\n\n `sticky` argument only used for top-level Comments.\n\n :returns: The json response from the server.\n\n \"\"\"\n url = self.reddit_session.config['distinguish']\n data = {'id': self.fullname,\n 'how': 'yes' if as_made_by == 'mod' else as_made_by}\n if isinstance(self, Comment) and self.is_root:\n data['sticky'] = sticky\n return self.reddit_session.request_json(url, data=data)"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nignoring future reports on this object.", "response": "def ignore_reports(self):\n \"\"\"Ignore future reports on this object.\n\n This prevents future reports from causing notifications or appearing\n in the various moderation listing. The report count will still\n increment.\n\n \"\"\"\n url = self.reddit_session.config['ignore_reports']\n data = {'id': self.fullname}\n return self.reddit_session.request_json(url, data=data)"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef remove(self, spam=False):\n url = self.reddit_session.config['remove']\n data = {'id': self.fullname,\n 'spam': 'True' if spam else 'False'}\n response = self.reddit_session.request_json(url, data=data)\n urls = [self.reddit_session.config[x] for x in ['modqueue', 'spam']]\n if isinstance(self, Submission) and hasattr(self, 'subreddit'):\n urls += self.subreddit._listing_urls # pylint: disable=W0212\n self.reddit_session.evict(urls)\n return response", "response": "Remove the object from the moderator."} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nremoving ignoring of future reports on this object.", "response": "def unignore_reports(self):\n \"\"\"Remove ignoring of future reports on this object.\n\n Undoes 'ignore_reports'. Future reports will now cause notifications\n and appear in the various moderation listings.\n\n \"\"\"\n url = self.reddit_session.config['unignore_reports']\n data = {'id': self.fullname}\n return self.reddit_session.request_json(url, data=data)"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef delete(self):\n url = self.reddit_session.config['del']\n data = {'id': self.fullname}\n response = self.reddit_session.request_json(url, data=data)\n self.reddit_session.evict(self.reddit_session.config['user'])\n return response", "response": "Delete this object.\n\n :returns: The json response from the server."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef edit(self, text):\n url = self.reddit_session.config['edit']\n data = {'thing_id': self.fullname,\n 'text': text}\n response = self.reddit_session.request_json(url, data=data)\n self.reddit_session.evict(self.reddit_session.config['user'])\n return response['data']['things'][0]", "response": "Replace the body of the object with text."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ngild the Redditor or author of the content.", "response": "def gild(self, months=None):\n \"\"\"Gild the Redditor or author of the content.\n\n :param months: Specifies the number of months to gild. This parameter\n is Only valid when the instance called upon is of type\n Redditor. When not provided, the value defaults to 1.\n :returns: True on success, otherwise raises an exception.\n\n \"\"\"\n if isinstance(self, Redditor):\n months = int(months) if months is not None else 1\n if months < 1:\n raise TypeError('months must be at least 1')\n if months > 36:\n raise TypeError('months must be no more than 36')\n response = self.reddit_session.request(\n self.reddit_session.config['gild_user'].format(\n username=six.text_type(self)), data={'months': months})\n elif months is not None:\n raise TypeError('months is not a valid parameter for {0}'\n .format(type(self)))\n else:\n response = self.reddit_session.request(\n self.reddit_session.config['gild_thing']\n .format(fullname=self.fullname), data=True)\n return response.status_code == 200"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef hide(self, _unhide=False):\n return self.reddit_session.hide(self.fullname, _unhide=_unhide)", "response": "Hide the object in the context of the logged in user."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef reply(self, text):\n # pylint: disable=W0212\n response = self.reddit_session._add_comment(self.fullname, text)\n # pylint: enable=W0212\n urls = [self.reddit_session.config['inbox']]\n if isinstance(self, Comment):\n urls.append(self.submission._api_link) # pylint: disable=W0212\n elif isinstance(self, Message):\n urls.append(self.reddit_session.config['sent'])\n self.reddit_session.evict(urls)\n return response", "response": "Reply to object with the specified text."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef refresh(self):\n unique = self.reddit_session._unique_count # pylint: disable=W0212\n self.reddit_session._unique_count += 1 # pylint: disable=W0212\n\n if isinstance(self, Redditor):\n other = Redditor(self.reddit_session, self._case_name, fetch=True,\n uniq=unique)\n elif isinstance(self, Comment):\n sub = Submission.from_url(self.reddit_session, self.permalink,\n params={'uniq': unique})\n if sub.comments:\n other = sub.comments[0]\n else:\n # comment is \"specially deleted\", a reddit inconsistency;\n # see #519, #524, #535, #537, and #552 it needs to be\n # retreived via /api/info, but that's okay since these\n # specially deleted comments always have the same json\n # structure. The unique count needs to be updated\n # in case the comment originally came from /api/info\n msg = (\"Comment {0} was deleted or removed, and had \"\n \"no replies when such happened, so a second \"\n \"request was made to /api/info.\".format(self.name))\n unique = self.reddit_session._unique_count\n self.reddit_session._unique_count += 1\n other = self.reddit_session.get_info(thing_id=self.name,\n params={'uniq': unique})\n oldkeys = set(self.__dict__.keys())\n newkeys = set(other.__dict__.keys())\n keydiff = \", \".join(oldkeys - newkeys)\n if keydiff:\n msg += \"\\nCould not retrieve:\\n{0}\".format(keydiff)\n self.__dict__.update(other.__dict__) # pylint: disable=W0201\n warn(msg, RuntimeWarning)\n return self\n elif isinstance(self, Multireddit):\n other = Multireddit(self.reddit_session, author=self._author,\n name=self.name, uniq=unique, fetch=True)\n elif isinstance(self, Submission):\n params = self._params.copy()\n params['uniq'] = unique\n other = Submission.from_url(self.reddit_session, self.permalink,\n comment_sort=self._comment_sort,\n params=params)\n elif isinstance(self, Subreddit):\n other = Subreddit(self.reddit_session, self._case_name, fetch=True,\n uniq=unique)\n elif isinstance(self, WikiPage):\n other = WikiPage(self.reddit_session,\n six.text_type(self.subreddit), self.page,\n fetch=True, uniq=unique)\n\n self.__dict__ = other.__dict__ # pylint: disable=W0201\n return self", "response": "Re - query to update the object with latest values. Return the object."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef report(self, reason=None):\n url = self.reddit_session.config['report']\n data = {'id': self.fullname}\n if reason:\n data['reason'] = reason\n response = self.reddit_session.request_json(url, data=data)\n # Reported objects are automatically hidden as well\n # pylint: disable=W0212\n self.reddit_session.evict(\n [self.reddit_session.config['user'],\n urljoin(self.reddit_session.user._url, 'hidden')])\n # pylint: enable=W0212\n return response", "response": "Report this object to moderators."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef save(self, unsave=False):\n url = self.reddit_session.config['unsave' if unsave else 'save']\n data = {'id': self.fullname,\n 'executed': 'unsaved' if unsave else 'saved'}\n response = self.reddit_session.request_json(url, data=data)\n self.reddit_session.evict(self.reddit_session.config['saved'])\n return response", "response": "Save the object.\n\n :returns: The json response from the server."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef vote(self, direction=0):\n url = self.reddit_session.config['vote']\n data = {'id': self.fullname,\n 'dir': six.text_type(direction)}\n if self.reddit_session.user:\n # pylint: disable=W0212\n urls = [urljoin(self.reddit_session.user._url, 'disliked'),\n urljoin(self.reddit_session.user._url, 'liked')]\n # pylint: enable=W0212\n self.reddit_session.evict(urls)\n return self.reddit_session.request_json(url, data=data)", "response": "Vote for the given item in the given direction."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn the short permalink to the comment.", "response": "def _fast_permalink(self):\n \"\"\"Return the short permalink to the comment.\"\"\"\n if hasattr(self, 'link_id'): # from /r or /u comments page\n sid = self.link_id.split('_')[1]\n else: # from user's /message page\n sid = self.context.split('/')[4]\n return urljoin(self.reddit_session.config['comments'], '{0}/_/{1}'\n .format(sid, self.id))"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _update_submission(self, submission):\n submission._comments_by_id[self.name] = self # pylint: disable=W0212\n self._submission = submission\n if self._replies:\n for reply in self._replies:\n reply._update_submission(submission)", "response": "Update the submission object with the current submission."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning True when the comment is a top level comment.", "response": "def is_root(self):\n \"\"\"Return True when the comment is a top level comment.\"\"\"\n sub_prefix = self.reddit_session.config.by_object[Submission]\n return self.parent_id.startswith(sub_prefix)"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef replies(self):\n if self._replies is None or not self._has_fetched_replies:\n response = self.reddit_session.request_json(self._fast_permalink)\n if response[1]['data']['children']:\n # pylint: disable=W0212\n self._replies = response[1]['data']['children'][0]._replies\n else:\n # comment is \"specially deleted\", a reddit inconsistency;\n # see #519, #524, #535, #537, and #552 it needs to be\n # retreived via /api/info, but that's okay since these\n # specially deleted comments always have the same json\n # structure.\n msg = (\"Comment {0} was deleted or removed, and had \"\n \"no replies when such happened, so it still \"\n \"has no replies\".format(self.name))\n warn(msg, RuntimeWarning)\n self._replies = []\n # pylint: enable=W0212\n self._has_fetched_replies = True\n # Set the submission object if it is not set.\n if not self._submission:\n self._submission = response[0]['data']['children'][0]\n return self._replies", "response": "Return a list of the comment replies to this comment."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn the Submission object this comment belongs to.", "response": "def submission(self):\n \"\"\"Return the Submission object this comment belongs to.\"\"\"\n if not self._submission: # Comment not from submission\n self._submission = self.reddit_session.get_submission(\n url=self._fast_permalink)\n return self._submission"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nrequest the url for a Message and return a Message object.", "response": "def from_id(reddit_session, message_id, *args, **kwargs):\n \"\"\"Request the url for a Message and return a Message object.\n\n :param reddit_session: The session to make the request with.\n :param message_id: The ID of the message to request.\n\n The additional parameters are passed directly into\n :meth:`.request_json`.\n\n \"\"\"\n # Reduce fullname to ID if necessary\n message_id = message_id.split('_', 1)[-1]\n url = reddit_session.config['message'].format(messageid=message_id)\n message_info = reddit_session.request_json(url, *args, **kwargs)\n message = message_info['data']['children'][0]\n\n # Messages are received as a listing such that\n # the first item is always the thread's root.\n # The ID requested by the user may be a child.\n if message.id == message_id:\n return message\n for child in message.replies:\n if child.id == message_id:\n return child"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ncollapses a private message or modmail.", "response": "def collapse(self):\n \"\"\"Collapse a private message or modmail.\"\"\"\n url = self.reddit_session.config['collapse_message']\n self.reddit_session.request_json(url, data={'id': self.name})"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nmute the sender of this modmail message.", "response": "def mute_modmail_author(self, _unmute=False):\n \"\"\"Mute the sender of this modmail message.\n\n :param _unmute: Unmute the user instead. Please use\n :meth:`unmute_modmail_author` instead of setting this directly.\n\n \"\"\"\n path = 'unmute_sender' if _unmute else 'mute_sender'\n return self.reddit_session.request_json(\n self.reddit_session.config[path], data={'id': self.fullname})"} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nfetch and return the comments for a single MoreComments object.", "response": "def comments(self, update=True):\n \"\"\"Fetch and return the comments for a single MoreComments object.\"\"\"\n if not self._comments:\n if self.count == 0: # Handle 'continue this thread' type\n return self._continue_comments(update)\n # pylint: disable=W0212\n children = [x for x in self.children if 't1_{0}'.format(x)\n not in self.submission._comments_by_id]\n # pylint: enable=W0212\n if not children:\n return None\n data = {'children': ','.join(children),\n 'link_id': self.submission.fullname,\n 'r': str(self.submission.subreddit)}\n\n # pylint: disable=W0212\n if self.submission._comment_sort:\n data['where'] = self.submission._comment_sort\n # pylint: enable=W0212\n url = self.reddit_session.config['morechildren']\n response = self.reddit_session.request_json(url, data=data)\n self._comments = response['data']['things']\n if update:\n for comment in self._comments:\n # pylint: disable=W0212\n comment._update_submission(self.submission)\n # pylint: enable=W0212\n return self._comments"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_downvoted(self, *args, **kwargs):\n # Sending an OAuth authenticated request for a redditor, who isn't the\n # authenticated user. But who has a public voting record will be\n # successful.\n kwargs['_use_oauth'] = self.reddit_session.is_oauth_session()\n return _get_redditor_listing('downvoted')(self, *args, **kwargs)", "response": "Return a generator of the items that have been downvoted by the user."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_friend_info(self):\n url = self.reddit_session.config['friend_v1'].format(user=self.name)\n data = {'id': self.name}\n return self.reddit_session.request_json(url, data=data, method='GET')", "response": "Return information about this user including personal notes."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a generator of the items that have been upvoted by the user.", "response": "def get_upvoted(self, *args, **kwargs):\n \"\"\"Return a listing of the Submissions the user has upvoted.\n\n :returns: get_content generator of Submission items.\n\n The additional parameters are passed directly into\n :meth:`.get_content`. Note: the `url` parameter cannot be altered.\n\n As a default, this listing is only accessible by the user. Thereby\n requirering either user/pswd authentication or OAuth authentication\n with the 'history' scope. Users may choose to make their voting record\n public by changing a user preference. In this case, no authentication\n will be needed to access this listing.\n\n \"\"\"\n kwargs['_use_oauth'] = self.reddit_session.is_oauth_session()\n return _get_redditor_listing('upvoted')(self, *args, **kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nmarks the given messages as read or unread.", "response": "def mark_as_read(self, messages, unread=False):\n \"\"\"Mark message(s) as read or unread.\n\n :returns: The json response from the server.\n\n \"\"\"\n ids = []\n if isinstance(messages, Inboxable):\n ids.append(messages.fullname)\n elif hasattr(messages, '__iter__'):\n for msg in messages:\n if not isinstance(msg, Inboxable):\n msg = 'Invalid message type: {0}'.format(type(msg))\n raise ClientException(msg)\n ids.append(msg.fullname)\n else:\n msg = 'Invalid message type: {0}'.format(type(messages))\n raise ClientException(msg)\n # pylint: disable=W0212\n retval = self.reddit_session._mark_as_read(ids, unread=unread)\n # pylint: enable=W0212\n return retval"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_blocked(self):\n url = self.reddit_session.config['blocked']\n return self.reddit_session.request_json(url)", "response": "Return a UserList of Redditors with whom the user has blocked."} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns a dictionary of the user s moderated reddits. This list is used internally. Consider using the get_my_moderation function instead.", "response": "def get_cached_moderated_reddits(self):\n \"\"\"Return a cached dictionary of the user's moderated reddits.\n\n This list is used internally. Consider using the `get_my_moderation`\n function instead.\n\n \"\"\"\n if self._mod_subs is None:\n self._mod_subs = {'mod': self.reddit_session.get_subreddit('mod')}\n for sub in self.reddit_session.get_my_moderation(limit=None):\n self._mod_subs[six.text_type(sub).lower()] = sub\n return self._mod_subs"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _extract_more_comments(tree):\n more_comments = []\n queue = [(None, x) for x in tree]\n while len(queue) > 0:\n parent, comm = queue.pop(0)\n if isinstance(comm, MoreComments):\n heappush(more_comments, comm)\n if parent:\n parent.replies.remove(comm)\n else:\n tree.remove(comm)\n else:\n for item in comm.replies:\n queue.append((comm, item))\n return more_comments", "response": "Return a list of MoreComments objects removed from tree."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef from_id(reddit_session, subreddit_id):\n pseudo_data = {'id': subreddit_id,\n 'permalink': '/comments/{0}'.format(subreddit_id)}\n return Submission(reddit_session, pseudo_data)", "response": "Return an edit - only submission object based on the id."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef from_url(reddit_session, url, comment_limit=0, comment_sort=None,\n comments_only=False, params=None):\n \"\"\"Request the url and return a Submission object.\n\n :param reddit_session: The session to make the request with.\n :param url: The url to build the Submission object from.\n :param comment_limit: The desired number of comments to fetch. If <= 0\n fetch the default number for the session's user. If None, fetch the\n maximum possible.\n :param comment_sort: The sort order for retrieved comments. When None\n use the default for the session's user.\n :param comments_only: Return only the list of comments.\n :param params: dictionary containing extra GET data to put in the url.\n\n \"\"\"\n if params is None:\n params = {}\n\n parsed = urlparse(url)\n query_pairs = parse_qs(parsed.query)\n get_params = dict((k, \",\".join(v)) for k, v in query_pairs.items())\n params.update(get_params)\n url = urlunparse(parsed[:3] + (\"\", \"\", \"\"))\n if comment_limit is None: # Fetch MAX\n params['limit'] = 2048 # Just use a big number\n elif comment_limit > 0: # Use value\n params['limit'] = comment_limit\n if comment_sort:\n params['sort'] = comment_sort\n\n response = reddit_session.request_json(url, params=params)\n if comments_only:\n return response[1]['data']['children']\n submission = Submission.from_json(response)\n submission._comment_sort = comment_sort # pylint: disable=W0212\n submission._params = params # pylint: disable=W0212\n return submission", "response": "Request the url and return a Submission object."} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nadds a comment on the submission using the specified text.", "response": "def add_comment(self, text):\n \"\"\"Comment on the submission using the specified text.\n\n :returns: A Comment object for the newly created comment.\n\n \"\"\"\n # pylint: disable=W0212\n response = self.reddit_session._add_comment(self.fullname, text)\n # pylint: enable=W0212\n self.reddit_session.evict(self._api_link) # pylint: disable=W0212\n return response"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef comments(self): # pylint: disable=E0202\n if self._comments is None:\n self.comments = Submission.from_url( # pylint: disable=W0212\n self.reddit_session, self._api_link, comments_only=True)\n return self._comments", "response": "Return forest of comments with top - level comments as tree roots."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a get_content generator for the submission s duplicates.", "response": "def get_duplicates(self, *args, **kwargs):\n \"\"\"Return a get_content generator for the submission's duplicates.\n\n :returns: get_content generator iterating over Submission objects.\n\n The additional parameters are passed directly into\n :meth:`.get_content`. Note: the `url` and `object_filter` parameters\n cannot be altered.\n\n \"\"\"\n url = self.reddit_session.config['duplicates'].format(\n submissionid=self.id)\n return self.reddit_session.get_content(url, *args, object_filter=1,\n **kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_flair_choices(self, *args, **kwargs):\n return self.subreddit.get_flair_choices(self.fullname, *args, **kwargs)", "response": "Return available link flair choices and current flair."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nlock thread. Requires that the currently authenticated user has the modposts oauth scope or has user/password authentication as a mod of the subreddit. :returns: The json response from the server.", "response": "def lock(self):\n \"\"\"Lock thread.\n\n Requires that the currently authenticated user has the modposts oauth\n scope or has user/password authentication as a mod of the subreddit.\n\n :returns: The json response from the server.\n\n \"\"\"\n url = self.reddit_session.config['lock']\n data = {'id': self.fullname}\n return self.reddit_session.request_json(url, data=data)"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nmark as Not Safe For Work.", "response": "def mark_as_nsfw(self, unmark_nsfw=False):\n \"\"\"Mark as Not Safe For Work.\n\n Requires that the currently authenticated user is the author of the\n submission, has the modposts oauth scope or has user/password\n authentication as a mod of the subreddit.\n\n :returns: The json response from the server.\n\n \"\"\"\n def mark_as_nsfw_helper(self): # pylint: disable=W0613\n # It is necessary to have the 'self' argument as it's needed in\n # restrict_access to determine what class the decorator is\n # operating on.\n url = self.reddit_session.config['unmarknsfw' if unmark_nsfw else\n 'marknsfw']\n data = {'id': self.fullname}\n return self.reddit_session.request_json(url, data=data)\n\n is_author = (self.reddit_session.is_logged_in() and self.author ==\n self.reddit_session.user)\n if is_author:\n return mark_as_nsfw_helper(self)\n else:\n return restrict_access('modposts')(mark_as_nsfw_helper)(self)"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef replace_more_comments(self, limit=32, threshold=1):\n if self._replaced_more:\n return []\n\n remaining = limit\n more_comments = self._extract_more_comments(self.comments)\n skipped = []\n\n # Fetch largest more_comments until reaching the limit or the threshold\n while more_comments:\n item = heappop(more_comments)\n if remaining == 0: # We're not going to replace any more\n heappush(more_comments, item) # It wasn't replaced\n break\n elif len(item.children) == 0 or 0 < item.count < threshold:\n heappush(skipped, item) # It wasn't replaced\n continue\n\n # Fetch new comments and decrease remaining if a request was made\n new_comments = item.comments(update=False)\n if new_comments is not None and remaining is not None:\n remaining -= 1\n elif new_comments is None:\n continue\n\n # Re-add new MoreComment objects to the heap of more_comments\n for more in self._extract_more_comments(new_comments):\n more._update_submission(self) # pylint: disable=W0212\n heappush(more_comments, more)\n # Insert the new comments into the tree\n for comment in new_comments:\n self._insert_comment(comment)\n\n self._replaced_more = True\n return more_comments + skipped", "response": "Updates the comment tree by replacing instances of MoreComments."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef set_flair(self, *args, **kwargs):\n return self.subreddit.set_flair(self, *args, **kwargs)", "response": "Convenience method that utilizes ModFlairMixin. set_flair to populate both the subreddit and item parameters."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef set_suggested_sort(self, sort='blank'):\n url = self.reddit_session.config['suggested_sort']\n data = {'id': self.fullname, 'sort': sort}\n return self.reddit_session.request_json(url, data=data)", "response": "Set Suggested Sort for the comments of the submission."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef sticky(self, bottom=True):\n url = self.reddit_session.config['sticky_submission']\n data = {'id': self.fullname, 'state': True}\n if not bottom:\n data['num'] = 1\n return self.reddit_session.request_json(url, data=data)", "response": "Sticky a post in its subreddit."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nlocking thread. Requires that the currently authenticated user has the modposts oauth scope or has user/password authentication as a mod of the subreddit. :returns: The json response from the server.", "response": "def unlock(self):\n \"\"\"Lock thread.\n\n Requires that the currently authenticated user has the modposts oauth\n scope or has user/password authentication as a mod of the subreddit.\n\n :returns: The json response from the server.\n\n \"\"\"\n url = self.reddit_session.config['unlock']\n data = {'id': self.fullname}\n return self.reddit_session.request_json(url, data=data)"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nremove all user flair on this subreddit.", "response": "def clear_all_flair(self):\n \"\"\"Remove all user flair on this subreddit.\n\n :returns: The json response from the server when there is flair to\n clear, otherwise returns None.\n\n \"\"\"\n csv = [{'user': x['user']} for x in self.get_flair_list(limit=None)]\n if csv:\n return self.set_flair_csv(csv)\n else:\n return"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef from_api_response(cls, reddit_session, json_dict):\n # The Multireddit response contains the Subreddits attribute as a list\n # of dicts of the form {'name': 'subredditname'}.\n # We must convert each of these into a Subreddit object.\n json_dict['subreddits'] = [Subreddit(reddit_session, item['name'])\n for item in json_dict['subreddits']]\n return cls(reddit_session, None, None, json_dict)", "response": "Return an instance of the appropriate class from the json dict."} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nadd a subreddit to the multireddit.", "response": "def add_subreddit(self, subreddit, _delete=False, *args, **kwargs):\n \"\"\"Add a subreddit to the multireddit.\n\n :param subreddit: The subreddit name or Subreddit object to add\n\n The additional parameters are passed directly into\n :meth:`~praw.__init__.BaseReddit.request_json`.\n\n \"\"\"\n subreddit = six.text_type(subreddit)\n url = self.reddit_session.config['multireddit_add'].format(\n user=self._author, multi=self.name, subreddit=subreddit)\n method = 'DELETE' if _delete else 'PUT'\n # The modhash isn't necessary for OAuth requests\n if not self.reddit_session._use_oauth:\n self.reddit_session.http.headers['x-modhash'] = \\\n self.reddit_session.modhash\n data = {'model': dumps({'name': subreddit})}\n try:\n self.reddit_session.request(url, data=data, method=method,\n *args, **kwargs)\n finally:\n # The modhash isn't necessary for OAuth requests\n if not self.reddit_session._use_oauth:\n del self.reddit_session.http.headers['x-modhash']"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncopy this multireddit to another name.", "response": "def copy(self, to_name):\n \"\"\"Copy this multireddit.\n\n Convenience function that utilizes\n :meth:`.MultiredditMixin.copy_multireddit` populating both\n the `from_redditor` and `from_name` parameters.\n\n \"\"\"\n return self.reddit_session.copy_multireddit(self._author, self.name,\n to_name)"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef edit(self, *args, **kwargs):\n return self.reddit_session.edit_multireddit(name=self.name, *args,\n **kwargs)", "response": "Edit this multireddit.\n\n Convenience function that utilizes\n :meth:`.MultiredditMixin.edit_multireddit` populating the `name`\n parameter."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef remove_subreddit(self, subreddit, *args, **kwargs):\n return self.add_subreddit(subreddit, True, *args, **kwargs)", "response": "Remove a subreddit from the user s multireddit."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef rename(self, new_name, *args, **kwargs):\n new = self.reddit_session.rename_multireddit(self.name, new_name,\n *args, **kwargs)\n self.__dict__ = new.__dict__ # pylint: disable=W0201\n return self", "response": "Rename this multireddit.\n\n This function is a handy shortcut to\n :meth:`rename_multireddit` of the reddit_session."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _convert(reddit_session, data):\n retval = Redditor(reddit_session, data['name'], fetch=False)\n retval.id = data['id'].split('_')[1] # pylint: disable=C0103,W0201\n return retval", "response": "Return a Redditor object from the data."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef from_api_response(cls, reddit_session, json_dict):\n # The WikiPage response does not contain the necessary information\n # in the JSON response to determine the name of the page nor the\n # subreddit it belongs to. Thus we must extract this information\n # from the request URL.\n # pylint: disable=W0212\n parts = reddit_session._request_url.split('/', 6)\n # pylint: enable=W0212\n subreddit = parts[4]\n page = parts[6].split('.', 1)[0]\n return cls(reddit_session, subreddit, page, json_dict=json_dict)", "response": "Return an instance of the appropriate class from the json_dict."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nadd an editor to this wiki page.", "response": "def add_editor(self, username, _delete=False, *args, **kwargs):\n \"\"\"Add an editor to this wiki page.\n\n :param username: The name or Redditor object of the user to add.\n :param _delete: If True, remove the user as an editor instead.\n Please use :meth:`remove_editor` rather than setting it manually.\n\n Additional parameters are passed into\n :meth:`~praw.__init__.BaseReddit.request_json`.\n \"\"\"\n url = self.reddit_session.config['wiki_page_editor']\n url = url.format(subreddit=six.text_type(self.subreddit),\n method='del' if _delete else 'add')\n\n data = {'page': self.page,\n 'username': six.text_type(username)}\n return self.reddit_session.request_json(url, data=data, *args,\n **kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning the settings for this wiki page.", "response": "def get_settings(self, *args, **kwargs):\n \"\"\"Return the settings for this wiki page.\n\n Includes permission level, names of editors, and whether\n the page is listed on /wiki/pages.\n\n Additional parameters are passed into\n :meth:`~praw.__init__.BaseReddit.request_json`\n \"\"\"\n url = self.reddit_session.config['wiki_page_settings']\n url = url.format(subreddit=six.text_type(self.subreddit),\n page=self.page)\n return self.reddit_session.request_json(url, *args, **kwargs)['data']"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef edit(self, *args, **kwargs):\n return self.subreddit.edit_wiki_page(self.page, *args, **kwargs)", "response": "Edit the wiki page."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef edit_settings(self, permlevel, listed, *args, **kwargs):\n url = self.reddit_session.config['wiki_page_settings']\n url = url.format(subreddit=six.text_type(self.subreddit),\n page=self.page)\n data = {'permlevel': permlevel,\n 'listed': 'on' if listed else 'off'}\n\n return self.reddit_session.request_json(url, data=data, *args,\n **kwargs)['data']", "response": "Edit the settings for this individual wiki page."} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nremoves an editor from this wiki page.", "response": "def remove_editor(self, username, *args, **kwargs):\n \"\"\"Remove an editor from this wiki page.\n\n :param username: The name or Redditor object of the user to remove.\n\n This method points to :meth:`add_editor` with _delete=True.\n\n Additional parameters are are passed to :meth:`add_editor` and\n subsequently into :meth:`~praw.__init__.BaseReddit.request_json`.\n \"\"\"\n return self.add_editor(username=username, _delete=True, *args,\n **kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _convert(reddit_session, data):\n # TODO: The _request_url hack shouldn't be necessary\n # pylint: disable=W0212\n subreddit = reddit_session._request_url.rsplit('/', 4)[1]\n # pylint: enable=W0212\n return WikiPage(reddit_session, subreddit, data, fetch=False)", "response": "Return a WikiPage object from the data."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef comment_stream(reddit_session, subreddit, limit=None, verbosity=1):\n get_function = partial(reddit_session.get_comments,\n six.text_type(subreddit))\n return _stream_generator(get_function, limit, verbosity)", "response": "Return a generator that yields new comments from the provided subreddit."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a generator that yields new submissions from the provided subreddit.", "response": "def submission_stream(reddit_session, subreddit, limit=None, verbosity=1):\n \"\"\"Indefinitely yield new submissions from the provided subreddit.\n\n Submissions are yielded from oldest to newest.\n\n :param reddit_session: The reddit_session to make requests from. In all the\n examples this is assigned to the variable ``r``.\n :param subreddit: Either a subreddit object, or the name of a\n subreddit. Use `all` to get the submissions stream for all submissions\n made to reddit.\n :param limit: The maximum number of submissions to fetch in a single\n iteration. When None, fetch all available submissions (reddit limits\n this to 1000 (or multiple of 1000 for multi-subreddits). If this number\n is too small, submissions may be missed. Since there isn't a limit to\n the number of submissions that can be retrieved from r/all, the limit\n will be set to 1000 when limit is None.\n :param verbosity: A number that controls the amount of output produced to\n stderr. <= 0: no output; >= 1: output the total number of submissions\n processed and provide the short-term number of submissions processed\n per second; >= 2: output when additional delays are added in order to\n avoid subsequent unexpected http errors. >= 3: output debugging\n information regarding the submission stream. (Default: 1)\n\n \"\"\"\n if six.text_type(subreddit).lower() == \"all\":\n if limit is None:\n limit = 1000\n if not hasattr(subreddit, 'reddit_session'):\n subreddit = reddit_session.get_subreddit(subreddit)\n return _stream_generator(subreddit.get_new, limit, verbosity)"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef valid_redditors(redditors, sub):\n simplified = list(set(six.text_type(x).lower() for x in redditors))\n return [sub.reddit_session.get_redditor(simplified[i], fetch=False)\n for (i, resp) in enumerate(sub.set_flair_csv(\n ({'user': x, 'flair_text': x} for x in simplified)))\n if resp['ok']]", "response": "Return a list of valid redditor instances."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef submissions_between(reddit_session,\n subreddit,\n lowest_timestamp=None,\n highest_timestamp=None,\n newest_first=True,\n extra_cloudsearch_fields=None,\n verbosity=1):\n \"\"\"Yield submissions between two timestamps.\n\n If both ``highest_timestamp`` and ``lowest_timestamp`` are unspecified,\n yields all submissions in the ``subreddit``.\n\n Submissions are yielded from newest to oldest(like in the \"new\" queue).\n\n :param reddit_session: The reddit_session to make requests from. In all the\n examples this is assigned to the variable ``r``.\n :param subreddit: Either a subreddit object, or the name of a\n subreddit. Use `all` to get the submissions stream for all submissions\n made to reddit.\n :param lowest_timestamp: The lower bound for ``created_utc`` atributed of\n submissions.\n (Default: subreddit's created_utc or 0 when subreddit == \"all\").\n :param highest_timestamp: The upper bound for ``created_utc`` attribute\n of submissions. (Default: current unix time)\n NOTE: both highest_timestamp and lowest_timestamp are proper\n unix timestamps(just like ``created_utc`` attributes)\n :param newest_first: If set to true, yields submissions\n from newest to oldest. Otherwise yields submissions\n from oldest to newest\n :param extra_cloudsearch_fields: Allows extra filtering of results by\n parameters like author, self. Full list is available here:\n https://www.reddit.com/wiki/search\n :param verbosity: A number that controls the amount of output produced to\n stderr. <= 0: no output; >= 1: output the total number of submissions\n processed; >= 2: output debugging information regarding\n the search queries. (Default: 1)\n \"\"\"\n def debug(msg, level):\n if verbosity >= level:\n sys.stderr.write(msg + '\\n')\n\n def format_query_field(k, v):\n if k in [\"nsfw\", \"self\"]:\n # even though documentation lists \"no\" and \"yes\"\n # as possible values, in reality they don't work\n if v not in [0, 1, \"0\", \"1\"]:\n raise PRAWException(\"Invalid value for the extra\"\n \"field {}. Only '0' and '1' are\"\n \"valid values.\".format(k))\n return \"{}:{}\".format(k, v)\n return \"{}:'{}'\".format(k, v)\n\n if extra_cloudsearch_fields is None:\n extra_cloudsearch_fields = {}\n\n extra_query_part = \" \".join(\n [format_query_field(k, v) for (k, v)\n in sorted(extra_cloudsearch_fields.items())]\n )\n\n if highest_timestamp is None:\n highest_timestamp = int(time.time()) + REDDIT_TIMESTAMP_OFFSET\n else:\n highest_timestamp = int(highest_timestamp) + REDDIT_TIMESTAMP_OFFSET\n\n if lowest_timestamp is not None:\n lowest_timestamp = int(lowest_timestamp) + REDDIT_TIMESTAMP_OFFSET\n elif not isinstance(subreddit, six.string_types):\n lowest_timestamp = int(subreddit.created)\n elif subreddit not in (\"all\", \"contrib\", \"mod\", \"friend\"):\n lowest_timestamp = int(reddit_session.get_subreddit(subreddit).created)\n else:\n lowest_timestamp = 0\n\n original_highest_timestamp = highest_timestamp\n original_lowest_timestamp = lowest_timestamp\n\n # When making timestamp:X..Y queries, reddit misses submissions\n # inside X..Y range, but they can be found inside Y..Z range\n # It is not clear what is the value of Z should be, but it seems\n # like the difference is usually about ~1 hour or less\n # To be sure, let's set the workaround offset to 2 hours\n out_of_order_submissions_workaround_offset = 7200\n highest_timestamp += out_of_order_submissions_workaround_offset\n lowest_timestamp -= out_of_order_submissions_workaround_offset\n\n # Those parameters work ok, but there may be a better set of parameters\n window_size = 60 * 60\n search_limit = 100\n min_search_results_in_window = 50\n window_adjustment_ratio = 1.25\n backoff = BACKOFF_START\n\n processed_submissions = 0\n prev_win_increased = False\n prev_win_decreased = False\n\n while highest_timestamp >= lowest_timestamp:\n try:\n if newest_first:\n t1 = max(highest_timestamp - window_size, lowest_timestamp)\n t2 = highest_timestamp\n else:\n t1 = lowest_timestamp\n t2 = min(lowest_timestamp + window_size, highest_timestamp)\n\n search_query = 'timestamp:{}..{}'.format(t1, t2)\n if extra_query_part:\n search_query = \"(and {} {})\".format(search_query,\n extra_query_part)\n\n debug(search_query, 3)\n search_results = list(reddit_session.search(search_query,\n subreddit=subreddit,\n limit=search_limit,\n syntax='cloudsearch',\n sort='new'))\n\n debug(\"Received {0} search results for query {1}\"\n .format(len(search_results), search_query),\n 2)\n\n backoff = BACKOFF_START\n except HTTPException as exc:\n debug(\"{0}. Sleeping for {1} seconds\".format(exc, backoff), 2)\n time.sleep(backoff)\n backoff *= 2\n continue\n\n if len(search_results) >= search_limit:\n power = 2 if prev_win_decreased else 1\n window_size = int(window_size / window_adjustment_ratio**power)\n prev_win_decreased = True\n debug(\"Decreasing window size to {0} seconds\".format(window_size),\n 2)\n # Since it is possible that there are more submissions\n # in the current window, we have to re-do the request\n # with reduced window\n continue\n else:\n prev_win_decreased = False\n\n search_results = [s for s in search_results\n if original_lowest_timestamp <= s.created and\n s.created <= original_highest_timestamp]\n\n for submission in sorted(search_results,\n key=attrgetter('created_utc', 'id'),\n reverse=newest_first):\n yield submission\n\n processed_submissions += len(search_results)\n debug('Total processed submissions: {}'\n .format(processed_submissions), 1)\n\n if newest_first:\n highest_timestamp -= (window_size + 1)\n else:\n lowest_timestamp += (window_size + 1)\n\n if len(search_results) < min_search_results_in_window:\n power = 2 if prev_win_increased else 1\n window_size = int(window_size * window_adjustment_ratio**power)\n prev_win_increased = True\n debug(\"Increasing window size to {0} seconds\"\n .format(window_size), 2)\n else:\n prev_win_increased = False", "response": "Return a generator of submissions between two timestamps."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef chunk_sequence(sequence, chunk_length, allow_incomplete=True):\n (complete, leftover) = divmod(len(sequence), chunk_length)\n if not allow_incomplete:\n leftover = 0\n\n chunk_count = complete + min(leftover, 1)\n\n chunks = []\n for x in range(chunk_count):\n left = chunk_length * x\n right = left + chunk_length\n chunks.append(sequence[left:right])\n\n return chunks", "response": "Given a sequence divide it into chunks of length chunk_length."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nconvert strings representing base36 numbers into an integer.", "response": "def convert_id36_to_numeric_id(id36):\n \"\"\"Convert strings representing base36 numbers into an integer.\"\"\"\n if not isinstance(id36, six.string_types) or id36.count(\"_\") > 0:\n raise ValueError(\"must supply base36 string, not fullname (e.g. use \"\n \"xxxxx, not t3_xxxxx)\")\n return int(id36, 36)"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nconvert an integer into its base36 string representation.", "response": "def convert_numeric_id_to_id36(numeric_id):\n \"\"\"Convert an integer into its base36 string representation.\n\n This method has been cleaned up slightly to improve readability. For more\n info see:\n\n https://github.com/reddit/reddit/blob/master/r2/r2/lib/utils/_utils.pyx\n\n https://www.reddit.com/r/redditdev/comments/n624n/submission_ids_question/\n\n https://en.wikipedia.org/wiki/Base36\n \"\"\"\n # base36 allows negative numbers, but reddit does not\n if not isinstance(numeric_id, six.integer_types) or numeric_id < 0:\n raise ValueError(\"must supply a positive int/long\")\n\n # Alphabet used for base 36 conversion\n alphabet = '0123456789abcdefghijklmnopqrstuvwxyz'\n alphabet_len = len(alphabet)\n\n # Temp assign\n current_number = numeric_id\n base36 = []\n\n # Current_number must be greater than alphabet length to while/divmod\n if 0 <= current_number < alphabet_len:\n return alphabet[current_number]\n\n # Break up into chunks\n while current_number != 0:\n current_number, rem = divmod(current_number, alphabet_len)\n base36.append(alphabet[rem])\n\n # String is built in reverse order\n return ''.join(reversed(base36))"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef flatten_tree(tree, nested_attr='replies', depth_first=False):\n stack = deque(tree)\n extend = stack.extend if depth_first else stack.extendleft\n retval = []\n while stack:\n item = stack.popleft()\n nested = getattr(item, nested_attr, None)\n if nested:\n extend(nested)\n retval.append(item)\n return retval", "response": "Return a flattened version of the passed in tree."} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning url after stripping trailing. json and trailing slashes.", "response": "def normalize_url(url):\n \"\"\"Return url after stripping trailing .json and trailing slashes.\"\"\"\n if url.endswith('.json'):\n url = url[:-5]\n if url.endswith('/'):\n url = url[:-1]\n return url"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef add(self, item):\n if item in self._set:\n self._fifo.remove(item)\n elif len(self._set) == self.max_items:\n self._set.remove(self._fifo.pop(0))\n self._fifo.append(item)\n self._set.add(item)", "response": "Add an item to the set discarding the oldest item if necessary."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ndownload an entire table as a zip file.", "response": "def export_table(datatable_code, **kwargs):\n \"\"\"Downloads an entire table as a zip file.\n :param str datatable_code: The datatable code to download, such as MER/F1\n :param str filename: The filename for the download. \\\n If not specified, will download to the current working directory\n :param str api_key: Most databases require api_key for bulk download\n \"\"\"\n\n # discourage users from using authtoken\n if 'authtoken' in kwargs:\n raise InvalidRequestError(Message.ERROR_AUTHTOKEN_NOT_SUPPORTED)\n\n ApiKeyUtil.init_api_key_from_args(kwargs)\n\n filename = kwargs.pop('filename', '.')\n return Datatable(datatable_code).download_file(filename, **kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ngets a single or multiple dataset from Quandl.", "response": "def get(dataset, **kwargs):\n \"\"\"Return dataframe of requested dataset from Quandl.\n :param dataset: str or list, depending on single dataset usage or multiset usage\n Dataset codes are available on the Quandl website\n :param str api_key: Downloads are limited to 50 unless api_key is specified\n :param str start_date, end_date: Optional datefilers, otherwise entire\n dataset is returned\n :param str collapse: Options are daily, weekly, monthly, quarterly, annual\n :param str transform: options are diff, rdiff, cumul, and normalize\n :param int rows: Number of rows which will be returned\n :param str order: options are asc, desc. Default: `asc`\n :param str returns: specify what format you wish your dataset returned as,\n either `numpy` for a numpy ndarray or `pandas`. Default: `pandas`\n :returns: :class:`pandas.DataFrame` or :class:`numpy.ndarray`\n Note that Pandas expects timeseries data to be sorted ascending for most\n timeseries functionality to work.\n Any other `kwargs` passed to `get` are sent as field/value params to Quandl\n with no interference.\n \"\"\"\n\n _convert_params_to_v3(kwargs)\n\n data_format = kwargs.pop('returns', 'pandas')\n\n ApiKeyUtil.init_api_key_from_args(kwargs)\n\n # Check whether dataset is given as a string\n # (for a single dataset) or an array (for a multiset call)\n\n # Unicode String\n if isinstance(dataset, string_types):\n dataset_args = _parse_dataset_code(dataset)\n if dataset_args['column_index'] is not None:\n kwargs.update({'column_index': dataset_args['column_index']})\n data = Dataset(dataset_args['code']).data(params=kwargs, handle_column_not_found=True)\n # Array\n elif isinstance(dataset, list):\n args = _build_merged_dataset_args(dataset)\n # handle_not_found_error if set to True will add an empty DataFrame\n # for a non-existent dataset instead of raising an error\n data = MergedDataset(args).data(params=kwargs,\n handle_not_found_error=True,\n handle_column_not_found=True)\n # If wrong format\n else:\n raise InvalidRequestError(Message.ERROR_DATASET_FORMAT)\n\n if data_format == 'numpy':\n return data.to_numpy()\n return data.to_pandas()"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ndownloading an entire database.", "response": "def bulkdownload(database, **kwargs):\n \"\"\"Downloads an entire database.\n :param str database: The database code to download\n :param str filename: The filename for the download. \\\n If not specified, will download to the current working directory\n :param str api_key: Most databases require api_key for bulk download\n :param str download_type: 'partial' or 'complete'. \\\n See: https://www.quandl.com/docs/api#database-metadata\n \"\"\"\n\n # discourage users from using authtoken\n if 'authtoken' in kwargs:\n raise InvalidRequestError(Message.ERROR_AUTHTOKEN_NOT_SUPPORTED)\n\n ApiKeyUtil.init_api_key_from_args(kwargs)\n\n filename = kwargs.pop('filename', '.')\n return Database(database).bulk_download_to_file(filename, params=kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns a JSON - RPC map of available calls.", "response": "def jsonrpc_map(self):\n \"\"\" Map of json-rpc available calls.\n\n :return str:\n\n \"\"\"\n result = \"

JSON-RPC map

{0}
\".format(\"\\n\\n\".join([\n \"{0}: {1}\".format(fname, f.__doc__)\n for fname, f in self.dispatcher.items()\n ]))\n return Response(result)"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef command(self, cmd, type=None):\n\n type = type or 'text'\n self.hunter.script(cmd, lang=type)", "response": "\u901a\u8fc7hunter\u8c03\u7528gm\u6307\u4ee4\uff0c\u53ef\u8c03\u7528hunter\u6307\u4ee4\u5e93\u4e2d\u5b9a\u4e49\u7684\u6240\u6709\u6307\u4ee4\uff0c\u4e5f\u53ef\u4ee5\u8c03\u7528text\u7c7b\u578b\u7684gm\u6307\u4ee4\n gm\u6307\u4ee4\u76f8\u5173\u529f\u80fd\u8bf7\u53c2\u8003safaia GM\u6307\u4ee4\u6269\u5c55\u6a21\u5757\n\n :param cmd: \u6307\u4ee4\n :param type: \u8bed\u8a00\uff0c\u9ed8\u8ba4text\n :return: None"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef add_method(self, f, name=None):\n self.method_map[name or f.__name__] = f\n return f", "response": "Add a method to the dispatcher."} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nbuild a mapping from prototype to public methods.", "response": "def build_method_map(self, prototype, prefix=''):\n \"\"\" Add prototype methods to the dispatcher.\n\n Parameters\n ----------\n prototype : object or dict\n Initial method mapping.\n If given prototype is a dictionary then all callable objects will\n be added to dispatcher.\n If given prototype is an object then all public methods will\n be used.\n prefix: string, optional\n Prefix of methods\n\n \"\"\"\n if not isinstance(prototype, dict):\n prototype = dict((method, getattr(prototype, method))\n for method in dir(prototype)\n if not method.startswith('_'))\n\n for attr, method in prototype.items():\n if callable(method):\n self[prefix + attr] = method"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nyielding all the content of the data in the order they were added.", "response": "def input(self, data):\n \"\"\" \u5c0f\u6570\u636e\u7247\u6bb5\u62fc\u63a5\u6210\u5b8c\u6574\u6570\u636e\u5305\n \u5982\u679c\u5185\u5bb9\u8db3\u591f\u5219yield\u6570\u636e\u5305\n \"\"\"\n self.buf += data\n while len(self.buf) > HEADER_SIZE:\n data_len = struct.unpack('i', self.buf[0:HEADER_SIZE])[0]\n if len(self.buf) >= data_len + HEADER_SIZE:\n content = self.buf[HEADER_SIZE:data_len + HEADER_SIZE]\n self.buf = self.buf[data_len + HEADER_SIZE:]\n yield content\n else:\n break"} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\npacks the content of a sequence of bytes into a string", "response": "def pack(content):\n \"\"\" content should be str\n \"\"\"\n if isinstance(content, six.text_type):\n content = content.encode(\"utf-8\")\n return struct.pack('i', len(content)) + content"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef unpack(data):\n length = struct.unpack('i', data[0:HEADER_SIZE])\n return length[0], data[HEADER_SIZE:]", "response": "unpacks the content of a log record into a tuple"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef sendFragmentStart(self, data):\n opcode = BINARY\n if _check_unicode(data):\n opcode = TEXT\n self._sendMessage(True, opcode, data)", "response": "Send a start of a data fragment stream to a websocket client."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef sendMessage(self, data):\n opcode = BINARY\n if _check_unicode(data):\n opcode = TEXT\n self._sendMessage(False, opcode, data)", "response": "Send a websocket data frame to the client."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nhandle the request data.", "response": "def handle_request(cls, request, dispatcher):\n \"\"\" Handle request data.\n\n At this moment request has correct jsonrpc format.\n\n :param dict request: data parsed from request_str.\n :param jsonrpc.dispatcher.Dispatcher dispatcher:\n\n .. versionadded: 1.8.0\n\n \"\"\"\n rs = request if isinstance(request, JSONRPC20BatchRequest) \\\n else [request]\n responses = [r for r in cls._get_responses(rs, dispatcher)\n if r is not None]\n\n # notifications\n if not responses:\n return\n\n if isinstance(request, JSONRPC20BatchRequest):\n return JSONRPC20BatchResponse(*responses)\n else:\n return responses[0]"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn an iterator over the responses from a list of JSON - RPC requests.", "response": "def _get_responses(cls, requests, dispatcher):\n \"\"\" Response to each single JSON-RPC Request.\n\n :return iterator(JSONRPC20Response):\n\n .. versionadded: 1.9.0\n TypeError inside the function is distinguished from Invalid Params.\n\n \"\"\"\n for request in requests:\n def response(**kwargs):\n return cls.RESPONSE_CLASS_MAP[request.JSONRPC_VERSION](\n _id=request._id, **kwargs)\n\n try:\n method = dispatcher[request.method]\n except KeyError:\n output = response(error=JSONRPCMethodNotFound()._data)\n else:\n try:\n result = method(*request.args, **request.kwargs)\n except JSONRPCDispatchException as e:\n output = response(error=e.error._data)\n except Exception as e:\n data = {\n \"type\": e.__class__.__name__,\n \"args\": e.args,\n \"message\": str(e),\n }\n if isinstance(e, TypeError) and is_invalid_params(\n method, *request.args, **request.kwargs):\n output = response(\n error=JSONRPCInvalidParams(data=data)._data)\n else:\n # logger.exception(\"API Exception: {0}\".format(data))\n print(\"API Exception: {0}\".format(data))\n output = response(\n error=JSONRPCServerError(data=data)._data)\n else:\n output = response(result=result)\n finally:\n if not request.is_notification:\n yield output"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef jsonrpc(self, request):\n if request.method != \"POST\":\n return HttpResponseNotAllowed([\"POST\"])\n\n request_str = request.body.decode('utf8')\n try:\n jsonrpc_request = JSONRPCRequest.from_json(request_str)\n except (TypeError, ValueError, JSONRPCInvalidRequestException):\n response = JSONRPCResponseManager.handle(\n request_str, self.dispatcher)\n else:\n jsonrpc_request.params = jsonrpc_request.params or {}\n jsonrpc_request_params = copy.copy(jsonrpc_request.params)\n if isinstance(jsonrpc_request.params, dict):\n jsonrpc_request.params.update(request=request)\n\n t1 = time.time()\n response = JSONRPCResponseManager.handle_request(\n jsonrpc_request, self.dispatcher)\n t2 = time.time()\n logger.info('{0}({1}) {2:.2f} sec'.format(\n jsonrpc_request.method, jsonrpc_request_params, t2 - t1))\n\n if response:\n def serialize(s):\n return json.dumps(s, cls=DatetimeDecimalEncoder)\n\n response.serialize = serialize\n response = response.json\n\n return HttpResponse(response, content_type=\"application/json\")", "response": "JSON - RPC 2. 0 handler."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef jsonrpc_map(self, request):\n result = \"

JSON-RPC map

{0}
\".format(\"\\n\\n\".join([\n \"{0}: {1}\".format(fname, f.__doc__)\n for fname, f in self.dispatcher.items()\n ]))\n return HttpResponse(result)", "response": "Return a JSON - RPC map of available calls."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef discretize(self, contact_id=0, accuracy=0.004, dt=0.001):\n\n if not self.event_points:\n return []\n\n events = []\n action_dt = accuracy / self.speed\n dt = dt or action_dt\n\n ep0 = self.event_points[0]\n for _ in range(int(ep0[0] / dt)):\n events.append(['s', dt])\n events.append(['d', ep0[1], contact_id])\n for i, ep in enumerate(self.event_points[1:]):\n prev_ts = self.event_points[i][0]\n curr_ts = ep[0]\n p0 = self.event_points[i][1]\n p1 = ep[1]\n if p0 == p1:\n # hold\n for _ in range(int((curr_ts - prev_ts) / dt)):\n events.append(['s', dt])\n else:\n # move\n dpoints = track_sampling([p0, p1], accuracy)\n for p in dpoints:\n events.append(['m', p, contact_id])\n for _ in range(int(action_dt / dt)):\n events.append(['s', dt])\n\n events.append(['u', contact_id])\n return events", "response": "Return a list of discretized events for this motion track."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef getRoot(self):\n\n root = Node(self.dumpHierarchy())\n self._linkParent(root)\n return root", "response": "Dump a hierarchy immediately from target runtime and store into a Node object."} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nchecks whether function func accepts parameters args kwargs.", "response": "def is_invalid_params(func, *args, **kwargs):\n \"\"\" Check, whether function 'func' accepts parameters 'args', 'kwargs'.\n\n NOTE: Method is called after funct(*args, **kwargs) generated TypeError,\n it is aimed to destinguish TypeError because of invalid parameters from\n TypeError from inside the function.\n\n .. versionadded: 1.9.0\n\n \"\"\"\n # For builtin functions inspect.getargspec(funct) return error. If builtin\n # function generates TypeError, it is because of wrong parameters.\n if not inspect.isfunction(func):\n return True\n\n funcargs, varargs, varkwargs, defaults = inspect.getargspec(func)\n if defaults:\n funcargs = funcargs[:-len(defaults)]\n\n if args and len(args) != len(funcargs):\n return True\n if kwargs and set(kwargs.keys()) != set(funcargs):\n return True\n\n if not args and not kwargs and funcargs:\n return True\n\n return False"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef getAppRefByBundleId(cls, bundleId):\n ra = AppKit.NSRunningApplication\n # return value (apps) is always an array. if there is a match it will\n # have an item, otherwise it won't.\n apps = ra.runningApplicationsWithBundleIdentifier_(bundleId)\n if len(apps) == 0:\n raise ValueError(('Specified bundle ID not found in '\n 'running apps: %s' % bundleId))\n pid = apps[0].processIdentifier()\n return cls.getAppRefByPid(pid)", "response": "Get the top level element for the application with the specified bundle ID."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef driver(self):\n\n if not self._driver:\n raise AttributeError(\"`driver` is not bound on this agent implementation({}). \"\n \"Do you forget to call `super().on_bind_driver` when you override the method \"\n \"`on_bind_driver` in your sub class?\"\n .format(repr(self)))\n return self._driver", "response": "Returns the driver of this agent."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ndumping the hierarchy structure.", "response": "def dumpHierarchyImpl(self, node, onlyVisibleNode=True):\n \"\"\"\n Crawl the hierarchy tree using the simple DFS algorithm. The ``dump`` procedure is the engine independent as\n the hierarchy structure is wrapped by :py:class:`AbstractNode ` and therefore the\n ``dump`` procedure can be algorithmized.\n\n Following code demonstrates the simplest implementation. Feel free to implement your own algorithms to\n optimize the performance.\n\n .. note:: Do not explicitly call this method as this is an internal function, call\n :py:meth:`dumpHierarchy() ` function instead\n if you want to dump the hierarchy.\n\n Args:\n node(:py:class:`inherit from AbstractNode `): root node of the hierarchy to be\n dumped\n onlyVisibleNode(:obj:`bool`): dump only the visible nodes or all nodes, default to True\n\n Returns:\n :obj:`dict`: json serializable dict holding the whole hierarchy data\n \"\"\"\n\n if not node:\n return None\n\n payload = {}\n\n # filter out all None values\n for attrName, attrVal in node.enumerateAttrs():\n if attrVal is not None:\n payload[attrName] = attrVal\n\n result = {}\n children = []\n for child in node.getChildren():\n if not onlyVisibleNode or child.getAttr('visible'):\n children.append(self.dumpHierarchyImpl(child, onlyVisibleNode))\n if len(children) > 0:\n result['children'] = children\n\n result['name'] = payload.get('name') or node.getAttr('name')\n result['payload'] = payload\n\n return result"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nselect the direct child from the UI element ( s given by the query expression.", "response": "def child(self, name=None, **attrs):\n \"\"\"\n Select the direct child(ren) from the UI element(s) given by the query expression, see ``QueryCondition`` for\n more details about the selectors.\n\n Args:\n name: query expression of attribute \"name\", i.e. the UI elements with ``name`` name will be selected\n attrs: other query expression except for the ``name``\n\n Returns:\n :py:class:`UIObjectProxy `: a new UI proxy object representing the child(ren) of\n current UI element(s)\n \"\"\"\n\n sub_query = build_query(name, **attrs)\n query = ('/', (self.query, sub_query))\n obj = UIObjectProxy(self.poco)\n obj.query = query\n return obj"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nselect the direct parent of the first UI element in the set of UI elements.", "response": "def parent(self):\n \"\"\"\n Select the direct child(ren) from the UI element(s) given by the query expression, see ``QueryCondition`` for\n more details about the selectors.\n\n Warnings:\n Experimental method, may not be available for all drivers.\n\n Returns:\n :py:class:`UIObjectProxy `: a new UI proxy object representing the direct parent\n of the first UI element.\n \"\"\"\n\n sub_query = build_query(None) # as placeholder\n query = ('^', (self.query, sub_query))\n obj = UIObjectProxy(self.poco)\n obj.query = query\n return obj"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nperform the click action on the UI element represented by this object.", "response": "def click(self, focus=None, sleep_interval=None):\n \"\"\"\n Perform the click action on the UI element(s) represented by the UI proxy. If this UI proxy represents a set of\n UI elements, the first one in the set is clicked and the anchor point of the UI element is used as the default\n one. It is also possible to click another point offset by providing ``focus`` argument.\n\n See ``CoordinateSystem`` for more details.\n\n Args:\n focus (2-:obj:`tuple`/2-:obj:`list`/:obj:`str`): an offset point (x, y) from the top left corner of the UI\n element(s), values must be in range of 0~1. This argument can be also specified by 'anchor' or 'center'.\n 'Center' means to click the center of bounding box of UI element. \n sleep_interval: number of seconds to wait after this action. Default is None which is the default sleep\n interval. This value can be configured by Poco initialization. See configuration at poco\n :py:class:`initialization ` for more details.\n\n Raises:\n PocoNoSuchNodeException: raised when the UI element does not exist\n \"\"\"\n\n focus = focus or self._focus or 'anchor'\n pos_in_percentage = self.get_position(focus)\n self.poco.pre_action('click', self, pos_in_percentage)\n ret = self.poco.click(pos_in_percentage)\n if sleep_interval:\n time.sleep(sleep_interval)\n else:\n self.poco.wait_stable()\n self.poco.post_action('click', self, pos_in_percentage)\n return ret"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef long_click(self, duration=2.0):\n\n try:\n duration = float(duration)\n except ValueError:\n raise ValueError('Argument `duration` should be . Got {}'.format(repr(duration)))\n\n pos_in_percentage = self.get_position(self._focus or 'anchor')\n self.poco.pre_action('long_click', self, pos_in_percentage)\n ret = self.poco.long_click(pos_in_percentage, duration)\n self.poco.post_action('long_click', self, pos_in_percentage)\n return ret", "response": "Long click action on the UI element represented by this object."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nperforming a swipe action given by the direction from this UI element.", "response": "def swipe(self, direction, focus=None, duration=0.5):\n \"\"\"\n Perform a swipe action given by the direction from this UI element. For notices and limitations see\n :py:meth:`.click() `. \n\n Args:\n direction (2-:obj:`tuple`/2-:obj:`list`/:obj:`str`): coordinates (x, y) in NormalizedCoordinate system, it\n can be also specified as 'up', 'down', 'left', 'right'. Swipe 'up' is equivalent to [0, -0.1], swipe\n 'down' is equivalent to [0, 0.1], swipe 'left' is equivalent to [-0.1, 0] and swipe 'right' is equivalent\n to [0, 0.1]\n focus (2-:obj:`tuple`/2-:obj:`list`/:obj:`str`): see :py:meth:`.click() `\n for more details\n duration (:py:obj:`float`): time interval in which the action is performed\n\n Raises:\n PocoNoSuchNodeException: raised when the UI element does not exist\n \"\"\"\n\n try:\n duration = float(duration)\n except ValueError:\n raise ValueError('Argument `duration` should be . Got {}'.format(repr(duration)))\n\n focus = focus or self._focus or 'anchor'\n dir_vec = self._direction_vector_of(direction)\n origin = self.get_position(focus)\n self.poco.pre_action('swipe', self, (origin, dir_vec))\n ret = self.poco.swipe(origin, direction=dir_vec, duration=duration)\n self.poco.post_action('swipe', self, (origin, dir_vec))\n return ret"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef scroll(self, direction='vertical', percent=0.6, duration=2.0):\n\n if direction not in ('vertical', 'horizontal'):\n raise ValueError('Argument `direction` should be one of \"vertical\" or \"horizontal\". Got {}'\n .format(repr(direction)))\n\n focus1 = self._focus or [0.5, 0.5]\n focus2 = list(focus1)\n half_distance = percent / 2\n if direction == 'vertical':\n focus1[1] += half_distance\n focus2[1] -= half_distance\n else:\n focus1[0] += half_distance\n focus2[0] -= half_distance\n\n return self.focus(focus1).drag_to(self.focus(focus2), duration=duration)", "response": "Simply touch down from point A and move to point B then release up finally. This action is performed within specific motion range and duration."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef pinch(self, direction='in', percent=0.6, duration=2.0, dead_zone=0.1):\n\n if direction not in ('in', 'out'):\n raise ValueError('Argument `direction` should be one of \"in\" or \"out\". Got {}'.format(repr(direction)))\n if dead_zone >= percent:\n raise ValueError('Argument `dead_zone` should not be greater than `percent`. dead_zoon={}, percent={}'\n .format(repr(dead_zone), repr(percent)))\n\n w, h = self.get_size()\n x, y = self.get_position()\n # focus = self._focus or [0.5, 0.5]\n tracks = make_pinching(direction, [x, y], [w, h], percent, dead_zone, duration)\n speed = math.sqrt(w * h) * (percent - dead_zone) / 2 / duration\n\n # \u901f\u5ea6\u6162\u7684\u65f6\u5019\uff0c\u7cbe\u5ea6\u9002\u5f53\u8981\u63d0\u9ad8\uff0c\u8fd9\u6837\u6709\u52a9\u4e8e\u63a7\u5236\u51c6\u786e\n ret = self.poco.apply_motion_tracks(tracks, accuracy=speed * 0.03)\n return ret", "response": "Pinchs the UI element to a specified duration."} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns a new copy with the given focus.", "response": "def focus(self, f):\n \"\"\"\n Get a new UI proxy copy with the given focus. Return a new UI proxy object as the UI proxy is immutable.\n\n Args:\n f (2-:obj:`tuple`/2-:obj:`list`/:obj:`str`): the focus point, it can be specified as 2-list/2-tuple\n coordinates (x, y) in NormalizedCoordinate system or as 'center' or 'anchor'.\n\n Returns:\n :py:class:`UIObjectProxy `: a new UI proxy object (copy)\n \"\"\"\n\n ret = copy.copy(self)\n ret._focus = f\n return ret"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ngetting the position of the UI elements in the system.", "response": "def get_position(self, focus=None):\n \"\"\"\n Get the position of the UI elements.\n\n Args:\n focus: focus point of UI proxy, see :py:meth:`.focus() ` for more details\n\n Returns:\n 2-list/2-tuple: coordinates (x, y) in NormalizedCoordinate system\n\n Raises:\n TypeError: raised when unsupported focus type is specified\n \"\"\"\n\n focus = focus or self._focus or 'anchor'\n if focus == 'anchor':\n pos = self.attr('pos')\n elif focus == 'center':\n x, y = self.attr('pos')\n w, h = self.get_size()\n ap_x, ap_y = self.attr(\"anchorPoint\")\n fx, fy = 0.5, 0.5\n pos = [x + w * (fx - ap_x), y + h * (fy - ap_y)]\n elif type(focus) in (list, tuple):\n x, y = self.attr('pos')\n w, h = self.get_size()\n ap_x, ap_y = self.attr(\"anchorPoint\")\n fx, fy = focus\n pos = [x + w * (fx - ap_x), y + h * (fy - ap_y)]\n else:\n raise TypeError('Unsupported focus type {}. '\n 'Only \"anchor/center\" or 2-list/2-tuple available.'.format(type(focus)))\n return pos"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef wait(self, timeout=3):\n\n start = time.time()\n while not self.exists():\n self.poco.sleep_for_polling_interval()\n if time.time() - start > timeout:\n break\n return self", "response": "Block and wait for max given time before the UI element appears."} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nblocks and wait until the UI element disappears within the given timeout.", "response": "def wait_for_disappearance(self, timeout=120):\n \"\"\"\n Block and wait until the UI element **disappears** within the given timeout.\n\n Args:\n timeout: maximum waiting time in seconds\n\n Raises:\n PocoTargetTimeout: when timeout\n \"\"\"\n\n start = time.time()\n while self.exists():\n self.poco.sleep_for_polling_interval()\n if time.time() - start > timeout:\n raise PocoTargetTimeout('disappearance', self)"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef attr(self, name):\n\n # to optimize performance speed, retrieve only the first matched element.\n # \u4f18\u5316\u901f\u5ea6\uff0c\u53ea\u9009\u62e9\u7b2c\u4e00\u4e2a\u5339\u914d\u5230\u7684\u8282\u70b9\n nodes = self._do_query(multiple=False)\n val = self.poco.agent.hierarchy.getAttr(nodes, name)\n if six.PY2 and isinstance(val, six.text_type):\n # \u6587\u672c\u7c7b\u578b\u7684\u5c5e\u6027\u503c\uff0c\u53ea\u5728python2\u91ccencode\u6210utf-8\u7684str\uff0cpython3\u4fdd\u6301str\u7c7b\u578b\n # \u8fd9\u662f\u4e3a\u4e86\u5728\u5199\u4ee3\u7801\u7684\u65f6\u5019\uff0c\u65e0\u8bbapy2/3\u59cb\u7ec8\u53ef\u4ee5\u50cf\u4e0b\u9762\u8fd9\u6837\u5199\n # node.attr('text') == '\u8282\u70b9\u5c5e\u6027\u503c'\n val = val.encode('utf-8')\n return val", "response": "Retrieves the attribute of the given name from the remote UI element."} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nchanges the value of an attribute on the UI element.", "response": "def setattr(self, name, val):\n \"\"\"\n Change the attribute value of the UI element. Not all attributes can be casted to text. If changing the\n immutable attributes or attributes which do not exist, the InvalidOperationException exception is raised.\n\n Args:\n name: attribute name\n val: new attribute value to cast\n\n Raises:\n InvalidOperationException: when it fails to set the attribute on UI element\n \"\"\"\n\n nodes = self._do_query(multiple=False)\n try:\n return self.poco.agent.hierarchy.setAttr(nodes, name, val)\n except UnableToSetAttributeException as e:\n raise InvalidOperationException('\"{}\" of \"{}\"'.format(str(e), self))"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_bounds(self):\n\n size = self.get_size()\n top_left = self.get_position([0, 0])\n\n # t, r, b, l\n bounds = [top_left[1], top_left[0] + size[0], top_left[1] + size[1], top_left[0]]\n return bounds", "response": "Get the parameters of bounding box of the UI element."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef compare(self, origin, pattern):\n\n if origin is None or pattern is None:\n return False\n return re.match(pattern, origin) is not None", "response": "Compare two strings in a log file."} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nseeing Also: :py:meth:`IMatcher.match `", "response": "def match(self, cond, node):\n \"\"\"\n See Also: :py:meth:`IMatcher.match `\n \"\"\"\n\n op, args = cond\n\n # \u6761\u4ef6\u5339\u914d\n if op == 'and':\n for arg in args:\n if not self.match(arg, node):\n return False\n return True\n\n if op == 'or':\n for arg in args:\n if self.match(arg, node):\n return True\n return False\n\n # \u5c5e\u6027\u5339\u914d\n comparator = self.comparators.get(op)\n if comparator:\n attribute, value = args\n targetValue = node.getAttr(attribute)\n return comparator.compare(targetValue, value)\n\n raise NoSuchComparatorException(op, 'poco.sdk.DefaultMatcher')"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nwaiting until any of the given objects show up before timeout.", "response": "def wait_for_any(self, objects, timeout=120):\n \"\"\"\n Wait until any of given UI proxies show up before timeout and return the first appeared UI proxy.\n All UI proxies will be polled periodically. See options :py:class:`poll_interval ` in\n ``Poco``'s initialization for more details.\n\n Args:\n objects (Iterable<:py:class:`UIObjectProxy `>): iterable object of the given UI \n proxies\n timeout (:obj:`float`): timeout in seconds, default is 120s\n\n Returns:\n :py:class:`UIObjectProxy `: the first appeared UI proxy\n\n Raises:\n PocoTargetTimeout: when none of UI proxies appeared before timeout\n \"\"\"\n\n start = time.time()\n while True:\n for obj in objects:\n if obj.exists():\n return obj\n if time.time() - start > timeout:\n raise PocoTargetTimeout('any to appear', objects)\n self.sleep_for_polling_interval()"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nwaits until all of the given objects appear before timeout.", "response": "def wait_for_all(self, objects, timeout=120):\n \"\"\"\n Wait until all of given UI proxies show up before timeout.\n All UI proxies will be polled periodically. See option :py:class:`poll_interval ` in \n ``Poco``'s initialization for more details.\n\n Args:\n objects (Iterable<:py:class:`UIObjectProxy `>): iterable object of the given UI \n proxies\n timeout (:obj:`float`): timeout in seconds, default is 120s\n\n Raises:\n PocoTargetTimeout: when not all of UI proxies appeared before timeout\n \"\"\"\n\n start = time.time()\n while True:\n all_exist = True\n for obj in objects:\n if not obj.exists():\n all_exist = False\n break\n if all_exist:\n return\n if time.time() - start > timeout:\n raise PocoTargetTimeout('all to appear', objects)\n self.sleep_for_polling_interval()"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nperform click action on target device at given coordinates.", "response": "def click(self, pos):\n \"\"\"\n Perform click (touch, tap, etc.) action on target device at given coordinates.\n\n The coordinates (x, y) are either a 2-list or 2-tuple. The coordinates values for x and y must be in the\n interval between 0 ~ 1 to represent the percentage of the screen. For example, the coordinates ``[0.5, 0.5]``\n represent the `center` of the screen and the coordinates ``[0, 0]`` represent the `top left corner`.\n\n See ``CoordinateSystem`` for more details about coordinate system.\n\n Examples:\n Click the point of ``(100, 100)`` of screen which resolution is ``(1920, 1080)``::\n\n poco.click([100.0 / 1920, 100.0 / 1080])\n\n Args:\n pos (:obj:`list(float, float)` / :obj:`tuple(float, float)`): coordinates (x, y) in range of 0 to 1\n\n Raises:\n InvalidOperationException: when clicked outside of the screen\n \"\"\"\n\n if not (0 <= pos[0] <= 1) or not (0 <= pos[1] <= 1):\n raise InvalidOperationException('Click position out of screen. pos={}'.format(repr(pos)))\n ret = self.agent.input.click(pos[0], pos[1])\n self.wait_stable()\n return ret"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef swipe(self, p1, p2=None, direction=None, duration=2.0):\n\n try:\n duration = float(duration)\n except ValueError:\n raise ValueError('Argument `duration` should be . Got {}'.format(repr(duration)))\n\n if not (0 <= p1[0] <= 1) or not (0 <= p1[1] <= 1):\n raise InvalidOperationException('Swipe origin out of screen. {}'.format(repr(p1)))\n if direction is not None:\n p2 = [p1[0] + direction[0], p1[1] + direction[1]]\n elif p2 is not None:\n p2 = p2\n else:\n raise TypeError('Swipe end not set.')\n return self.agent.input.swipe(p1[0], p1[1], p2[0], p2[1], duration)", "response": "Perform a swipe action on target device from point to point given by start point and end point and direction."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef long_click(self, pos, duration=2.0):\n\n try:\n duration = float(duration)\n except ValueError:\n raise ValueError('Argument `duration` should be . Got {}'.format(repr(duration)))\n\n if not (0 <= pos[0] <= 1) or not (0 <= pos[1] <= 1):\n raise InvalidOperationException('Click position out of screen. {}'.format(repr(pos)))\n return self.agent.input.longClick(pos[0], pos[1], duration)", "response": "Similar to click but press the screen for the given time interval and then release the entry."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef scroll(self, direction='vertical', percent=0.6, duration=2.0):\n\n if direction not in ('vertical', 'horizontal'):\n raise ValueError('Argument `direction` should be one of \"vertical\" or \"horizontal\". Got {}'\n .format(repr(direction)))\n\n start = [0.5, 0.5]\n half_distance = percent / 2\n if direction == 'vertical':\n start[1] += half_distance\n direction = [0, -percent]\n else:\n start[0] += half_distance\n direction = [-percent, 0]\n\n return self.swipe(start, direction=direction, duration=duration)", "response": "Scroll from the lower part to the upper part of the entire screen."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef pinch(self, direction='in', percent=0.6, duration=2.0, dead_zone=0.1):\n\n if direction not in ('in', 'out'):\n raise ValueError('Argument `direction` should be one of \"in\" or \"out\". Got {}'.format(repr(direction)))\n if dead_zone >= percent:\n raise ValueError('Argument `dead_zone` should not be greater than `percent`. dead_zoon={}, percent={}'\n .format(repr(dead_zone), repr(percent)))\n\n tracks = make_pinching(direction, [0.5, 0.5], [1, 1], percent, dead_zone, duration)\n speed = (percent - dead_zone) / 2 / duration\n\n # \u901f\u5ea6\u6162\u7684\u65f6\u5019\uff0c\u7cbe\u5ea6\u9002\u5f53\u8981\u63d0\u9ad8\uff0c\u8fd9\u6837\u6709\u52a9\u4e8e\u63a7\u5236\u51c6\u786e\n ret = self.apply_motion_tracks(tracks, accuracy=speed * 0.03)\n return ret", "response": "Pinch the squeezed entry in the entire screen."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get_instance(cls, device):\n\n if cls._nuis.get(device) is None:\n cls._nuis[device] = AndroidUiautomationPoco(device)\n return cls._nuis[device]", "response": "Get the AndroidUiautomationPoco instance for a given device."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef select(self, cond, multiple=False):\n\n return self.selectImpl(cond, multiple, self.getRoot(), 9999, True, True)", "response": "Select the set of items that satisfy the condition."} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nget API key details by id.", "response": "def find_by_id(key: str, user: str=None) -> Optional['ApiKey']:\n \"\"\"\n Get API key details.\n \"\"\"\n return ApiKey.from_db(db.get_key(key, user))"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef find_all(query: Query=None) -> List['ApiKey']:\n return [ApiKey.from_db(key) for key in db.get_keys(query)]", "response": "List all API keys that match the query."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef find_by_user(user: str) -> List['ApiKey']:\n return [ApiKey.from_db(key) for key in db.get_keys(qb.from_dict({'user': user}))]", "response": "List API keys for a user."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ncreating an admin API key.", "response": "def key(username, key, all):\n \"\"\"Create an admin API key.\"\"\"\n if username and username not in current_app.config['ADMIN_USERS']:\n raise click.UsageError('User {} not an admin'.format(username))\n\n def create_key(admin, key):\n key = ApiKey(\n user=admin,\n key=key,\n scopes=[Scope.admin, Scope.write, Scope.read],\n text='Admin key created by alertad script',\n expire_time=None\n )\n try:\n db.get_db() # init db on global app context\n key = key.create()\n except Exception as e:\n click.echo('ERROR: {}'.format(e))\n else:\n click.echo('{} {}'.format(key.key, key.user))\n\n if all:\n for admin in current_app.config['ADMIN_USERS']:\n create_key(admin, key)\n elif username:\n create_key(username, key)\n else:\n raise click.UsageError(\"Must set '--username' or use '--all'\")"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef keys():\n for admin in current_app.config['ADMIN_USERS']:\n try:\n db.get_db() # init db on global app context\n keys = ApiKey.find_by_user(admin)\n except Exception as e:\n click.echo('ERROR: {}'.format(e))\n else:\n for key in keys:\n click.echo('{:40} {}'.format(key.key, key.user))", "response": "List admin API keys."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ncreate admin users (BasicAuth only).", "response": "def user(username, password, all):\n \"\"\"Create admin users (BasicAuth only).\"\"\"\n if current_app.config['AUTH_PROVIDER'] != 'basic':\n raise click.UsageError('Not required for {} admin users'.format(current_app.config['AUTH_PROVIDER']))\n if username and username not in current_app.config['ADMIN_USERS']:\n raise click.UsageError('User {} not an admin'.format(username))\n if not username and not all:\n raise click.UsageError('Missing option \"--username\".')\n\n def create_user(admin):\n email = admin if '@' in admin else None\n user = User(\n name='Admin user',\n login=admin,\n password=generate_password_hash(password),\n roles=['admin'],\n text='Created by alertad script',\n email=email,\n email_verified=bool(email)\n )\n try:\n db.get_db() # init db on global app context\n user = user.create()\n except Exception as e:\n click.echo('ERROR: {}'.format(e))\n else:\n click.echo('{} {}'.format(user.id, user.name))\n\n if all:\n for admin in current_app.config['ADMIN_USERS']:\n create_user(admin)\n else:\n create_user(username)"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get_severity(self, alert):\n query = {\n 'environment': alert.environment,\n 'resource': alert.resource,\n '$or': [\n {\n 'event': alert.event,\n 'severity': {'$ne': alert.severity}\n },\n {\n 'event': {'$ne': alert.event},\n 'correlate': alert.event\n }],\n 'customer': alert.customer\n }\n r = self.get_db().alerts.find_one(query, projection={'severity': 1, '_id': 0})\n return r['severity'] if r else None", "response": "Get severity of correlated alert. Used to determine previous severity."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef is_flapping(self, alert, window=1800, count=2):\n pipeline = [\n {'$match': {\n 'environment': alert.environment,\n 'resource': alert.resource,\n 'event': alert.event,\n 'customer': alert.customer\n }},\n {'$unwind': '$history'},\n {'$match': {\n 'history.updateTime': {'$gt': datetime.utcnow() - timedelta(seconds=window)},\n 'history.type': 'severity'\n }},\n {'$group': {'_id': '$history.type', 'count': {'$sum': 1}}}\n ]\n responses = self.get_db().alerts.aggregate(pipeline)\n for r in responses:\n if r['count'] > count:\n return True\n return False", "response": "Return true if alert severity has changed more than X times in Y seconds"} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nupdates alert status, service, value, text, timeout and rawData, increment duplicate count and set repeat=True, and keep track of last receive id and time but don't append to history unless status changes.", "response": "def dedup_alert(self, alert, history):\n \"\"\"\n Update alert status, service, value, text, timeout and rawData, increment duplicate count and set\n repeat=True, and keep track of last receive id and time but don't append to history unless status changes.\n \"\"\"\n query = {\n 'environment': alert.environment,\n 'resource': alert.resource,\n 'event': alert.event,\n 'severity': alert.severity,\n 'customer': alert.customer\n }\n\n now = datetime.utcnow()\n update = {\n '$set': {\n 'status': alert.status,\n 'service': alert.service,\n 'value': alert.value,\n 'text': alert.text,\n 'timeout': alert.timeout,\n 'rawData': alert.raw_data,\n 'repeat': True,\n 'lastReceiveId': alert.id,\n 'lastReceiveTime': now\n },\n '$addToSet': {'tags': {'$each': alert.tags}},\n '$inc': {'duplicateCount': 1}\n }\n\n # only update those attributes that are specifically defined\n attributes = {'attributes.' + k: v for k, v in alert.attributes.items()}\n update['$set'].update(attributes)\n\n if alert.update_time:\n update['$set']['updateTime'] = alert.update_time\n\n if history:\n update['$push'] = {\n 'history': {\n '$each': [history.serialize],\n '$slice': -abs(current_app.config['HISTORY_LIMIT'])\n }\n }\n\n return self.get_db().alerts.find_one_and_update(\n query,\n update=update,\n return_document=ReturnDocument.AFTER\n )"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nupdating alert key attributes, reset duplicate count and set repeat=False, keep track of last receive id and time, appending all to history. Append to history again if status changes.", "response": "def correlate_alert(self, alert, history):\n \"\"\"\n Update alert key attributes, reset duplicate count and set repeat=False, keep track of last\n receive id and time, appending all to history. Append to history again if status changes.\n \"\"\"\n query = {\n 'environment': alert.environment,\n 'resource': alert.resource,\n '$or': [\n {\n 'event': alert.event,\n 'severity': {'$ne': alert.severity}\n },\n {\n 'event': {'$ne': alert.event},\n 'correlate': alert.event\n }],\n 'customer': alert.customer\n }\n\n update = {\n '$set': {\n 'event': alert.event,\n 'severity': alert.severity,\n 'status': alert.status,\n 'service': alert.service,\n 'value': alert.value,\n 'text': alert.text,\n 'createTime': alert.create_time,\n 'timeout': alert.timeout,\n 'rawData': alert.raw_data,\n 'duplicateCount': alert.duplicate_count,\n 'repeat': alert.repeat,\n 'previousSeverity': alert.previous_severity,\n 'trendIndication': alert.trend_indication,\n 'receiveTime': alert.receive_time,\n 'lastReceiveId': alert.last_receive_id,\n 'lastReceiveTime': alert.last_receive_time\n },\n '$addToSet': {'tags': {'$each': alert.tags}},\n '$push': {\n 'history': {\n '$each': [h.serialize for h in history],\n '$slice': -abs(current_app.config['HISTORY_LIMIT'])\n }\n }\n }\n\n # only update those attributes that are specifically defined\n attributes = {'attributes.' + k: v for k, v in alert.attributes.items()}\n update['$set'].update(attributes)\n\n if alert.update_time:\n update['$set']['updateTime'] = alert.update_time\n\n return self.get_db().alerts.find_one_and_update(\n query,\n update=update,\n return_document=ReturnDocument.AFTER\n )"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef set_status(self, id, status, timeout, update_time, history=None):\n query = {'_id': {'$regex': '^' + id}}\n\n update = {\n '$set': {'status': status, 'timeout': timeout, 'updateTime': update_time},\n '$push': {\n 'history': {\n '$each': [history.serialize],\n '$slice': -abs(current_app.config['HISTORY_LIMIT'])\n }\n }\n }\n return self.get_db().alerts.find_one_and_update(\n query,\n update=update,\n projection={'history': 0},\n return_document=ReturnDocument.AFTER\n )", "response": "Set status and update history."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef tag_alert(self, id, tags):\n response = self.get_db().alerts.update_one(\n {'_id': {'$regex': '^' + id}}, {'$addToSet': {'tags': {'$each': tags}}})\n return response.matched_count > 0", "response": "Tag an alert with a list of tags."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef update_attributes(self, id, old_attrs, new_attrs):\n update = dict()\n set_value = {'attributes.' + k: v for k, v in new_attrs.items() if v is not None}\n if set_value:\n update['$set'] = set_value\n unset_value = {'attributes.' + k: v for k, v in new_attrs.items() if v is None}\n if unset_value:\n update['$unset'] = unset_value\n\n if update:\n response = self.get_db().alerts.update_one({'_id': {'$regex': '^' + id}}, update=update)\n return response.matched_count > 0", "response": "Update all attributes and unset attributes by using a value of null."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_count(self, query=None):\n query = query or Query()\n return self.get_db().alerts.find(query.where).count()", "response": "Get the number of alerts that meet the query filter."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef update_user_attributes(self, id, old_attrs, new_attrs):\n from alerta.utils.collections import merge\n merge(old_attrs, new_attrs)\n attrs = {k: v for k, v in old_attrs.items() if v is not None}\n update = {\n '$set': {'attributes': attrs}\n }\n response = self.get_db().users.update_one({'_id': {'$regex': '^' + id}}, update=update)\n return response.matched_count > 0", "response": "Update user attributes by using a value of null."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef is_blackout(self) -> bool:\n if not current_app.config['NOTIFICATION_BLACKOUT']:\n if self.severity in current_app.config['BLACKOUT_ACCEPT']:\n return False\n return db.is_blackout_period(self)", "response": "Does this alert match a blackout period?"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef take_action(self, alert, action, text, **kwargs):\n\n BASE_URL = '{}/projects/{}'.format(GITLAB_URL, quote(GITLAB_PROJECT_ID, safe=''))\n\n if action == 'createIssue':\n if 'issue_iid' not in alert.attributes:\n url = BASE_URL + '/issues?title=' + alert.text\n r = requests.post(url, headers=self.headers)\n\n alert.attributes['issue_iid'] = r.json().get('iid', None)\n alert.attributes['gitlabUrl'] = 'Issue #{}'.format(\n r.json().get('web_url', None),\n r.json().get('iid', None)\n )\n\n elif action == 'updateIssue':\n if 'issue_iid' in alert.attributes:\n issue_iid = alert.attributes['issue_iid']\n body = 'Update: ' + alert.text\n url = BASE_URL + '/issues/{}/discussions?body={}'.format(issue_iid, body)\n r = requests.post(url, headers=self.headers)\n\n return alert, action, text", "response": "take action from external system"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nwraps JSONified output for JSONP requests.", "response": "def jsonp(func):\n \"\"\"Wraps JSONified output for JSONP requests.\"\"\"\n @wraps(func)\n def decorated(*args, **kwargs):\n callback = request.args.get('callback', False)\n if callback:\n data = str(func(*args, **kwargs).data)\n content = str(callback) + '(' + data + ')'\n mimetype = 'application/javascript'\n return current_app.response_class(content, mimetype=mimetype)\n else:\n return func(*args, **kwargs)\n return decorated"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef is_flapping(self, alert, window=1800, count=2):\n select = \"\"\"\n SELECT COUNT(*)\n FROM alerts, unnest(history) h\n WHERE environment=%(environment)s\n AND resource=%(resource)s\n AND h.event=%(event)s\n AND h.update_time > (NOW() at time zone 'utc' - INTERVAL '{window} seconds')\n AND h.type='severity'\n AND {customer}\n \"\"\".format(window=window, customer='customer=%(customer)s' if alert.customer else 'customer IS NULL')\n return self._fetchone(select, vars(alert)).count > count", "response": "Return true if the alert has changed more than X times in Y seconds in X seconds in Y seconds in X seconds in Y seconds."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef dedup_alert(self, alert, history):\n alert.history = history\n update = \"\"\"\n UPDATE alerts\n SET status=%(status)s, service=%(service)s, value=%(value)s, text=%(text)s,\n timeout=%(timeout)s, raw_data=%(raw_data)s, repeat=%(repeat)s,\n last_receive_id=%(last_receive_id)s, last_receive_time=%(last_receive_time)s,\n tags=ARRAY(SELECT DISTINCT UNNEST(tags || %(tags)s)), attributes=attributes || %(attributes)s,\n duplicate_count=duplicate_count + 1, {update_time}, history=(%(history)s || history)[1:{limit}]\n WHERE environment=%(environment)s\n AND resource=%(resource)s\n AND event=%(event)s\n AND severity=%(severity)s\n AND {customer}\n RETURNING *\n \"\"\".format(\n limit=current_app.config['HISTORY_LIMIT'],\n update_time='update_time=%(update_time)s' if alert.update_time else 'update_time=update_time',\n customer='customer=%(customer)s' if alert.customer else 'customer IS NULL'\n )\n return self._updateone(update, vars(alert), returning=True)", "response": "Update alert status, service, value, text, timeout and rawData, increment duplicate count and set\n repeat=True, and keep track of last receive id and time but don't append to history unless status changes."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _fetchone(self, query, vars):\n cursor = self.get_db().cursor()\n self._log(cursor, query, vars)\n cursor.execute(query, vars)\n return cursor.fetchone()", "response": "Internal method to fetch one row from the database."} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nfetches all entries from the database.", "response": "def _fetchall(self, query, vars, limit=None, offset=0):\n \"\"\"\n Return multiple rows.\n \"\"\"\n if limit is None:\n limit = current_app.config['DEFAULT_PAGE_SIZE']\n query += ' LIMIT %s OFFSET %s''' % (limit, offset)\n cursor = self.get_db().cursor()\n self._log(cursor, query, vars)\n cursor.execute(query, vars)\n return cursor.fetchall()"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nupdates the record set with the given query and return the first result.", "response": "def _updateone(self, query, vars, returning=False):\n \"\"\"\n Update, with optional return.\n \"\"\"\n cursor = self.get_db().cursor()\n self._log(cursor, query, vars)\n cursor.execute(query, vars)\n self.get_db().commit()\n return cursor.fetchone() if returning else None"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _updateall(self, query, vars, returning=False):\n cursor = self.get_db().cursor()\n self._log(cursor, query, vars)\n cursor.execute(query, vars)\n self.get_db().commit()\n return cursor.fetchall() if returning else None", "response": "Update all the keys in the database."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef is_in_scope(cls, want_scope: str, have_scopes: List[Scope]) -> bool:\n if not want_scope:\n return True\n if want_scope in have_scopes or want_scope.split(':')[0] in have_scopes:\n return True\n elif want_scope.startswith('read'):\n return cls.is_in_scope(want_scope.replace('read', 'write'), have_scopes)\n elif want_scope.startswith('write'):\n return cls.is_in_scope(want_scope.replace('write', 'admin'), have_scopes)\n else:\n return False", "response": "Return True if want_scope is in list of scopes or derived scopes."} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nmerge two dictionaries. :param dict1: :param dict2: :return:", "response": "def merge(dict1, dict2):\n \"\"\"\n Merge two dictionaries.\n :param dict1:\n :param dict2:\n :return:\n \"\"\"\n for k in dict2:\n if k in dict1 and isinstance(dict1[k], dict) and isinstance(dict2[k], dict):\n merge(dict1[k], dict2[k])\n else:\n dict1[k] = dict2[k]"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns a Scope object based on the supplied action and resource.", "response": "def from_str(action: str, resource: str=None):\n \"\"\"Return a scope based on the supplied action and resource.\n\n :param action: the scope action eg. read, write or admin\n :param resource: the specific resource of the scope, if any eg. alerts,\n blackouts, heartbeats, users, perms, customers, keys, webhooks,\n oembed, management or userinfo or None\n :return: Scope\n \"\"\"\n if resource:\n return Scope('{}:{}'.format(action, resource))\n else:\n return Scope(action)"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ngiving a code block and a source tree return a new code tree.", "response": "def code_deparse(co, out=sys.stdout, version=None, debug_opts=DEFAULT_DEBUG_OPTS,\n code_objects={}, compile_mode='exec', is_pypy=IS_PYPY, walker=SourceWalker):\n \"\"\"\n ingests and deparses a given code block 'co'. If version is None,\n we will use the current Python interpreter version.\n \"\"\"\n\n assert iscode(co)\n\n if version is None:\n version = float(sys.version[0:3])\n\n # store final output stream for case of error\n scanner = get_scanner(version, is_pypy=is_pypy)\n\n tokens, customize = scanner.ingest(co, code_objects=code_objects,\n show_asm=debug_opts['asm'])\n\n debug_parser = dict(PARSER_DEFAULT_DEBUG)\n if debug_opts.get('grammar', None):\n debug_parser['reduce'] = debug_opts['grammar']\n debug_parser['errorstack'] = 'full'\n\n # Build Syntax Tree from disassembly.\n linestarts = dict(scanner.opc.findlinestarts(co))\n deparsed = walker(version, out, scanner, showast=debug_opts.get('ast', None),\n debug_parser=debug_parser, compile_mode=compile_mode,\n is_pypy=is_pypy, linestarts=linestarts)\n\n isTopLevel = co.co_name == ''\n deparsed.ast = deparsed.build_ast(tokens, customize, isTopLevel=isTopLevel)\n\n #### XXX workaround for profiling\n if deparsed.ast is None:\n return None\n\n assert deparsed.ast == 'stmts', 'Should have parsed grammar start'\n\n # save memory\n del tokens\n\n deparsed.mod_globs, nonlocals = find_globals_and_nonlocals(deparsed.ast,\n set(), set(),\n co, version)\n\n assert not nonlocals\n\n # convert leading '__doc__ = \"...\" into doc string\n try:\n if deparsed.ast[0][0] == ASSIGN_DOC_STRING(co.co_consts[0]):\n print_docstring(deparsed, '', co.co_consts[0])\n del deparsed.ast[0]\n if deparsed.ast[-1] == RETURN_NONE:\n deparsed.ast.pop() # remove last node\n # todo: if empty, add 'pass'\n except:\n pass\n\n deparsed.FUTURE_UNICODE_LITERALS = (\n COMPILER_FLAG_BIT['FUTURE_UNICODE_LITERALS'] & co.co_flags != 0)\n\n # What we've been waiting for: Generate source from Syntax Tree!\n deparsed.gen_source(deparsed.ast, co.co_name, customize)\n\n for g in sorted(deparsed.mod_globs):\n deparsed.write('# global %s ## Warning: Unused global\\n' % g)\n\n if deparsed.ast_errors:\n deparsed.write(\"# NOTE: have internal decompilation grammar errors.\\n\")\n deparsed.write(\"# Use -t option to show full context.\")\n for err in deparsed.ast_errors:\n deparsed.write(err)\n raise SourceWalkerError(\"Deparsing hit an internal grammar-rule bug\")\n\n if deparsed.ERROR:\n raise SourceWalkerError(\"Deparsing stopped due to parse error\")\n return deparsed"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef deparse_code2str(code, out=sys.stdout, version=None,\n debug_opts=DEFAULT_DEBUG_OPTS,\n code_objects={}, compile_mode='exec',\n is_pypy=IS_PYPY, walker=SourceWalker):\n \"\"\"Return the deparsed text for a Python code object. `out` is where any intermediate\n output for assembly or tree output will be sent.\n \"\"\"\n return deparse_code(version, code, out, showasm=debug_opts.get('asm', None),\n showast=debug_opts.get('tree', None),\n showgrammar=debug_opts.get('grammar', None), code_objects=code_objects,\n compile_mode=compile_mode, is_pypy=is_pypy, walker=walker).text", "response": "Return the deparsed text for a Python code object."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef pp_tuple(self, tup):\n last_line = self.f.getvalue().split(\"\\n\")[-1]\n l = len(last_line)+1\n indent = ' ' * l\n self.write('(')\n sep = ''\n for item in tup:\n self.write(sep)\n l += len(sep)\n s = repr(item)\n l += len(s)\n self.write(s)\n sep = ','\n if l > LINE_LENGTH:\n l = 0\n sep += '\\n' + indent\n else:\n sep += ' '\n pass\n pass\n if len(tup) == 1:\n self.write(\", \")\n self.write(')')", "response": "Pretty print a tuple"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef n_exec_stmt(self, node):\n self.write(self.indent, 'exec ')\n self.preorder(node[0])\n if not node[1][0].isNone():\n sep = ' in '\n for subnode in node[1]:\n self.write(sep); sep = \", \"\n self.preorder(subnode)\n self.println()\n self.prune()", "response": "n_exec_stmt is the entry point for the EXEC_STMT node"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nlisting comprehensions in PyPY.", "response": "def n_list_comp_pypy27(self, node):\n \"\"\"List comprehensions in PYPY.\"\"\"\n p = self.prec\n self.prec = 27\n if node[-1].kind == 'list_iter':\n n = node[-1]\n elif self.is_pypy and node[-1] == 'JUMP_BACK':\n n = node[-2]\n list_expr = node[1]\n\n if len(node) >= 3:\n store = node[3]\n elif self.is_pypy and n[0] == 'list_for':\n store = n[0][2]\n\n assert n == 'list_iter'\n assert store == 'store'\n\n # Find the list comprehension body. It is the inner-most\n # node.\n # FIXME: DRY with other use\n while n == 'list_iter':\n n = n[0] # iterate one nesting deeper\n if n == 'list_for':\tn = n[3]\n elif n == 'list_if':\tn = n[2]\n elif n == 'list_if_not': n = n[2]\n assert n == 'lc_body'\n self.write( '[ ')\n\n expr = n[0]\n if self.is_pypy and node[-1] == 'JUMP_BACK':\n list_iter = node[-2]\n else:\n list_iter = node[-1]\n\n assert expr == 'expr'\n assert list_iter == 'list_iter'\n\n # FIXME: use source line numbers for directing line breaks\n\n self.preorder(expr)\n self.preorder(list_expr)\n self.write( ' ]')\n self.prec = p\n self.prune()"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef comprehension_walk_newer(self, node, iter_index, code_index=-5):\n p = self.prec\n self.prec = 27\n code = node[code_index].attr\n\n assert iscode(code), node[code_index]\n code = Code(code, self.scanner, self.currentclass)\n\n ast = self.build_ast(code._tokens, code._customize)\n self.customize(code._customize)\n\n # skip over: sstmt, stmt, return, ret_expr\n # and other singleton derivations\n while (len(ast) == 1\n or (ast in ('sstmt', 'return')\n and ast[-1] in ('RETURN_LAST', 'RETURN_VALUE'))):\n self.prec = 100\n ast = ast[0]\n\n # Pick out important parts of the comprehension:\n # * the variable we interate over: \"store\"\n # * the results we accumulate: \"n\"\n\n is_30_dict_comp = False\n store = None\n n = ast[iter_index]\n if ast in ('set_comp_func', 'dict_comp_func',\n 'list_comp', 'set_comp_func_header'):\n for k in ast:\n if k == 'comp_iter':\n n = k\n elif k == 'store':\n store = k\n pass\n pass\n pass\n elif ast in ('dict_comp', 'set_comp'):\n assert self.version == 3.0\n for k in ast:\n if k in ('dict_comp_header', 'set_comp_header'):\n n = k\n elif k == 'store':\n store = k\n elif k == 'dict_comp_iter':\n is_30_dict_comp = True\n n = (k[3], k[1])\n pass\n elif k == 'comp_iter':\n n = k[0]\n pass\n pass\n else:\n assert n == 'list_iter', n\n\n # FIXME: I'm not totally sure this is right.\n\n # Find the list comprehension body. It is the inner-most\n # node that is not list_.. .\n if_node = None\n comp_for = None\n comp_store = None\n if n == 'comp_iter':\n comp_for = n\n comp_store = ast[3]\n\n have_not = False\n while n in ('list_iter', 'comp_iter'):\n # iterate one nesting deeper\n if self.version == 3.0 and len(n) == 3:\n assert n[0] == 'expr' and n[1] == 'expr'\n n = n[1]\n else:\n n = n[0]\n\n if n in ('list_for', 'comp_for'):\n if n[2] == 'store':\n store = n[2]\n n = n[3]\n elif n in ('list_if', 'list_if_not', 'comp_if', 'comp_if_not'):\n have_not = n in ('list_if_not', 'comp_if_not')\n if_node = n[0]\n if n[1] == 'store':\n store = n[1]\n n = n[2]\n pass\n pass\n\n # Python 2.7+ starts including set_comp_body\n # Python 3.5+ starts including set_comp_func\n # Python 3.0 is yet another snowflake\n if self.version != 3.0:\n assert n.kind in ('lc_body', 'comp_body', 'set_comp_func', 'set_comp_body'), ast\n assert store, \"Couldn't find store in list/set comprehension\"\n\n # A problem created with later Python code generation is that there\n # is a lamda set up with a dummy argument name that is then called\n # So we can't just translate that as is but need to replace the\n # dummy name. Below we are picking out the variable name as seen\n # in the code. And trying to generate code for the other parts\n # that don't have the dummy argument name in it.\n # Another approach might be to be able to pass in the source name\n # for the dummy argument.\n\n if is_30_dict_comp:\n self.preorder(n[0])\n self.write(': ')\n self.preorder(n[1])\n else:\n self.preorder(n[0])\n self.write(' for ')\n if comp_store:\n self.preorder(comp_store)\n else:\n self.preorder(store)\n\n # FIXME this is all merely approximate\n self.write(' in ')\n self.preorder(node[-3])\n\n if ast == 'list_comp' and self.version != 3.0:\n list_iter = ast[1]\n assert list_iter == 'list_iter'\n if list_iter == 'list_for':\n self.preorder(list_iter[3])\n self.prec = p\n return\n pass\n\n if comp_store:\n self.preorder(comp_for)\n elif if_node:\n self.write(' if ')\n if have_not:\n self.write('not ')\n self.preorder(if_node)\n pass\n self.prec = p", "response": "Walk the tree for non - closure - based comprehensions."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef listcomprehension_walk2(self, node):\n p = self.prec\n self.prec = 27\n\n code = Code(node[1].attr, self.scanner, self.currentclass)\n ast = self.build_ast(code._tokens, code._customize)\n self.customize(code._customize)\n\n # skip over: sstmt, stmt, return, ret_expr\n # and other singleton derivations\n while (len(ast) == 1\n or (ast in ('sstmt', 'return')\n and ast[-1] in ('RETURN_LAST', 'RETURN_VALUE'))):\n self.prec = 100\n ast = ast[0]\n\n n = ast[1]\n # collection = node[-3]\n collections = [node[-3]]\n list_ifs = []\n\n if self.version == 3.0 and n != 'list_iter':\n # FIXME 3.0 is a snowflake here. We need\n # special code for this. Not sure if this is totally\n # correct.\n stores = [ast[3]]\n assert ast[4] == 'comp_iter'\n n = ast[4]\n # Find the list comprehension body. It is the inner-most\n # node that is not comp_.. .\n while n == 'comp_iter':\n if n[0] == 'comp_for':\n n = n[0]\n stores.append(n[2])\n n = n[3]\n elif n[0] in ('comp_if', 'comp_if_not'):\n n = n[0]\n # FIXME: just a guess\n if n[0].kind == 'expr':\n list_ifs.append(n)\n else:\n list_ifs.append([1])\n n = n[2]\n pass\n else:\n break\n pass\n\n # Skip over n[0] which is something like: _[1]\n self.preorder(n[1])\n\n else:\n assert n == 'list_iter'\n stores = []\n # Find the list comprehension body. It is the inner-most\n # node that is not list_.. .\n while n == 'list_iter':\n n = n[0] # recurse one step\n if n == 'list_for':\n stores.append(n[2])\n n = n[3]\n if self.version >= 3.6 and n[0] == 'list_for':\n # Dog-paddle down largely singleton reductions\n # to find the collection (expr)\n c = n[0][0]\n if c == 'expr':\n c = c[0]\n # FIXME: grammar is wonky here? Is this really an attribute?\n if c == 'attribute':\n c = c[0]\n collections.append(c)\n pass\n elif n in ('list_if', 'list_if_not'):\n # FIXME: just a guess\n if n[0].kind == 'expr':\n list_ifs.append(n)\n else:\n list_ifs.append([1])\n n = n[2]\n pass\n pass\n\n assert n == 'lc_body', ast\n self.preorder(n[0])\n\n # FIXME: add indentation around \"for\"'s and \"in\"'s\n if self.version < 3.6:\n self.write(' for ')\n self.preorder(stores[0])\n self.write(' in ')\n self.preorder(collections[0])\n if list_ifs:\n self.preorder(list_ifs[0])\n pass\n else:\n for i, store in enumerate(stores):\n self.write(' for ')\n self.preorder(store)\n self.write(' in ')\n self.preorder(collections[i])\n if i < len(list_ifs):\n self.preorder(list_ifs[i])\n pass\n pass\n self.prec = p", "response": "Walk the tree for list comprehensions."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nsetting comprehensions in Python 3.", "response": "def setcomprehension_walk3(self, node, collection_index):\n \"\"\"Set comprehensions the way they are done in Python3.\n They're more other comprehensions, e.g. set comprehensions\n See if we can combine code.\n \"\"\"\n p = self.prec\n self.prec = 27\n\n code = Code(node[1].attr, self.scanner, self.currentclass)\n ast = self.build_ast(code._tokens, code._customize)\n self.customize(code._customize)\n ast = ast[0][0][0]\n store = ast[3]\n collection = node[collection_index]\n\n n = ast[4]\n list_if = None\n assert n == 'comp_iter'\n\n # find innermost node\n while n == 'comp_iter':\n n = n[0] # recurse one step\n # FIXME: adjust for set comprehension\n if n == 'list_for':\n store = n[2]\n n = n[3]\n elif n in ('list_if', 'list_if_not', 'comp_if', 'comp_if_not'):\n # FIXME: just a guess\n if n[0].kind == 'expr':\n list_if = n\n else:\n list_if = n[1]\n n = n[2]\n pass\n pass\n\n assert n == 'comp_body', ast\n\n self.preorder(n[0])\n self.write(' for ')\n self.preorder(store)\n self.write(' in ')\n self.preorder(collection)\n if list_if:\n self.preorder(list_if)\n self.prec = p"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef n_list(self, node):\n p = self.prec\n self.prec = 100\n lastnode = node.pop()\n lastnodetype = lastnode.kind\n\n # If this build list is inside a CALL_FUNCTION_VAR,\n # then the first * has already been printed.\n # Until I have a better way to check for CALL_FUNCTION_VAR,\n # will assume that if the text ends in *.\n last_was_star = self.f.getvalue().endswith('*')\n\n if lastnodetype.endswith('UNPACK'):\n # FIXME: need to handle range of BUILD_LIST_UNPACK\n have_star = True\n # endchar = ''\n else:\n have_star = False\n\n if lastnodetype.startswith('BUILD_LIST'):\n self.write('['); endchar = ']'\n elif lastnodetype.startswith('BUILD_TUPLE'):\n # Tuples can appear places that can NOT\n # have parenthesis around them, like array\n # subscripts. We check for that by seeing\n # if a tuple item is some sort of slice.\n no_parens = False\n for n in node:\n if n == 'expr' and n[0].kind.startswith('build_slice'):\n no_parens = True\n break\n pass\n if no_parens:\n endchar = ''\n else:\n self.write('('); endchar = ')'\n pass\n\n elif lastnodetype.startswith('BUILD_SET'):\n self.write('{'); endchar = '}'\n elif lastnodetype.startswith('BUILD_MAP_UNPACK'):\n self.write('{*'); endchar = '}'\n elif lastnodetype.startswith('ROT_TWO'):\n self.write('('); endchar = ')'\n else:\n raise TypeError('Internal Error: n_build_list expects list, tuple, set, or unpack')\n\n flat_elems = flatten_list(node)\n\n self.indent_more(INDENT_PER_LEVEL)\n sep = ''\n for elem in flat_elems:\n if elem in ('ROT_THREE', 'EXTENDED_ARG'):\n continue\n assert elem == 'expr'\n line_number = self.line_number\n value = self.traverse(elem)\n if line_number != self.line_number:\n sep += '\\n' + self.indent + INDENT_PER_LEVEL[:-1]\n else:\n if sep != '': sep += ' '\n if not last_was_star:\n if have_star:\n sep += '*'\n pass\n pass\n else:\n last_was_star = False\n self.write(sep, value)\n sep = ','\n if lastnode.attr == 1 and lastnodetype.startswith('BUILD_TUPLE'):\n self.write(',')\n self.write(endchar)\n self.indent_less(INDENT_PER_LEVEL)\n\n self.prec = p\n self.prune()\n return", "response": "prettyprint a list or tuple"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_tuple_parameter(self, ast, name):\n\n assert ast == 'stmts'\n for i in range(len(ast)):\n # search for an assign-statement\n assert ast[i][0] == 'stmt'\n node = ast[i][0][0]\n if (node == 'assign'\n and node[0] == ASSIGN_TUPLE_PARAM(name)):\n # okay, this assigns '.n' to something\n del ast[i]\n # walk lhs; this\n # returns a tuple of identifiers as used\n # within the function definition\n assert node[1] == 'store'\n # if lhs is not a UNPACK_TUPLE (or equiv.),\n # add parenteses to make this a tuple\n # if node[1][0] not in ('unpack', 'unpack_list'):\n return '(' + self.traverse(node[1]) + ')'\n # return self.traverse(node[1])\n raise Exception(\"Can't find tuple parameter \" + name)", "response": "Given a statement and a parameter name return the name of the parameter."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ndumping class definition doc string and class body.", "response": "def build_class(self, code):\n \"\"\"Dump class definition, doc string and class body.\"\"\"\n\n assert iscode(code)\n self.classes.append(self.currentclass)\n code = Code(code, self.scanner, self.currentclass)\n\n indent = self.indent\n # self.println(indent, '#flags:\\t', int(code.co_flags))\n ast = self.build_ast(code._tokens, code._customize)\n code._tokens = None # save memory\n assert ast == 'stmts'\n\n first_stmt = ast[0][0]\n if 3.0 <= self.version <= 3.3:\n try:\n if first_stmt[0] == 'store_locals':\n if self.hide_internal:\n del ast[0]\n first_stmt = ast[0][0]\n except:\n pass\n\n try:\n if first_stmt == NAME_MODULE:\n if self.hide_internal:\n del ast[0]\n first_stmt = ast[0][0]\n pass\n except:\n pass\n\n have_qualname = False\n if self.version < 3.0:\n # Should we ditch this in favor of the \"else\" case?\n qualname = '.'.join(self.classes)\n QUAL_NAME = SyntaxTree('stmt',\n [ SyntaxTree('assign',\n [ SyntaxTree('expr', [Token('LOAD_CONST', pattr=qualname)]),\n SyntaxTree('store', [ Token('STORE_NAME', pattr='__qualname__')])\n ])])\n have_qualname = (ast[0][0] == QUAL_NAME)\n else:\n # Python 3.4+ has constants like 'cmp_to_key..K'\n # which are not simple classes like the < 3 case.\n try:\n if (first_stmt[0] == 'assign' and\n first_stmt[0][0][0] == 'LOAD_CONST' and\n first_stmt[0][1] == 'store' and\n first_stmt[0][1][0] == Token('STORE_NAME', pattr='__qualname__')):\n have_qualname = True\n except:\n pass\n\n if have_qualname:\n if self.hide_internal: del ast[0]\n pass\n\n # if docstring exists, dump it\n if (code.co_consts and code.co_consts[0] is not None and len(ast) > 0):\n do_doc = False\n if is_docstring(ast[0]):\n i = 0\n do_doc = True\n elif (len(ast) > 1 and is_docstring(ast[1])):\n i = 1\n do_doc = True\n if do_doc and self.hide_internal:\n try:\n docstring = ast[i][0][0][0][0].pattr\n except:\n docstring = code.co_consts[0]\n if print_docstring(self, indent, docstring):\n self.println()\n del ast[i]\n\n # the function defining a class normally returns locals(); we\n # don't want this to show up in the source, thus remove the node\n if len(ast) > 0 and ast[-1][0] == RETURN_LOCALS:\n if self.hide_internal: del ast[-1] # remove last node\n # else:\n # print ast[-1][-1]\n\n globals, nonlocals = find_globals_and_nonlocals(ast, set(), set(),\n code, self.version)\n # Add \"global\" declaration statements at the top\n # of the function\n for g in sorted(globals):\n self.println(indent, 'global ', g)\n\n for nl in sorted(nonlocals):\n self.println(indent, 'nonlocal ', nl)\n\n old_name = self.name\n self.gen_source(ast, code.co_name, code._customize)\n self.name = old_name\n code._tokens = None; code._customize = None # save memory\n self.classes.pop(-1)"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nconverting SyntaxTree to Python source code", "response": "def gen_source(self, ast, name, customize, is_lambda=False, returnNone=False):\n \"\"\"convert SyntaxTree to Python source code\"\"\"\n\n rn = self.return_none\n self.return_none = returnNone\n old_name = self.name\n self.name = name\n # if code would be empty, append 'pass'\n if len(ast) == 0:\n self.println(self.indent, 'pass')\n else:\n self.customize(customize)\n if is_lambda:\n self.write(self.traverse(ast, is_lambda=is_lambda))\n else:\n self.text = self.traverse(ast, is_lambda=is_lambda)\n self.println(self.text)\n self.name = old_name\n self.return_none = rn"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef disco(version, co, out=None, is_pypy=False):\n\n assert iscode(co)\n\n # store final output stream for case of error\n real_out = out or sys.stdout\n print('# Python %s' % version, file=real_out)\n if co.co_filename:\n print('# Embedded file name: %s' % co.co_filename,\n file=real_out)\n\n scanner = get_scanner(version, is_pypy=is_pypy)\n\n queue = deque([co])\n disco_loop(scanner.ingest, queue, real_out)", "response": "Disparses a given code block."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef disassemble_file(filename, outstream=None):\n filename = check_object_path(filename)\n (version, timestamp, magic_int, co, is_pypy,\n source_size) = load_module(filename)\n if type(co) == list:\n for con in co:\n disco(version, con, outstream)\n else:\n disco(version, co, outstream, is_pypy=is_pypy)\n co = None", "response": "Disassemble a Python byte - code file into a single object."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncustomizing the grammar rules for a specific Python version.", "response": "def customize_grammar_rules(self, tokens, customize):\n \"\"\"The base grammar we start out for a Python version even with the\n subclassing is, well, is pretty base. And we want it that way: lean and\n mean so that parsing will go faster.\n\n Here, we add additional grammar rules based on specific instructions\n that are in the instruction/token stream. In classes that\n inherit from from here and other versions, grammar rules may\n also be removed.\n\n For example if we see a pretty rare JUMP_IF_NOT_DEBUG\n instruction we'll add the grammar for that.\n\n More importantly, here we add grammar rules for instructions\n that may access a variable number of stack items. CALL_FUNCTION,\n BUILD_LIST and so on are like this.\n\n Without custom rules, there can be an super-exponential number of\n derivations. See the deparsing paper for an elaboration of\n this.\n \"\"\"\n\n if 'PyPy' in customize:\n # PyPy-specific customizations\n self.addRule(\"\"\"\n stmt ::= assign3_pypy\n stmt ::= assign2_pypy\n assign3_pypy ::= expr expr expr store store store\n assign2_pypy ::= expr expr store store\n list_comp ::= expr BUILD_LIST_FROM_ARG for_iter store list_iter\n JUMP_BACK\n \"\"\", nop_func)\n\n # For a rough break out on the first word. This may\n # include instructions that don't need customization,\n # but we'll do a finer check after the rough breakout.\n customize_instruction_basenames = frozenset(\n ('BUILD', 'CALL', 'CONTINUE', 'DELETE',\n 'DUP', 'EXEC', 'GET', 'JUMP',\n 'LOAD', 'LOOKUP', 'MAKE', 'SETUP',\n 'RAISE', 'UNPACK'))\n\n # Opcode names in the custom_seen_ops set have rules that get added\n # unconditionally and the rules are constant. So they need to be done\n # only once and if we see the opcode a second we don't have to consider\n # adding more rules.\n #\n custom_seen_ops = set()\n\n for i, token in enumerate(tokens):\n opname = token.kind\n\n # Do a quick breakout before testing potentially\n # each of the dozen or so instruction in if elif.\n if (opname[:opname.find('_')] not in customize_instruction_basenames\n or opname in custom_seen_ops):\n continue\n\n opname_base = opname[:opname.rfind('_')]\n\n # The order of opname listed is roughly sorted below\n if opname_base in ('BUILD_LIST', 'BUILD_SET', 'BUILD_TUPLE'):\n # We do this complicated test to speed up parsing of\n # pathelogically long literals, especially those over 1024.\n build_count = token.attr\n thousands = (build_count//1024)\n thirty32s = ((build_count//32) % 32)\n if thirty32s > 0:\n rule = \"expr32 ::=%s\" % (' expr' * 32)\n self.add_unique_rule(rule, opname_base, build_count, customize)\n if thousands > 0:\n self.add_unique_rule(\"expr1024 ::=%s\" % (' expr32' * 32),\n opname_base, build_count, customize)\n collection = opname_base[opname_base.find('_')+1:].lower()\n rule = (('%s ::= ' % collection) + 'expr1024 '*thousands +\n 'expr32 '*thirty32s + 'expr '*(build_count % 32) + opname)\n self.add_unique_rules([\n \"expr ::= %s\" % collection,\n rule], customize)\n continue\n elif opname_base == 'BUILD_MAP':\n if opname == 'BUILD_MAP_n':\n # PyPy sometimes has no count. Sigh.\n self.add_unique_rules([\n 'kvlist_n ::= kvlist_n kv3',\n 'kvlist_n ::=',\n 'dict ::= BUILD_MAP_n kvlist_n',\n ], customize)\n if self.version >= 2.7:\n self.add_unique_rule(\n 'dict_comp_func ::= BUILD_MAP_n LOAD_FAST FOR_ITER store '\n 'comp_iter JUMP_BACK RETURN_VALUE RETURN_LAST',\n 'dict_comp_func', 0, customize)\n\n else:\n kvlist_n = ' kv3' * token.attr\n rule = \"dict ::= %s%s\" % (opname, kvlist_n)\n self.addRule(rule, nop_func)\n continue\n elif opname_base == 'BUILD_SLICE':\n slice_num = token.attr\n if slice_num == 2:\n self.add_unique_rules([\n 'expr ::= build_slice2',\n 'build_slice2 ::= expr expr BUILD_SLICE_2'\n ], customize)\n else:\n assert slice_num == 3, (\"BUILD_SLICE value must be 2 or 3; is %s\" %\n slice_num)\n self.add_unique_rules([\n 'expr ::= build_slice3',\n 'build_slice3 ::= expr expr expr BUILD_SLICE_3',\n ], customize)\n continue\n elif opname_base in ('CALL_FUNCTION', 'CALL_FUNCTION_VAR',\n 'CALL_FUNCTION_VAR_KW', 'CALL_FUNCTION_KW'):\n\n args_pos, args_kw = self.get_pos_kw(token)\n\n # number of apply equiv arguments:\n nak = ( len(opname_base)-len('CALL_FUNCTION') ) // 3\n rule = 'call ::= expr ' + 'expr '*args_pos + 'kwarg '*args_kw \\\n + 'expr ' * nak + opname\n elif opname_base == 'CALL_METHOD':\n # PyPy only - DRY with parse3\n\n args_pos, args_kw = self.get_pos_kw(token)\n\n # number of apply equiv arguments:\n nak = ( len(opname_base)-len('CALL_METHOD') ) // 3\n rule = 'call ::= expr ' + 'expr '*args_pos + 'kwarg '*args_kw \\\n + 'expr ' * nak + opname\n elif opname == 'CONTINUE_LOOP':\n self.addRule('continue ::= CONTINUE_LOOP', nop_func)\n custom_seen_ops.add(opname)\n continue\n elif opname == 'DELETE_ATTR':\n self.addRule('del_stmt ::= expr DELETE_ATTR', nop_func)\n custom_seen_ops.add(opname)\n continue\n elif opname.startswith('DELETE_SLICE'):\n self.addRule(\"\"\"\n del_expr ::= expr\n del_stmt ::= del_expr DELETE_SLICE+0\n del_stmt ::= del_expr del_expr DELETE_SLICE+1\n del_stmt ::= del_expr del_expr DELETE_SLICE+2\n del_stmt ::= del_expr del_expr del_expr DELETE_SLICE+3\n \"\"\", nop_func)\n custom_seen_ops.add(opname)\n self.check_reduce['del_expr'] = 'AST'\n continue\n elif opname == 'DELETE_DEREF':\n self.addRule(\"\"\"\n stmt ::= del_deref_stmt\n del_deref_stmt ::= DELETE_DEREF\n \"\"\", nop_func)\n custom_seen_ops.add(opname)\n continue\n elif opname == 'DELETE_SUBSCR':\n self.addRule(\"\"\"\n del_stmt ::= delete_subscr\n delete_subscr ::= expr expr DELETE_SUBSCR\n \"\"\", nop_func)\n self.check_reduce['delete_subscr'] = 'AST'\n custom_seen_ops.add(opname)\n continue\n elif opname == 'GET_ITER':\n self.addRule(\"\"\"\n expr ::= get_iter\n attribute ::= expr GET_ITER\n \"\"\", nop_func)\n custom_seen_ops.add(opname)\n continue\n elif opname_base in ('DUP_TOPX', 'RAISE_VARARGS'):\n # FIXME: remove these conditions if they are not needed.\n # no longer need to add a rule\n continue\n elif opname == 'EXEC_STMT':\n self.addRule(\"\"\"\n stmt ::= exec_stmt\n exec_stmt ::= expr exprlist DUP_TOP EXEC_STMT\n exec_stmt ::= expr exprlist EXEC_STMT\n exprlist ::= expr+\n \"\"\", nop_func)\n continue\n elif opname == 'JUMP_IF_NOT_DEBUG':\n self.addRule(\"\"\"\n jmp_true_false ::= POP_JUMP_IF_TRUE\n jmp_true_false ::= POP_JUMP_IF_FALSE\n stmt ::= assert_pypy\n stmt ::= assert2_pypy\n assert_pypy ::= JUMP_IF_NOT_DEBUG assert_expr jmp_true_false\n LOAD_ASSERT RAISE_VARARGS_1 COME_FROM\n assert2_pypy ::= JUMP_IF_NOT_DEBUG assert_expr jmp_true_false\n LOAD_ASSERT expr CALL_FUNCTION_1\n RAISE_VARARGS_1 COME_FROM\n \"\"\", nop_func)\n continue\n elif opname == 'LOAD_ATTR':\n self.addRule(\"\"\"\n expr ::= attribute\n attribute ::= expr LOAD_ATTR\n \"\"\", nop_func)\n custom_seen_ops.add(opname)\n continue\n elif opname == 'LOAD_LISTCOMP':\n self.addRule(\"expr ::= listcomp\", nop_func)\n custom_seen_ops.add(opname)\n continue\n elif opname == 'LOAD_SETCOMP':\n self.add_unique_rules([\n \"expr ::= set_comp\",\n \"set_comp ::= LOAD_SETCOMP MAKE_FUNCTION_0 expr GET_ITER CALL_FUNCTION_1\"\n ], customize)\n custom_seen_ops.add(opname)\n continue\n elif opname == 'LOOKUP_METHOD':\n # A PyPy speciality - DRY with parse3\n self.addRule(\"\"\"\n expr ::= attribute\n attribute ::= expr LOOKUP_METHOD\n \"\"\",\n nop_func)\n custom_seen_ops.add(opname)\n continue\n elif opname_base == 'MAKE_FUNCTION':\n if i > 0 and tokens[i-1] == 'LOAD_LAMBDA':\n self.addRule('mklambda ::= %s LOAD_LAMBDA %s' %\n ('pos_arg ' * token.attr, opname), nop_func)\n rule = 'mkfunc ::= %s LOAD_CONST %s' % ('expr ' * token.attr, opname)\n elif opname_base == 'MAKE_CLOSURE':\n # FIXME: use add_unique_rules to tidy this up.\n if i > 0 and tokens[i-1] == 'LOAD_LAMBDA':\n self.addRule('mklambda ::= %s load_closure LOAD_LAMBDA %s' %\n ('expr ' * token.attr, opname), nop_func)\n if i > 0:\n prev_tok = tokens[i-1]\n if prev_tok == 'LOAD_GENEXPR':\n self.add_unique_rules([\n ('generator_exp ::= %s load_closure LOAD_GENEXPR %s expr'\n ' GET_ITER CALL_FUNCTION_1' %\n ('expr ' * token.attr, opname))], customize)\n pass\n self.add_unique_rules([\n ('mkfunc ::= %s load_closure LOAD_CONST %s' %\n ('expr ' * token.attr, opname))], customize)\n\n if self.version >= 2.7:\n if i > 0:\n prev_tok = tokens[i-1]\n if prev_tok == 'LOAD_DICTCOMP':\n self.add_unique_rules([\n ('dict_comp ::= %s load_closure LOAD_DICTCOMP %s expr'\n ' GET_ITER CALL_FUNCTION_1' %\n ('expr ' * token.attr, opname))], customize)\n elif prev_tok == 'LOAD_SETCOMP':\n self.add_unique_rules([\n \"expr ::= set_comp\",\n ('set_comp ::= %s load_closure LOAD_SETCOMP %s expr'\n ' GET_ITER CALL_FUNCTION_1' %\n ('expr ' * token.attr, opname))\n ], customize)\n pass\n pass\n continue\n elif opname == 'SETUP_EXCEPT':\n if 'PyPy' in customize:\n self.add_unique_rules([\n \"stmt ::= try_except_pypy\",\n \"try_except_pypy ::= SETUP_EXCEPT suite_stmts_opt except_handler_pypy\",\n \"except_handler_pypy ::= COME_FROM except_stmts END_FINALLY COME_FROM\"\n ], customize)\n custom_seen_ops.add(opname)\n continue\n elif opname == 'SETUP_FINALLY':\n if 'PyPy' in customize:\n self.addRule(\"\"\"\n stmt ::= tryfinallystmt_pypy\n tryfinallystmt_pypy ::= SETUP_FINALLY suite_stmts_opt COME_FROM_FINALLY\n suite_stmts_opt END_FINALLY\"\"\", nop_func)\n\n custom_seen_ops.add(opname)\n continue\n elif opname_base in ('UNPACK_TUPLE', 'UNPACK_SEQUENCE'):\n custom_seen_ops.add(opname)\n rule = 'unpack ::= ' + opname + ' store' * token.attr\n elif opname_base == 'UNPACK_LIST':\n custom_seen_ops.add(opname)\n rule = 'unpack_list ::= ' + opname + ' store' * token.attr\n else:\n continue\n self.addRule(rule, nop_func)\n pass\n\n self.check_reduce['raise_stmt1'] = 'tokens'\n self.check_reduce['aug_assign2'] = 'AST'\n self.check_reduce['or'] = 'AST'\n # self.check_reduce['_stmts'] = 'AST'\n\n # Dead code testing...\n # self.check_reduce['while1elsestmt'] = 'tokens'\n return"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef code_deparse_align(co, out=sys.stderr, version=None, is_pypy=None,\n debug_opts=DEFAULT_DEBUG_OPTS,\n code_objects={}, compile_mode='exec'):\n \"\"\"\n ingests and deparses a given code block 'co'\n \"\"\"\n\n assert iscode(co)\n\n if version is None:\n version = float(sys.version[0:3])\n if is_pypy is None:\n is_pypy = IS_PYPY\n\n\n # store final output stream for case of error\n scanner = get_scanner(version, is_pypy=is_pypy)\n\n tokens, customize = scanner.ingest(co, code_objects=code_objects)\n show_asm = debug_opts.get('asm', None)\n maybe_show_asm(show_asm, tokens)\n\n debug_parser = dict(PARSER_DEFAULT_DEBUG)\n show_grammar = debug_opts.get('grammar', None)\n show_grammar = debug_opts.get('grammar', None)\n if show_grammar:\n debug_parser['reduce'] = show_grammar\n debug_parser['errorstack'] = True\n\n # Build a parse tree from tokenized and massaged disassembly.\n show_ast = debug_opts.get('ast', None)\n deparsed = AligningWalker(version, scanner, out, showast=show_ast,\n debug_parser=debug_parser, compile_mode=compile_mode,\n is_pypy = is_pypy)\n\n isTopLevel = co.co_name == ''\n deparsed.ast = deparsed.build_ast(tokens, customize, isTopLevel=isTopLevel)\n\n assert deparsed.ast == 'stmts', 'Should have parsed grammar start'\n\n del tokens # save memory\n\n deparsed.mod_globs = find_globals(deparsed.ast, set())\n\n # convert leading '__doc__ = \"...\" into doc string\n try:\n if deparsed.ast[0][0] == ASSIGN_DOC_STRING(co.co_consts[0]):\n deparsed.print_docstring('', co.co_consts[0])\n del deparsed.ast[0]\n if deparsed.ast[-1] == RETURN_NONE:\n deparsed.ast.pop() # remove last node\n # todo: if empty, add 'pass'\n except:\n pass\n\n # What we've been waiting for: Generate Python source from the parse tree!\n deparsed.gen_source(deparsed.ast, co.co_name, customize)\n\n for g in sorted(deparsed.mod_globs):\n deparsed.write('# global %s ## Warning: Unused global\\n' % g)\n\n if deparsed.ERROR:\n raise SourceWalkerError(\"Deparsing stopped due to parse error\")\n return deparsed", "response": "Given a code block and a list of code objects and a list of code objects return a list of code objects."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nshowing the asm based on the showasm flag or file object.", "response": "def maybe_show_asm(showasm, tokens):\n \"\"\"\n Show the asm based on the showasm flag (or file object), writing to the\n appropriate stream depending on the type of the flag.\n\n :param showasm: Flag which determines whether the ingested code is\n written to sys.stdout or not. (It is also to pass a file\n like object, into which the asm will be written).\n :param tokens: The asm tokens to show.\n \"\"\"\n if showasm:\n stream = showasm if hasattr(showasm, 'write') else sys.stdout\n for t in tokens:\n stream.write(str(t))\n stream.write('\\n')"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nshow the parse tree based on the showast flag or file object.", "response": "def maybe_show_tree(walker, ast):\n \"\"\"\n Show the ast based on the showast flag (or file object), writing to the\n appropriate stream depending on the type of the flag.\n\n :param show_tree: Flag which determines whether the parse tree is\n written to sys.stdout or not. (It is also to pass a file\n like object, into which the ast will be written).\n :param ast: The ast to show.\n \"\"\"\n if walker.showast:\n if hasattr(walker.showast, 'write'):\n stream = walker.showast\n else:\n stream = sys.stdout\n if walker.showast == 'Full':\n walker.str_with_template(ast)\n else:\n stream.write(str(ast))\n stream.write('\\n')"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef maybe_show_tree_param_default(show_tree, name, default):\n if show_tree:\n stream = show_tree if hasattr(show_tree, 'write') else sys.stdout\n stream.write('\\n')\n stream.write('--' + name)\n stream.write('\\n')\n stream.write(str(default))\n stream.write('\\n')\n stream.write('--')\n stream.write('\\n')", "response": "Show a function parameter with default for an grammar - tree based on the show_tree flag."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ningesting a code object into a list of uncompyle6 Token s.", "response": "def ingest(self, co, classname=None, code_objects={}, show_asm=None):\n \"\"\"\n Pick out tokens from an uncompyle6 code object, and transform them,\n returning a list of uncompyle6 Token's.\n\n The transformations are made to assist the deparsing grammar.\n \"\"\"\n tokens, customize = scan.Scanner21.ingest(self, co, classname, code_objects, show_asm)\n for t in tokens:\n if t.op == self.opc.UNPACK_LIST:\n t.kind = 'UNPACK_LIST_%d' % t.attr\n pass\n return tokens, customize"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef ingest(self, co, classname=None, code_objects={}, show_asm=None):\n\n if not show_asm:\n show_asm = self.show_asm\n\n bytecode = self.build_instructions(co)\n\n # show_asm = 'both'\n if show_asm in ('both', 'before'):\n for instr in bytecode.get_instructions(co):\n print(instr.disassemble())\n\n # list of tokens/instructions\n tokens = []\n\n # \"customize\" is in the process of going away here\n customize = {}\n\n if self.is_pypy:\n customize['PyPy'] = 0\n\n # Scan for assertions. Later we will\n # turn 'LOAD_GLOBAL' to 'LOAD_ASSERT'.\n # 'LOAD_ASSERT' is used in assert statements.\n self.load_asserts = set()\n\n n = len(self.insts)\n for i, inst in enumerate(self.insts):\n\n # We need to detect the difference between:\n # raise AssertionError\n # and\n # assert ...\n # If we have a JUMP_FORWARD after the\n # RAISE_VARARGS then we have a \"raise\" statement\n # else we have an \"assert\" statement.\n if self.version == 3.0:\n # There is a an implied JUMP_IF_TRUE that we are not testing for (yet?) here\n assert_can_follow = inst.opname == 'POP_TOP' and i+1 < n\n else:\n assert_can_follow = inst.opname == 'POP_JUMP_IF_TRUE' and i+1 < n\n if assert_can_follow:\n next_inst = self.insts[i+1]\n if (next_inst.opname == 'LOAD_GLOBAL' and\n next_inst.argval == 'AssertionError'):\n if (i + 2 < n and self.insts[i+2].opname.startswith('RAISE_VARARGS')):\n self.load_asserts.add(next_inst.offset)\n pass\n pass\n\n # Get jump targets\n # Format: {target offset: [jump offsets]}\n jump_targets = self.find_jump_targets(show_asm)\n # print(\"XXX2\", jump_targets)\n\n last_op_was_break = False\n\n for i, inst in enumerate(self.insts):\n\n argval = inst.argval\n op = inst.opcode\n\n if inst.opname == 'EXTENDED_ARG':\n # FIXME: The EXTENDED_ARG is used to signal annotation\n # parameters\n if (i+1 < n and\n self.insts[i+1].opcode != self.opc.MAKE_FUNCTION):\n continue\n\n if inst.offset in jump_targets:\n jump_idx = 0\n # We want to process COME_FROMs to the same offset to be in *descending*\n # offset order so we have the larger range or biggest instruction interval\n # last. (I think they are sorted in increasing order, but for safety\n # we sort them). That way, specific COME_FROM tags will match up\n # properly. For example, a \"loop\" with an \"if\" nested in it should have the\n # \"loop\" tag last so the grammar rule matches that properly.\n for jump_offset in sorted(jump_targets[inst.offset], reverse=True):\n come_from_name = 'COME_FROM'\n opname = self.opname_for_offset(jump_offset)\n if opname == 'EXTENDED_ARG':\n j = xdis.next_offset(op, self.opc, jump_offset)\n opname = self.opname_for_offset(j)\n\n if opname.startswith('SETUP_'):\n come_from_type = opname[len('SETUP_'):]\n come_from_name = 'COME_FROM_%s' % come_from_type\n pass\n elif inst.offset in self.except_targets:\n come_from_name = 'COME_FROM_EXCEPT_CLAUSE'\n tokens.append(Token(come_from_name,\n jump_offset, repr(jump_offset),\n offset='%s_%s' % (inst.offset, jump_idx),\n has_arg = True, opc=self.opc))\n jump_idx += 1\n pass\n pass\n elif inst.offset in self.else_start:\n end_offset = self.else_start[inst.offset]\n tokens.append(Token('ELSE',\n None, repr(end_offset),\n offset='%s' % (inst.offset),\n has_arg = True, opc=self.opc))\n\n pass\n\n pattr = inst.argrepr\n opname = inst.opname\n\n if op in self.opc.CONST_OPS:\n const = argval\n if iscode(const):\n if const.co_name == '':\n assert opname == 'LOAD_CONST'\n opname = 'LOAD_LAMBDA'\n elif const.co_name == '':\n opname = 'LOAD_GENEXPR'\n elif const.co_name == '':\n opname = 'LOAD_DICTCOMP'\n elif const.co_name == '':\n opname = 'LOAD_SETCOMP'\n elif const.co_name == '':\n opname = 'LOAD_LISTCOMP'\n # verify() uses 'pattr' for comparison, since 'attr'\n # now holds Code(const) and thus can not be used\n # for comparison (todo: think about changing this)\n # pattr = 'code_object @ 0x%x %s->%s' %\\\n # (id(const), const.co_filename, const.co_name)\n pattr = ''\n else:\n if isinstance(inst.arg, int) and inst.arg < len(co.co_consts):\n argval, _ = _get_const_info(inst.arg, co.co_consts)\n # Why don't we use _ above for \"pattr\" rather than \"const\"?\n # This *is* a little hoaky, but we have to coordinate with\n # other parts like n_LOAD_CONST in pysource.py for example.\n pattr = const\n pass\n elif opname in ('MAKE_FUNCTION', 'MAKE_CLOSURE'):\n if self.version >= 3.6:\n # 3.6+ doesn't have MAKE_CLOSURE, so opname == 'MAKE_FUNCTION'\n flags = argval\n opname = 'MAKE_FUNCTION_%d' % (flags)\n attr = []\n for flag in self.MAKE_FUNCTION_FLAGS:\n bit = flags & 1\n attr.append(bit)\n flags >>= 1\n attr = attr[:4] # remove last value: attr[5] == False\n else:\n pos_args, name_pair_args, annotate_args = parse_fn_counts(inst.argval)\n pattr = (\"%d positional, %d keyword pair, %d annotated\" %\n (pos_args, name_pair_args, annotate_args))\n if name_pair_args > 0:\n opname = '%s_N%d' % (opname, name_pair_args)\n pass\n if annotate_args > 0:\n opname = '%s_A_%d' % (opname, annotate_args)\n pass\n opname = '%s_%d' % (opname, pos_args)\n attr = (pos_args, name_pair_args, annotate_args)\n tokens.append(\n Token(\n opname = opname,\n attr = attr,\n pattr = pattr,\n offset = inst.offset,\n linestart = inst.starts_line,\n op = op,\n has_arg = inst.has_arg,\n opc = self.opc\n )\n )\n continue\n elif op in self.varargs_ops:\n pos_args = argval\n if self.is_pypy and not pos_args and opname == 'BUILD_MAP':\n opname = 'BUILD_MAP_n'\n else:\n opname = '%s_%d' % (opname, pos_args)\n\n elif self.is_pypy and opname == 'JUMP_IF_NOT_DEBUG':\n # The value in the dict is in special cases in semantic actions, such\n # as JUMP_IF_NOT_DEBUG. The value is not used in these cases, so we put\n # in arbitrary value 0.\n customize[opname] = 0\n elif opname == 'UNPACK_EX':\n # FIXME: try with scanner and parser by\n # changing argval\n before_args = argval & 0xFF\n after_args = (argval >> 8) & 0xff\n pattr = \"%d before vararg, %d after\" % (before_args, after_args)\n argval = (before_args, after_args)\n opname = '%s_%d+%d' % (opname, before_args, after_args)\n\n elif op == self.opc.JUMP_ABSOLUTE:\n # Further classify JUMP_ABSOLUTE into backward jumps\n # which are used in loops, and \"CONTINUE\" jumps which\n # may appear in a \"continue\" statement. The loop-type\n # and continue-type jumps will help us classify loop\n # boundaries The continue-type jumps help us get\n # \"continue\" statements with would otherwise be turned\n # into a \"pass\" statement because JUMPs are sometimes\n # ignored in rules as just boundary overhead. In\n # comprehensions we might sometimes classify JUMP_BACK\n # as CONTINUE, but that's okay since we add a grammar\n # rule for that.\n pattr = argval\n target = self.get_target(inst.offset)\n if target <= inst.offset:\n next_opname = self.insts[i+1].opname\n\n # 'Continue's include jumps to loops that are not\n # and the end of a block which follow with POP_BLOCK and COME_FROM_LOOP.\n # If the JUMP_ABSOLUTE is to a FOR_ITER and it is followed by another JUMP_FORWARD\n # then we'll take it as a \"continue\".\n is_continue = (self.insts[self.offset2inst_index[target]]\n .opname == 'FOR_ITER'\n and self.insts[i+1].opname == 'JUMP_FORWARD')\n\n if (is_continue or\n (inst.offset in self.stmts and (inst.starts_line and\n next_opname not in self.not_continue_follow))):\n opname = 'CONTINUE'\n else:\n opname = 'JUMP_BACK'\n # FIXME: this is a hack to catch stuff like:\n # if x: continue\n # the \"continue\" is not on a new line.\n # There are other situations where we don't catch\n # CONTINUE as well.\n if tokens[-1].kind == 'JUMP_BACK' and tokens[-1].attr <= argval:\n if tokens[-2].kind == 'BREAK_LOOP':\n del tokens[-1]\n else:\n # intern is used because we are changing the *previous* token\n tokens[-1].kind = intern('CONTINUE')\n if last_op_was_break and opname == 'CONTINUE':\n last_op_was_break = False\n continue\n\n # FIXME: go over for Python 3.6+. This is sometimes wrong\n elif op == self.opc.RETURN_VALUE:\n if inst.offset in self.return_end_ifs:\n opname = 'RETURN_END_IF'\n\n elif inst.offset in self.load_asserts:\n opname = 'LOAD_ASSERT'\n\n last_op_was_break = opname == 'BREAK_LOOP'\n tokens.append(\n Token(\n opname = opname,\n attr = argval,\n pattr = pattr,\n offset = inst.offset,\n linestart = inst.starts_line,\n op = op,\n has_arg = inst.has_arg,\n opc = self.opc\n )\n )\n pass\n\n if show_asm in ('both', 'after'):\n for t in tokens:\n print(t.format(line_prefix='L.'))\n print()\n return tokens, customize", "response": "Ingests a code object into a list of uncompyle6 Code objects."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef find_jump_targets(self, debug):\n code = self.code\n n = len(code)\n self.structs = [{'type': 'root',\n 'start': 0,\n 'end': n-1}]\n\n # All loop entry points\n self.loops = []\n\n # Map fixed jumps to their real destination\n self.fixed_jumps = {}\n self.except_targets = {}\n self.ignore_if = set()\n self.build_statement_indices()\n self.else_start = {}\n\n # Containers filled by detect_control_flow()\n self.not_continue = set()\n self.return_end_ifs = set()\n self.setup_loop_targets = {} # target given setup_loop offset\n self.setup_loops = {} # setup_loop offset given target\n\n targets = {}\n for i, inst in enumerate(self.insts):\n offset = inst.offset\n op = inst.opcode\n\n # Determine structures and fix jumps in Python versions\n # since 2.3\n self.detect_control_flow(offset, targets, i)\n\n if inst.has_arg:\n label = self.fixed_jumps.get(offset)\n oparg = inst.arg\n if (self.version >= 3.6 and\n self.code[offset] == self.opc.EXTENDED_ARG):\n j = xdis.next_offset(op, self.opc, offset)\n next_offset = xdis.next_offset(op, self.opc, j)\n else:\n next_offset = xdis.next_offset(op, self.opc, offset)\n\n if label is None:\n if op in self.opc.hasjrel and op != self.opc.FOR_ITER:\n label = next_offset + oparg\n elif op in self.opc.hasjabs:\n if op in self.jump_if_pop:\n if oparg > offset:\n label = oparg\n\n if label is not None and label != -1:\n targets[label] = targets.get(label, []) + [offset]\n elif op == self.opc.END_FINALLY and offset in self.fixed_jumps:\n label = self.fixed_jumps[offset]\n targets[label] = targets.get(label, []) + [offset]\n pass\n\n pass # for loop\n\n # DEBUG:\n if debug in ('both', 'after'):\n import pprint as pp\n pp.pprint(self.structs)\n\n return targets", "response": "Find all jump targets in a byte code."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef detect_control_flow(self, offset, targets, inst_index):\n\n code = self.code\n inst = self.insts[inst_index]\n op = inst.opcode\n\n\n # Detect parent structure\n parent = self.structs[0]\n start = parent['start']\n end = parent['end']\n\n # Pick inner-most parent for our offset\n for struct in self.structs:\n current_start = struct['start']\n current_end = struct['end']\n if ((current_start <= offset < current_end)\n and (current_start >= start and current_end <= end)):\n start = current_start\n end = current_end\n parent = struct\n\n if self.version < 3.8 and op == self.opc.SETUP_LOOP:\n # We categorize loop types: 'for', 'while', 'while 1' with\n # possibly suffixes '-loop' and '-else'\n # Try to find the jump_back instruction of the loop.\n # It could be a return instruction.\n\n start += inst.inst_size\n target = self.get_target(offset)\n end = self.restrict_to_parent(target, parent)\n self.setup_loops[target] = offset\n\n if target != end:\n self.fixed_jumps[offset] = end\n\n (line_no, next_line_byte) = self.lines[offset]\n jump_back = self.last_instr(start, end, self.opc.JUMP_ABSOLUTE,\n next_line_byte, False)\n\n if jump_back:\n jump_forward_offset = xdis.next_offset(code[jump_back], self.opc, jump_back)\n else:\n jump_forward_offset = None\n\n return_val_offset1 = self.prev[self.prev[end]]\n\n if (jump_back and jump_back != self.prev_op[end]\n and self.is_jump_forward(jump_forward_offset)):\n if (code[self.prev_op[end]] == self.opc.RETURN_VALUE or\n (code[self.prev_op[end]] == self.opc.POP_BLOCK\n and code[return_val_offset1] == self.opc.RETURN_VALUE)):\n jump_back = None\n if not jump_back:\n # loop suite ends in return\n jump_back = self.last_instr(start, end, self.opc.RETURN_VALUE)\n if not jump_back:\n return\n\n jb_inst = self.get_inst(jump_back)\n jump_back = self.next_offset(jb_inst.opcode, jump_back)\n\n if_offset = None\n if code[self.prev_op[next_line_byte]] not in self.pop_jump_tf:\n if_offset = self.prev[next_line_byte]\n if if_offset:\n loop_type = 'while'\n self.ignore_if.add(if_offset)\n else:\n loop_type = 'for'\n target = next_line_byte\n end = xdis.next_offset(code[jump_back], self.opc, jump_back)\n else:\n if self.get_target(jump_back) >= next_line_byte:\n jump_back = self.last_instr(start, end, self.opc.JUMP_ABSOLUTE, start, False)\n\n jb_inst = self.get_inst(jump_back)\n\n jb_next_offset = self.next_offset(jb_inst.opcode, jump_back)\n if end > jb_next_offset and self.is_jump_forward(end):\n if self.is_jump_forward(jb_next_offset):\n if self.get_target(jb_next_offset) == self.get_target(end):\n self.fixed_jumps[offset] = jb_next_offset\n end = jb_next_offset\n elif target < offset:\n self.fixed_jumps[offset] = jb_next_offset\n end = jb_next_offset\n\n target = self.get_target(jump_back)\n\n if code[target] in (self.opc.FOR_ITER, self.opc.GET_ITER):\n loop_type = 'for'\n else:\n loop_type = 'while'\n test = self.prev_op[next_line_byte]\n\n if test == offset:\n loop_type = 'while 1'\n elif self.code[test] in self.opc.JUMP_OPs:\n self.ignore_if.add(test)\n test_target = self.get_target(test)\n if test_target > (jump_back+3):\n jump_back = test_target\n self.not_continue.add(jump_back)\n self.loops.append(target)\n self.structs.append({'type': loop_type + '-loop',\n 'start': target,\n 'end': jump_back})\n after_jump_offset = xdis.next_offset(code[jump_back], self.opc, jump_back)\n if after_jump_offset != end:\n self.structs.append({'type': loop_type + '-else',\n 'start': after_jump_offset,\n 'end': end})\n elif op in self.pop_jump_tf:\n start = offset + inst.inst_size\n target = inst.argval\n rtarget = self.restrict_to_parent(target, parent)\n prev_op = self.prev_op\n\n # Do not let jump to go out of parent struct bounds\n if target != rtarget and parent['type'] == 'and/or':\n self.fixed_jumps[offset] = rtarget\n return\n\n # Does this jump to right after another conditional jump that is\n # not myself? If so, it's part of a larger conditional.\n # rocky: if we have a conditional jump to the next instruction, then\n # possibly I am \"skipping over\" a \"pass\" or null statement.\n pretarget = self.get_inst(prev_op[target])\n\n if (pretarget.opcode in self.pop_jump_if_pop and\n (target > offset) and pretarget.offset != offset):\n\n # FIXME: hack upon hack...\n # In some cases the pretarget can be a jump to the next instruction\n # and these aren't and/or's either. We limit to 3.5+ since we experienced there\n # but it might be earlier versions, or might be a general principle.\n if self.version < 3.5 or pretarget.argval != target:\n # FIXME: this is not accurate The commented out below\n # is what it should be. However grammar rules right now\n # assume the incorrect offsets.\n # self.fixed_jumps[offset] = target\n self.fixed_jumps[offset] = pretarget.offset\n self.structs.append({'type': 'and/or',\n 'start': start,\n 'end': pretarget.offset})\n return\n\n # The opcode *two* instructions before the target jump offset is important\n # in making a determination of what we have. Save that.\n pre_rtarget = prev_op[rtarget]\n\n # Is it an \"and\" inside an \"if\" or \"while\" block\n if op == self.opc.POP_JUMP_IF_FALSE:\n\n # Search for another POP_JUMP_IF_FALSE targetting the same op,\n # in current statement, starting from current offset, and filter\n # everything inside inner 'or' jumps and midline ifs\n match = self.rem_or(start, self.next_stmt[offset],\n self.opc.POP_JUMP_IF_FALSE, target)\n\n # If we still have any offsets in set, start working on it\n if match:\n is_jump_forward = self.is_jump_forward(pre_rtarget)\n if (is_jump_forward and pre_rtarget not in self.stmts and\n self.restrict_to_parent(self.get_target(pre_rtarget), parent) == rtarget):\n if (code[prev_op[pre_rtarget]] == self.opc.JUMP_ABSOLUTE\n and self.remove_mid_line_ifs([offset]) and\n target == self.get_target(prev_op[pre_rtarget]) and\n (prev_op[pre_rtarget] not in self.stmts or\n self.get_target(prev_op[pre_rtarget]) > prev_op[pre_rtarget]) and\n 1 == len(self.remove_mid_line_ifs(self.rem_or(start, prev_op[pre_rtarget], self.pop_jump_tf, target)))):\n pass\n elif (code[prev_op[pre_rtarget]] == self.opc.RETURN_VALUE\n and self.remove_mid_line_ifs([offset]) and\n 1 == (len(set(self.remove_mid_line_ifs(self.rem_or(start, prev_op[pre_rtarget],\n self.pop_jump_tf, target))) |\n set(self.remove_mid_line_ifs(self.rem_or(start, prev_op[pre_rtarget],\n (self.opc.POP_JUMP_IF_FALSE,\n self.opc.POP_JUMP_IF_TRUE,\n self.opc.JUMP_ABSOLUTE),\n pre_rtarget, True)))))):\n pass\n else:\n fix = None\n jump_ifs = self.inst_matches(start, self.next_stmt[offset],\n self.opc.POP_JUMP_IF_FALSE)\n last_jump_good = True\n for j in jump_ifs:\n if target == self.get_target(j):\n # FIXME: remove magic number\n if self.lines[j].next == j + 3 and last_jump_good:\n fix = j\n break\n else:\n last_jump_good = False\n self.fixed_jumps[offset] = fix or match[-1]\n return\n else:\n self.fixed_jumps[offset] = match[-1]\n return\n # op == POP_JUMP_IF_TRUE\n else:\n next = self.next_stmt[offset]\n if prev_op[next] == offset:\n pass\n elif self.is_jump_forward(next) and target == self.get_target(next):\n if code[prev_op[next]] == self.opc.POP_JUMP_IF_FALSE:\n if (code[next] == self.opc.JUMP_FORWARD\n or target != rtarget\n or code[prev_op[pre_rtarget]] not in\n (self.opc.JUMP_ABSOLUTE, self.opc.RETURN_VALUE)):\n self.fixed_jumps[offset] = prev_op[next]\n return\n elif (code[next] == self.opc.JUMP_ABSOLUTE and self.is_jump_forward(target) and\n self.get_target(target) == self.get_target(next)):\n self.fixed_jumps[offset] = prev_op[next]\n return\n\n # Don't add a struct for a while test, it's already taken care of\n if offset in self.ignore_if:\n return\n\n rtarget_is_ja = code[pre_rtarget] == self.opc.JUMP_ABSOLUTE\n if ( rtarget_is_ja and\n pre_rtarget in self.stmts and\n pre_rtarget != offset and\n prev_op[pre_rtarget] != offset and\n not (code[rtarget] == self.opc.JUMP_ABSOLUTE and\n code[rtarget+3] == self.opc.POP_BLOCK and\n code[prev_op[pre_rtarget]] != self.opc.JUMP_ABSOLUTE)):\n rtarget = pre_rtarget\n\n # Does the \"jump if\" jump beyond a jump op?\n # That is, we have something like:\n # POP_JUMP_IF_FALSE HERE\n # ...\n # JUMP_FORWARD\n # HERE:\n #\n # If so, this can be block inside an \"if\" statement\n # or a conditional assignment like:\n # x = 1 if x else 2\n #\n # For 3.5, in addition the JUMP_FORWARD above we could have\n # JUMP_BACK or CONTINUE\n #\n # There are other situations we may need to consider, like\n # if the condition jump is to a forward location.\n # Also the existence of a jump to the instruction after \"END_FINALLY\"\n # will distinguish \"try/else\" from \"try\".\n if self.version < 3.8:\n rtarget_break = (self.opc.RETURN_VALUE, self.opc.BREAK_LOOP)\n else:\n rtarget_break = (self.opc.RETURN_VALUE,)\n\n if self.is_jump_forward(pre_rtarget) or (rtarget_is_ja and self.version >= 3.5):\n if_end = self.get_target(pre_rtarget)\n\n # If the jump target is back, we are looping\n if (if_end < pre_rtarget and self.version < 3.8 and\n (code[prev_op[if_end]] == self.opc.SETUP_LOOP)):\n if (if_end > start):\n return\n\n end = self.restrict_to_parent(if_end, parent)\n\n self.structs.append({'type': 'if-then',\n 'start': start,\n 'end': pre_rtarget})\n\n # FIXME: add this\n # self.fixed_jumps[offset] = rtarget\n self.not_continue.add(pre_rtarget)\n\n if rtarget < end and (\n code[rtarget] not in (self.opc.END_FINALLY,\n self.opc.JUMP_ABSOLUTE) and\n code[prev_op[pre_rtarget]] not in (self.opc.POP_EXCEPT,\n self.opc.END_FINALLY)):\n self.structs.append({'type': 'else',\n 'start': rtarget,\n 'end': end})\n self.else_start[rtarget] = end\n elif self.is_jump_back(pre_rtarget, 0):\n if_end = rtarget\n self.structs.append({'type': 'if-then',\n 'start': start,\n 'end': pre_rtarget})\n self.not_continue.add(pre_rtarget)\n elif code[pre_rtarget] in rtarget_break:\n self.structs.append({'type': 'if-then',\n 'start': start,\n 'end': rtarget})\n # It is important to distingish if this return is inside some sort\n # except block return\n jump_prev = prev_op[offset]\n if self.is_pypy and code[jump_prev] == self.opc.COMPARE_OP:\n if self.opc.cmp_op[code[jump_prev+1]] == 'exception-match':\n return\n if self.version >= 3.5:\n # Python 3.5 may remove as dead code a JUMP\n # instruction after a RETURN_VALUE. So we check\n # based on seeing SETUP_EXCEPT various places.\n if self.version < 3.8 and code[rtarget] == self.opc.SETUP_EXCEPT:\n return\n # Check that next instruction after pops and jump is\n # not from SETUP_EXCEPT\n next_op = rtarget\n if code[next_op] == self.opc.POP_BLOCK:\n next_op += instruction_size(self.code[next_op], self.opc)\n if code[next_op] == self.opc.JUMP_ABSOLUTE:\n next_op += instruction_size(self.code[next_op], self.opc)\n if next_op in targets:\n for try_op in targets[next_op]:\n come_from_op = code[try_op]\n if self.version < 3.8 and come_from_op == self.opc.SETUP_EXCEPT:\n return\n pass\n pass\n\n if self.version >= 3.4:\n self.fixed_jumps[offset] = rtarget\n\n if code[pre_rtarget] == self.opc.RETURN_VALUE:\n # If we are at some sort of POP_JUMP_IF and the instruction before was\n # COMPARE_OP exception-match, then pre_rtarget is not an end_if\n if not (inst_index > 0 and self.insts[inst_index-1].argval == 'exception-match'):\n self.return_end_ifs.add(pre_rtarget)\n else:\n self.fixed_jumps[offset] = rtarget\n self.not_continue.add(pre_rtarget)\n else:\n\n # FIXME: this is very convoluted and based on rather hacky\n # empirical evidence. It should go a way when\n # we have better control-flow analysis\n normal_jump = self.version >= 3.6\n if self.version == 3.5:\n j = self.offset2inst_index[target]\n if j+2 < len(self.insts) and self.insts[j+2].is_jump_target:\n normal_jump = self.insts[j+1].opname == 'POP_BLOCK'\n\n if normal_jump:\n # For now, we'll only tag forward jump.\n if target > offset:\n self.fixed_jumps[offset] = target\n pass\n else:\n # FIXME: This is probably a bug in < 3.5 and we should\n # instead use the above code. But until we smoke things\n # out we'll stick with it.\n if rtarget > offset:\n self.fixed_jumps[offset] = rtarget\n\n elif self.version < 3.8 and op == self.opc.SETUP_EXCEPT:\n target = self.get_target(offset)\n end = self.restrict_to_parent(target, parent)\n self.fixed_jumps[offset] = end\n elif op == self.opc.POP_EXCEPT:\n next_offset = xdis.next_offset(op, self.opc, offset)\n target = self.get_target(next_offset)\n if target > next_offset:\n next_op = code[next_offset]\n if (self.opc.JUMP_ABSOLUTE == next_op and\n self.opc.END_FINALLY != code[xdis.next_offset(next_op, self.opc, next_offset)]):\n self.fixed_jumps[next_offset] = target\n self.except_targets[target] = next_offset\n\n elif op == self.opc.SETUP_FINALLY:\n target = self.get_target(offset)\n end = self.restrict_to_parent(target, parent)\n self.fixed_jumps[offset] = end\n elif op in self.jump_if_pop:\n target = self.get_target(offset)\n if target > offset:\n unop_target = self.last_instr(offset, target, self.opc.JUMP_FORWARD, target)\n if unop_target and code[unop_target+3] != self.opc.ROT_TWO:\n self.fixed_jumps[offset] = unop_target\n else:\n self.fixed_jumps[offset] = self.restrict_to_parent(target, parent)\n pass\n pass\n elif self.version >= 3.5:\n # 3.5+ has Jump optimization which too often causes RETURN_VALUE to get\n # misclassified as RETURN_END_IF. Handle that here.\n # In RETURN_VALUE, JUMP_ABSOLUTE, RETURN_VALUE is never RETURN_END_IF\n if op == self.opc.RETURN_VALUE:\n next_offset = xdis.next_offset(op, self.opc, offset)\n if ( next_offset < len(code) and\n (code[next_offset] == self.opc.JUMP_ABSOLUTE and\n offset in self.return_end_ifs) ):\n self.return_end_ifs.remove(offset)\n pass\n pass\n elif op == self.opc.JUMP_FORWARD:\n # If we have:\n # JUMP_FORWARD x, [non-jump, insns], RETURN_VALUE, x:\n # then RETURN_VALUE is not RETURN_END_IF\n rtarget = self.get_target(offset)\n rtarget_prev = self.prev[rtarget]\n if (code[rtarget_prev] == self.opc.RETURN_VALUE and\n rtarget_prev in self.return_end_ifs):\n i = rtarget_prev\n while i != offset:\n if code[i] in [op3.JUMP_FORWARD, op3.JUMP_ABSOLUTE]:\n return\n i = self.prev[i]\n self.return_end_ifs.remove(rtarget_prev)\n pass\n return", "response": "Detects the type of control flow for a given instruction."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef is_jump_back(self, offset, extended_arg):\n if self.code[offset] != self.opc.JUMP_ABSOLUTE:\n return False\n return offset > self.get_target(offset, extended_arg)", "response": "Return True if the code at offset is some sort of jump back."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef next_except_jump(self, start):\n\n if self.code[start] == self.opc.DUP_TOP:\n except_match = self.first_instr(start, len(self.code), self.opc.POP_JUMP_IF_FALSE)\n if except_match:\n jmp = self.prev_op[self.get_target(except_match)]\n self.ignore_if.add(except_match)\n self.not_continue.add(jmp)\n return jmp\n\n count_END_FINALLY = 0\n count_SETUP_ = 0\n for i in self.op_range(start, len(self.code)):\n op = self.code[i]\n if op == self.opc.END_FINALLY:\n if count_END_FINALLY == count_SETUP_:\n assert self.code[self.prev_op[i]] in frozenset([self.opc.JUMP_ABSOLUTE,\n self.opc.JUMP_FORWARD,\n self.opc.RETURN_VALUE])\n self.not_continue.add(self.prev_op[i])\n return self.prev_op[i]\n count_END_FINALLY += 1\n elif op in self.setup_opts_no_loop:\n count_SETUP_ += 1", "response": "Return the next except statement that was generated by an except... else clause."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef rem_or(self, start, end, instr, target=None, include_beyond_target=False):\n assert(start >= 0 and end <= len(self.code) and start <= end)\n\n # Find all offsets of requested instructions\n instr_offsets = self.inst_matches(start, end, instr, target,\n include_beyond_target)\n # Get all POP_JUMP_IF_TRUE (or) offsets\n if self.version == 3.0:\n jump_true_op = self.opc.JUMP_IF_TRUE\n else:\n jump_true_op = self.opc.POP_JUMP_IF_TRUE\n pjit_offsets = self.inst_matches(start, end, jump_true_op)\n filtered = []\n for pjit_offset in pjit_offsets:\n pjit_tgt = self.get_target(pjit_offset) - 3\n for instr_offset in instr_offsets:\n if instr_offset <= pjit_offset or instr_offset >= pjit_tgt:\n filtered.append(instr_offset)\n instr_offsets = filtered\n filtered = []\n return instr_offsets", "response": "Return a list of foundCOOKIE_ID s which are not within any POP_JUMP_IF_TRUE jumps."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef build_instructions(self, co):\n # FIXME: remove this when all subsidiary functions have been removed.\n # We should be able to get everything from the self.insts list.\n self.code = array('B', co.co_code)\n\n bytecode = Bytecode(co, self.opc)\n self.build_prev_op()\n self.insts = self.remove_extended_args(list(bytecode))\n self.lines = self.build_lines_data(co)\n self.offset2inst_index = {}\n for i, inst in enumerate(self.insts):\n self.offset2inst_index[inst.offset] = i\n\n return bytecode", "response": "Build the list of instructions for a single object."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef build_lines_data(self, code_obj):\n\n # Offset: lineno pairs, only for offsets which start line.\n # Locally we use list for more convenient iteration using indices\n if self.version > 1.4:\n linestarts = list(self.opc.findlinestarts(code_obj))\n else:\n linestarts = [[0, 1]]\n self.linestarts = dict(linestarts)\n\n # 'List-map' which shows line number of current op and offset of\n # first op on following line, given offset of op as index\n lines = []\n LineTuple = namedtuple('LineTuple', ['l_no', 'next'])\n\n # Iterate through available linestarts, and fill\n # the data for all code offsets encountered until\n # last linestart offset\n _, prev_line_no = linestarts[0]\n offset = 0\n for start_offset, line_no in linestarts[1:]:\n while offset < start_offset:\n lines.append(LineTuple(prev_line_no, start_offset))\n offset += 1\n prev_line_no = line_no\n\n # Fill remaining offsets with reference to last line number\n # and code length as start offset of following non-existing line\n codelen = len(self.code)\n while offset < codelen:\n lines.append(LineTuple(prev_line_no, codelen))\n offset += 1\n return lines", "response": "Generate various line - related helper data."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef build_prev_op(self):\n code = self.code\n codelen = len(code)\n # 2.x uses prev 3.x uses prev_op. Sigh\n # Until we get this sorted out.\n self.prev = self.prev_op = [0]\n for offset in self.op_range(0, codelen):\n op = code[offset]\n for _ in range(instruction_size(op, self.opc)):\n self.prev_op.append(offset)", "response": "Build the list - map which allows to jump to previous\n op given offset of current op as index."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning True if the code at offset is some sort of jump forward.", "response": "def is_jump_forward(self, offset):\n \"\"\"\n Return True if the code at offset is some sort of jump forward.\n That is, it is ether \"JUMP_FORWARD\" or an absolute jump that\n goes forward.\n \"\"\"\n opname = self.get_inst(offset).opname\n if opname == 'JUMP_FORWARD':\n return True\n if opname != 'JUMP_ABSOLUTE':\n return False\n return offset < self.get_target(offset)"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_target(self, offset, extended_arg=0):\n inst = self.get_inst(offset)\n if inst.opcode in self.opc.JREL_OPS | self.opc.JABS_OPS:\n target = inst.argval\n else:\n # No jump offset, so use fall-through offset\n target = next_offset(inst.opcode, self.opc, inst.offset)\n return target", "response": "Get the target instruction offset for the given opcode."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef first_instr(self, start, end, instr, target=None, exact=True):\n code = self.code\n assert(start >= 0 and end <= len(code))\n\n try:\n None in instr\n except:\n instr = [instr]\n\n result_offset = None\n current_distance = len(code)\n for offset in self.op_range(start, end):\n op = code[offset]\n if op in instr:\n if target is None:\n return offset\n dest = self.get_target(offset)\n if dest == target:\n return offset\n elif not exact:\n new_distance = abs(target - dest)\n if new_distance < current_distance:\n current_distance = new_distance\n result_offset = offset\n return result_offset", "response": "Find the first instruction in the python bytecode block from start to end."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef last_instr(self, start, end, instr, target=None, exact=True):\n\n code = self.code\n # Make sure requested positions do not go out of\n # code bounds\n if not (start >= 0 and end <= len(code)):\n return None\n\n try:\n None in instr\n except:\n instr = [instr]\n\n result_offset = None\n current_distance = self.insts[-1].offset - self.insts[0].offset\n extended_arg = 0\n # FIXME: use self.insts rather than code[]\n for offset in self.op_range(start, end):\n op = code[offset]\n\n if op == self.opc.EXTENDED_ARG:\n arg = code2num(code, offset+1) | extended_arg\n extended_arg = extended_arg_val(self.opc, arg)\n continue\n\n if op in instr:\n if target is None:\n result_offset = offset\n else:\n dest = self.get_target(offset, extended_arg)\n if dest == target:\n current_distance = 0\n result_offset = offset\n elif not exact:\n new_distance = abs(target - dest)\n if new_distance <= current_distance:\n current_distance = new_distance\n result_offset = offset\n pass\n pass\n pass\n pass\n extended_arg = 0\n pass\n return result_offset", "response": "Find the last instruction in the block from start to end."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef inst_matches(self, start, end, instr, target=None, include_beyond_target=False):\n try:\n None in instr\n except:\n instr = [instr]\n\n first = self.offset2inst_index[start]\n result = []\n for inst in self.insts[first:]:\n if inst.opcode in instr:\n if target is None:\n result.append(inst.offset)\n else:\n t = self.get_target(inst.offset)\n if include_beyond_target and t >= target:\n result.append(inst.offset)\n elif t == target:\n result.append(inst.offset)\n pass\n pass\n pass\n if inst.offset >= end:\n break\n pass\n\n # FIXME: put in a test\n # check = self.all_instr(start, end, instr, target, include_beyond_target)\n # assert result == check\n\n return result", "response": "Find all instruction in the block from start to end and return a list of indexes to those instructions that match the given instruction."} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nfinds all instructions in a block from start to end.", "response": "def all_instr(self, start, end, instr, target=None, include_beyond_target=False):\n \"\"\"\n Find all `instr` in the block from start to end.\n `instr` is any Python opcode or a list of opcodes\n If `instr` is an opcode with a target (like a jump), a target\n destination can be specified which must match precisely.\n\n Return a list with indexes to them or [] if none found.\n \"\"\"\n\n code = self.code\n assert(start >= 0 and end <= len(code))\n\n try:\n None in instr\n except:\n instr = [instr]\n\n result = []\n extended_arg = 0\n for offset in self.op_range(start, end):\n\n op = code[offset]\n\n if op == self.opc.EXTENDED_ARG:\n arg = code2num(code, offset+1) | extended_arg\n extended_arg = extended_arg_val(self.opc, arg)\n continue\n\n if op in instr:\n if target is None:\n result.append(offset)\n else:\n t = self.get_target(offset, extended_arg)\n if include_beyond_target and t >= target:\n result.append(offset)\n elif t == target:\n result.append(offset)\n pass\n pass\n pass\n extended_arg = 0\n pass\n\n return result"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef op_range(self, start, end):\n while start < end:\n yield start\n start += instruction_size(self.code[start], self.opc)", "response": "Iterate through the opcodes in the sequence starting at start and ending at end."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef remove_extended_args(self, instructions):\n new_instructions = []\n last_was_extarg = False\n n = len(instructions)\n for i, inst in enumerate(instructions):\n if (inst.opname == 'EXTENDED_ARG'\n and i+1 < n and instructions[i+1].opname != 'MAKE_FUNCTION'):\n last_was_extarg = True\n starts_line = inst.starts_line\n is_jump_target = inst.is_jump_target\n offset = inst.offset\n continue\n if last_was_extarg:\n\n # j = self.stmts.index(inst.offset)\n # self.lines[j] = offset\n\n new_inst = inst._replace(starts_line=starts_line,\n is_jump_target=is_jump_target,\n offset=offset)\n inst = new_inst\n if i < n:\n new_prev = self.prev_op[instructions[i].offset]\n j = instructions[i+1].offset\n old_prev = self.prev_op[j]\n while self.prev_op[j] == old_prev and j < n:\n self.prev_op[j] = new_prev\n j += 1\n\n last_was_extarg = False\n new_instructions.append(inst)\n return new_instructions", "response": "Go through instructions removing extended ARG."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef remove_mid_line_ifs(self, ifs):\n\n # FIXME: this doesn't work for Python 3.6+\n\n filtered = []\n for i in ifs:\n # For each offset, if line number of current and next op\n # is the same\n if self.lines[i].l_no == self.lines[i+3].l_no:\n # Skip last op on line if it is some sort of POP_JUMP.\n if self.code[self.prev[self.lines[i].next]] in (self.opc.PJIT, self.opc.PJIF):\n continue\n filtered.append(i)\n return filtered", "response": "Remove the ifs from the mid - line."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef restrict_to_parent(self, target, parent):\n if not (parent['start'] < target < parent['end']):\n target = parent['end']\n return target", "response": "Restrict target to parent structure boundaries."} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns a parser object for the specified Python version.", "response": "def get_python_parser(\n version, debug_parser=PARSER_DEFAULT_DEBUG, compile_mode='exec',\n is_pypy = False):\n \"\"\"Returns parser object for Python version 2 or 3, 3.2, 3.5on,\n etc., depending on the parameters passed. *compile_mode* is either\n 'exec', 'eval', or 'single'. See\n https://docs.python.org/3.6/library/functions.html#compile for an\n explanation of the different modes.\n \"\"\"\n\n # If version is a string, turn that into the corresponding float.\n if isinstance(version, str):\n version = py_str2float(version)\n\n # FIXME: there has to be a better way...\n # We could do this as a table lookup, but that would force us\n # in import all of the parsers all of the time. Perhaps there is\n # a lazy way of doing the import?\n\n if version < 3.0:\n if version < 2.2:\n if version == 1.3:\n import uncompyle6.parsers.parse13 as parse13\n if compile_mode == 'exec':\n p = parse13.Python14Parser(debug_parser)\n else:\n p = parse13.Python14ParserSingle(debug_parser)\n elif version == 1.4:\n import uncompyle6.parsers.parse14 as parse14\n if compile_mode == 'exec':\n p = parse14.Python14Parser(debug_parser)\n else:\n p = parse14.Python14ParserSingle(debug_parser)\n elif version == 1.5:\n import uncompyle6.parsers.parse15 as parse15\n if compile_mode == 'exec':\n p = parse15.Python15Parser(debug_parser)\n else:\n p = parse15.Python15ParserSingle(debug_parser)\n elif version == 2.1:\n import uncompyle6.parsers.parse21 as parse21\n if compile_mode == 'exec':\n p = parse21.Python21Parser(debug_parser)\n else:\n p = parse21.Python21ParserSingle(debug_parser)\n elif version == 2.2:\n import uncompyle6.parsers.parse22 as parse22\n if compile_mode == 'exec':\n p = parse22.Python22Parser(debug_parser)\n else:\n p = parse22.Python22ParserSingle(debug_parser)\n elif version == 2.3:\n import uncompyle6.parsers.parse23 as parse23\n if compile_mode == 'exec':\n p = parse23.Python23Parser(debug_parser)\n else:\n p = parse23.Python23ParserSingle(debug_parser)\n elif version == 2.4:\n import uncompyle6.parsers.parse24 as parse24\n if compile_mode == 'exec':\n p = parse24.Python24Parser(debug_parser)\n else:\n p = parse24.Python24ParserSingle(debug_parser)\n elif version == 2.5:\n import uncompyle6.parsers.parse25 as parse25\n if compile_mode == 'exec':\n p = parse25.Python25Parser(debug_parser)\n else:\n p = parse25.Python25ParserSingle(debug_parser)\n elif version == 2.6:\n import uncompyle6.parsers.parse26 as parse26\n if compile_mode == 'exec':\n p = parse26.Python26Parser(debug_parser)\n else:\n p = parse26.Python26ParserSingle(debug_parser)\n elif version == 2.7:\n import uncompyle6.parsers.parse27 as parse27\n if compile_mode == 'exec':\n p = parse27.Python27Parser(debug_parser)\n else:\n p = parse27.Python27ParserSingle(debug_parser)\n else:\n import uncompyle6.parsers.parse2 as parse2\n if compile_mode == 'exec':\n p = parse2.Python2Parser(debug_parser)\n else:\n p = parse2.Python2ParserSingle(debug_parser)\n pass\n pass\n pass\n else:\n import uncompyle6.parsers.parse3 as parse3\n if version == 3.0:\n import uncompyle6.parsers.parse30 as parse30\n if compile_mode == 'exec':\n p = parse30.Python30Parser(debug_parser)\n else:\n p = parse30.Python30ParserSingle(debug_parser)\n elif version == 3.1:\n import uncompyle6.parsers.parse31 as parse31\n if compile_mode == 'exec':\n p = parse31.Python31Parser(debug_parser)\n else:\n p = parse31.Python31ParserSingle(debug_parser)\n elif version == 3.2:\n import uncompyle6.parsers.parse32 as parse32\n if compile_mode == 'exec':\n p = parse32.Python32Parser(debug_parser)\n else:\n p = parse32.Python32ParserSingle(debug_parser)\n elif version == 3.3:\n import uncompyle6.parsers.parse33 as parse33\n if compile_mode == 'exec':\n p = parse33.Python33Parser(debug_parser)\n else:\n p = parse33.Python33ParserSingle(debug_parser)\n elif version == 3.4:\n import uncompyle6.parsers.parse34 as parse34\n if compile_mode == 'exec':\n p = parse34.Python34Parser(debug_parser)\n else:\n p = parse34.Python34ParserSingle(debug_parser)\n elif version == 3.5:\n import uncompyle6.parsers.parse35 as parse35\n if compile_mode == 'exec':\n p = parse35.Python35Parser(debug_parser)\n else:\n p = parse35.Python35ParserSingle(debug_parser)\n elif version == 3.6:\n import uncompyle6.parsers.parse36 as parse36\n if compile_mode == 'exec':\n p = parse36.Python36Parser(debug_parser)\n else:\n p = parse36.Python36ParserSingle(debug_parser)\n elif version == 3.7:\n import uncompyle6.parsers.parse37 as parse37\n if compile_mode == 'exec':\n p = parse37.Python37Parser(debug_parser)\n else:\n p = parse37.Python37ParserSingle(debug_parser)\n elif version == 3.8:\n import uncompyle6.parsers.parse38 as parse38\n if compile_mode == 'exec':\n p = parse38.Python38Parser(debug_parser)\n else:\n p = parse38.Python38ParserSingle(debug_parser)\n else:\n if compile_mode == 'exec':\n p = parse3.Python3Parser(debug_parser)\n else:\n p = parse3.Python3ParserSingle(debug_parser)\n p.version = version\n # p.dump_grammar() # debug\n return p"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nparsing a code object to an abstract syntax tree representation.", "response": "def python_parser(version, co, out=sys.stdout, showasm=False,\n parser_debug=PARSER_DEFAULT_DEBUG, is_pypy=False):\n \"\"\"\n Parse a code object to an abstract syntax tree representation.\n\n :param version: The python version this code is from as a float, for\n example 2.6, 2.7, 3.2, 3.3, 3.4, 3.5 etc.\n :param co: The code object to parse.\n :param out: File like object to write the output to.\n :param showasm: Flag which determines whether the disassembled and\n ingested code is written to sys.stdout or not.\n :param parser_debug: dict containing debug flags for the spark parser.\n\n :return: Abstract syntax tree representation of the code object.\n \"\"\"\n\n assert iscode(co)\n from uncompyle6.scanner import get_scanner\n scanner = get_scanner(version, is_pypy)\n tokens, customize = scanner.ingest(co)\n maybe_show_asm(showasm, tokens)\n\n # For heavy grammar debugging\n # parser_debug = {'rules': True, 'transition': True, 'reduce' : True,\n # 'showstack': 'full'}\n p = get_python_parser(version, parser_debug)\n return parse(p, tokens, customize)"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nadds rule to grammar but only if it hasn t been added previously availabe.", "response": "def add_unique_rule(self, rule, opname, arg_count, customize):\n \"\"\"Add rule to grammar, but only if it hasn't been added previously\n opname and stack_count are used in the customize() semantic\n the actions to add the semantic action rule. Stack_count is\n used in custom opcodes like MAKE_FUNCTION to indicate how\n many arguments it has. Often it is not used.\n \"\"\"\n if rule not in self.new_rules:\n # print(\"XXX \", rule) # debug\n self.new_rules.add(rule)\n self.addRule(rule, nop_func)\n customize[opname] = arg_count\n pass\n return"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nadds unique rules to grammar.", "response": "def add_unique_rules(self, rules, customize):\n \"\"\"Add rules (a list of string) to grammar. Note that\n the rules must not be those that set arg_count in the\n custom dictionary.\n \"\"\"\n for rule in rules:\n if len(rule) == 0:\n continue\n opname = rule.split('::=')[0].strip()\n self.add_unique_rule(rule, opname, 0, customize)\n return"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef add_unique_doc_rules(self, rules_str, customize):\n rules = [r.strip() for r in rules_str.split(\"\\n\")]\n self.add_unique_rules(rules, customize)\n return", "response": "Add rules to grammar. The rules can be a docstring - like list of rules."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef cleanup(self):\n for dict in (self.rule2func, self.rules, self.rule2name):\n for i in list(dict.keys()):\n dict[i] = None\n for i in dir(self):\n setattr(self, i, None)", "response": "Remove recursive references to allow garbage collector to collect this object."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ncustomizing format and print for our kind of tokens which gets called in debugging grammar reduce rules", "response": "def debug_reduce(self, rule, tokens, parent, last_token_pos):\n \"\"\"Customized format and print for our kind of tokens\n which gets called in debugging grammar reduce rules\n \"\"\"\n def fix(c):\n s = str(c)\n last_token_pos = s.find('_')\n if last_token_pos == -1:\n return s\n else:\n return s[:last_token_pos]\n\n prefix = ''\n if parent and tokens:\n p_token = tokens[parent]\n if hasattr(p_token, 'linestart') and p_token.linestart:\n prefix = 'L.%3d: ' % p_token.linestart\n else:\n prefix = ' '\n if hasattr(p_token, 'offset'):\n prefix += \"%3s\" % fix(p_token.offset)\n if len(rule[1]) > 1:\n prefix += '-%-3s ' % fix(tokens[last_token_pos-1].offset)\n else:\n prefix += ' '\n else:\n prefix = ' '\n\n print(\"%s%s ::= %s (%d)\" % (prefix, rule[0], ' '.join(rule[1]), last_token_pos))"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get_pos_kw(self, token):\n # Low byte indicates number of positional paramters,\n # high byte number of keyword parameters\n args_pos = token.attr & 0xff\n args_kw = (token.attr >> 8) & 0xff\n return args_pos, args_kw", "response": "Return then the number of positional parameters and the number of keyword parameters represented by the attr field of token"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nremoving __ from the end of _name_.", "response": "def unmangle_name(name, classname):\n \"\"\"Remove __ from the end of _name_ if it starts with __classname__\n return the \"unmangled\" name.\n \"\"\"\n if name.startswith(classname) and name[-2:] != '__':\n return name[len(classname) - 2:]\n return name"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef unmangle_code_names(self, co, classname):\n if classname:\n classname = '_' + classname.lstrip('_') + '__'\n\n free = [ self.unmangle_name(name, classname)\n for name in (co.co_cellvars + co.co_freevars) ]\n names = [ self.unmangle_name(name, classname)\n for name in co.co_names ]\n varnames = [ self.unmangle_name(name, classname)\n for name in co.co_varnames ]\n else:\n free = co.co_cellvars + co.co_freevars\n names = co.co_names\n varnames = co.co_varnames\n return free, names, varnames", "response": "Remove _ from the end of _name_ return the unmangled name."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ningesting a code object into a list of uncompyle6 Token s.", "response": "def ingest(self, co, classname=None, code_objects={}, show_asm=None):\n \"\"\"\n Pick out tokens from an uncompyle6 code object, and transform them,\n returning a list of uncompyle6 Token's.\n\n The transformations are made to assist the deparsing grammar.\n Specificially:\n - various types of LOAD_CONST's are categorized in terms of what they load\n - COME_FROM instructions are added to assist parsing control structures\n - MAKE_FUNCTION and FUNCTION_CALLS append the number of positional arguments\n - some EXTENDED_ARGS instructions are removed\n\n Also, when we encounter certain tokens, we add them to a set which will cause custom\n grammar rules. Specifically, variable arg tokens like MAKE_FUNCTION or BUILD_LIST\n cause specific rules for the specific number of arguments they take.\n \"\"\"\n\n if not show_asm:\n show_asm = self.show_asm\n\n bytecode = self.build_instructions(co)\n\n # show_asm = 'after'\n if show_asm in ('both', 'before'):\n for instr in bytecode.get_instructions(co):\n print(instr.disassemble())\n\n # list of tokens/instructions\n tokens = []\n\n # \"customize\" is in the process of going away here\n customize = {}\n if self.is_pypy:\n customize['PyPy'] = 0\n\n codelen = len(self.code)\n\n free, names, varnames = self.unmangle_code_names(co, classname)\n self.names = names\n\n # Scan for assertions. Later we will\n # turn 'LOAD_GLOBAL' to 'LOAD_ASSERT'.\n # 'LOAD_ASSERT' is used in assert statements.\n self.load_asserts = set()\n for i in self.op_range(0, codelen):\n\n # We need to detect the difference between:\n # raise AssertionError\n # and\n # assert ...\n # Below we use the heuristic that an \"sssert\" is preceded by a POP_JUMP.\n # however we could also use followed by RAISE_VARARGS\n # or for PyPy there may be a JUMP_IF_NOT_DEBUG before.\n # FIXME: remove uses of PJIF, and PJIT\n if self.is_pypy:\n have_pop_jump = self.code[i] in (self.opc.PJIF,\n self.opc.PJIT)\n else:\n have_pop_jump = self.code[i] == self.opc.PJIT\n\n if have_pop_jump and self.code[i+3] == self.opc.LOAD_GLOBAL:\n if names[self.get_argument(i+3)] == 'AssertionError':\n self.load_asserts.add(i+3)\n\n # Get jump targets\n # Format: {target offset: [jump offsets]}\n load_asserts_save = copy(self.load_asserts)\n jump_targets = self.find_jump_targets(show_asm)\n self.load_asserts = load_asserts_save\n # print(\"XXX2\", jump_targets)\n\n last_stmt = self.next_stmt[0]\n i = self.next_stmt[last_stmt]\n replace = {}\n while i < codelen - 1:\n if self.lines[last_stmt].next > i:\n # Distinguish \"print ...\" from \"print ...,\"\n if self.code[last_stmt] == self.opc.PRINT_ITEM:\n if self.code[i] == self.opc.PRINT_ITEM:\n replace[i] = 'PRINT_ITEM_CONT'\n elif self.code[i] == self.opc.PRINT_NEWLINE:\n replace[i] = 'PRINT_NEWLINE_CONT'\n last_stmt = i\n i = self.next_stmt[i]\n\n extended_arg = 0\n for offset in self.op_range(0, codelen):\n if offset in jump_targets:\n jump_idx = 0\n # We want to process COME_FROMs to the same offset to be in *descending*\n # offset order so we have the larger range or biggest instruction interval\n # last. (I think they are sorted in increasing order, but for safety\n # we sort them). That way, specific COME_FROM tags will match up\n # properly. For example, a \"loop\" with an \"if\" nested in it should have the\n # \"loop\" tag last so the grammar rule matches that properly.\n for jump_offset in sorted(jump_targets[offset], reverse=True):\n # if jump_offset == last_offset:\n # continue\n # last_offset = jump_offset\n come_from_name = 'COME_FROM'\n op_name = self.opname_for_offset(jump_offset)\n if op_name.startswith('SETUP_') and self.version == 2.7:\n come_from_type = op_name[len('SETUP_'):]\n if come_from_type not in ('LOOP', 'EXCEPT'):\n come_from_name = 'COME_FROM_%s' % come_from_type\n pass\n tokens.append(Token(\n come_from_name, jump_offset, repr(jump_offset),\n offset=\"%s_%d\" % (offset, jump_idx),\n has_arg = True))\n jump_idx += 1\n pass\n\n op = self.code[offset]\n op_name = self.op_name(op)\n\n oparg = None; pattr = None\n has_arg = op_has_argument(op, self.opc)\n if has_arg:\n oparg = self.get_argument(offset) + extended_arg\n extended_arg = 0\n if op == self.opc.EXTENDED_ARG:\n extended_arg += self.extended_arg_val(oparg)\n continue\n if op in self.opc.CONST_OPS:\n const = co.co_consts[oparg]\n if iscode(const):\n oparg = const\n if const.co_name == '':\n assert op_name == 'LOAD_CONST'\n op_name = 'LOAD_LAMBDA'\n elif const.co_name == '':\n op_name = 'LOAD_GENEXPR'\n elif const.co_name == '':\n op_name = 'LOAD_DICTCOMP'\n elif const.co_name == '':\n op_name = 'LOAD_SETCOMP'\n # verify() uses 'pattr' for comparison, since 'attr'\n # now holds Code(const) and thus can not be used\n # for comparison (todo: think about changing this)\n # pattr = 'code_object @ 0x%x %s->%s' %\\\n # (id(const), const.co_filename, const.co_name)\n pattr = ''\n else:\n if oparg < len(co.co_consts):\n argval, _ = _get_const_info(oparg, co.co_consts)\n # Why don't we use _ above for \"pattr\" rather than \"const\"?\n # This *is* a little hoaky, but we have to coordinate with\n # other parts like n_LOAD_CONST in pysource.py for example.\n pattr = const\n pass\n elif op in self.opc.NAME_OPS:\n pattr = names[oparg]\n elif op in self.opc.JREL_OPS:\n # use instead: hasattr(self, 'patch_continue'): ?\n if self.version == 2.7:\n self.patch_continue(tokens, offset, op)\n pattr = repr(offset + 3 + oparg)\n elif op in self.opc.JABS_OPS:\n # use instead: hasattr(self, 'patch_continue'): ?\n if self.version == 2.7:\n self.patch_continue(tokens, offset, op)\n pattr = repr(oparg)\n elif op in self.opc.LOCAL_OPS:\n pattr = varnames[oparg]\n elif op in self.opc.COMPARE_OPS:\n pattr = self.opc.cmp_op[oparg]\n elif op in self.opc.FREE_OPS:\n pattr = free[oparg]\n\n if op in self.varargs_ops:\n # CE - Hack for >= 2.5\n # Now all values loaded via LOAD_CLOSURE are packed into\n # a tuple before calling MAKE_CLOSURE.\n if op == self.opc.BUILD_TUPLE and \\\n self.code[self.prev[offset]] == self.opc.LOAD_CLOSURE:\n continue\n else:\n if self.is_pypy and not oparg and op_name == 'BUILD_MAP':\n op_name = 'BUILD_MAP_n'\n else:\n op_name = '%s_%d' % (op_name, oparg)\n pass\n # FIXME: Figure out why this is needed and remove.\n customize[op_name] = oparg\n elif op == self.opc.JUMP_ABSOLUTE:\n # Further classify JUMP_ABSOLUTE into backward jumps\n # which are used in loops, and \"CONTINUE\" jumps which\n # may appear in a \"continue\" statement. The loop-type\n # and continue-type jumps will help us classify loop\n # boundaries The continue-type jumps help us get\n # \"continue\" statements with would otherwise be turned\n # into a \"pass\" statement because JUMPs are sometimes\n # ignored in rules as just boundary overhead. In\n # comprehensions we might sometimes classify JUMP_BACK\n # as CONTINUE, but that's okay since we add a grammar\n # rule for that.\n target = self.get_target(offset)\n if target <= offset:\n op_name = 'JUMP_BACK'\n\n # 'Continue's include jumps to loops that are not\n # and the end of a block which follow with POP_BLOCK and COME_FROM_LOOP.\n # If the JUMP_ABSOLUTE is\n # either to a FOR_ITER or the instruction after a SETUP_LOOP\n # and it is followed by another JUMP_FORWARD\n # then we'll take it as a \"continue\".\n j = self.offset2inst_index[offset]\n target_index = self.offset2inst_index[target]\n is_continue = (self.insts[target_index-1].opname == 'SETUP_LOOP'\n and self.insts[j+1].opname == 'JUMP_FORWARD')\n if is_continue:\n op_name = 'CONTINUE'\n if (offset in self.stmts and\n self.code[offset+3] not in (self.opc.END_FINALLY,\n self.opc.POP_BLOCK)):\n if ((offset in self.linestarts and\n self.code[self.prev[offset]] == self.opc.JUMP_ABSOLUTE)\n or self.code[target] == self.opc.FOR_ITER\n or offset not in self.not_continue):\n op_name = 'CONTINUE'\n\n elif op == self.opc.LOAD_GLOBAL:\n if offset in self.load_asserts:\n op_name = 'LOAD_ASSERT'\n elif op == self.opc.RETURN_VALUE:\n if offset in self.return_end_ifs:\n op_name = 'RETURN_END_IF'\n\n linestart = self.linestarts.get(offset, None)\n\n if offset not in replace:\n tokens.append(Token(\n op_name, oparg, pattr, offset, linestart, op,\n has_arg, self.opc))\n else:\n tokens.append(Token(\n replace[offset], oparg, pattr, offset, linestart,\n op, has_arg, self.opc))\n pass\n pass\n\n if show_asm in ('both', 'after'):\n for t in tokens:\n print(t.format(line_prefix='L.'))\n print()\n return tokens, customize"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef next_except_jump(self, start):\n\n if self.code[start] == self.opc.DUP_TOP:\n except_match = self.first_instr(start, len(self.code), self.opc.PJIF)\n if except_match:\n jmp = self.prev[self.get_target(except_match)]\n\n # In Python < 2.7 we may have jumps to jumps\n if self.version < 2.7 and self.code[jmp] in self.jump_forward:\n self.not_continue.add(jmp)\n jmp = self.get_target(jmp)\n prev_offset = self.prev[except_match]\n # COMPARE_OP argument should be \"exception-match\" or 10\n if (self.code[prev_offset] == self.opc.COMPARE_OP and\n self.code[prev_offset+1] != 10):\n return None\n if jmp not in self.pop_jump_if | self.jump_forward:\n self.ignore_if.add(except_match)\n return None\n\n self.ignore_if.add(except_match)\n self.not_continue.add(jmp)\n return jmp\n\n count_END_FINALLY = 0\n count_SETUP_ = 0\n for i in self.op_range(start, len(self.code)):\n op = self.code[i]\n if op == self.opc.END_FINALLY:\n if count_END_FINALLY == count_SETUP_:\n if self.version == 2.7:\n assert self.code[self.prev[i]] in \\\n self.jump_forward | frozenset([self.opc.RETURN_VALUE])\n self.not_continue.add(self.prev[i])\n return self.prev[i]\n count_END_FINALLY += 1\n elif op in self.setup_ops:\n count_SETUP_ += 1", "response": "Return the next except - jump that was generated by an except - jump clause or None if no more jumps were generated."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ndetects the type of control flow for the current instruction and sets up the appropriate attributes for the loop instructions.", "response": "def detect_control_flow(self, offset, op, extended_arg):\n \"\"\"\n Detect type of block structures and their boundaries to fix optimized jumps\n in python2.3+\n \"\"\"\n\n code = self.code\n\n # Detect parent structure\n parent = self.structs[0]\n start = parent['start']\n end = parent['end']\n\n # Pick inner-most parent for our offset\n for struct in self.structs:\n current_start = struct['start']\n current_end = struct['end']\n if ((current_start <= offset < current_end)\n and (current_start >= start and current_end <= end)):\n start = current_start\n end = current_end\n parent = struct\n\n if op == self.opc.SETUP_LOOP:\n # We categorize loop types: 'for', 'while', 'while 1' with\n # possibly suffixes '-loop' and '-else'\n # Try to find the jump_back instruction of the loop.\n # It could be a return instruction.\n\n inst = self.insts[self.offset2inst_index[offset]]\n start += instruction_size(op, self.opc)\n setup_target = inst.argval\n loop_end_offset = self.restrict_to_parent(setup_target, parent)\n self.setup_loop_targets[offset] = setup_target\n self.setup_loops[setup_target] = offset\n\n if setup_target != loop_end_offset:\n self.fixed_jumps[offset] = loop_end_offset\n\n (line_no, next_line_byte) = self.lines[offset]\n\n # jump_back_offset is the instruction after the SETUP_LOOP\n # where we iterate back to.\n jump_back_offset = self.last_instr(start, loop_end_offset, self.opc.JUMP_ABSOLUTE,\n next_line_byte, False)\n\n if jump_back_offset:\n # Account for the fact that < 2.7 has an explicit\n # POP_TOP instruction in the equivalate POP_JUMP_IF\n # construct\n if self.version < 2.7:\n jump_forward_offset = jump_back_offset+4\n return_val_offset1 = self.prev[self.prev[self.prev[loop_end_offset]]]\n # Is jump back really \"back\"?\n jump_target = self.get_target(jump_back_offset, code[jump_back_offset])\n if (jump_target > jump_back_offset or\n code[jump_back_offset+3] in [self.opc.JUMP_FORWARD, self.opc.JUMP_ABSOLUTE]):\n jump_back_offset = None\n pass\n else:\n jump_forward_offset = jump_back_offset+3\n return_val_offset1 = self.prev[self.prev[loop_end_offset]]\n\n if (jump_back_offset and jump_back_offset != self.prev[loop_end_offset]\n and code[jump_forward_offset] in self.jump_forward):\n if (code[self.prev[loop_end_offset]] == self.opc.RETURN_VALUE or\n (code[self.prev[loop_end_offset]] == self.opc.POP_BLOCK\n and code[return_val_offset1] == self.opc.RETURN_VALUE)):\n jump_back_offset = None\n\n if not jump_back_offset:\n # loop suite ends in return\n # scanner26 of wbiti had:\n # jump_back_offset = self.last_instr(start, loop_end_offset, self.opc.JUMP_ABSOLUTE, start, False)\n jump_back_offset = self.last_instr(start, loop_end_offset, self.opc.RETURN_VALUE)\n if not jump_back_offset:\n return\n jump_back_offset += 1\n\n if_offset = None\n if self.version < 2.7:\n # Look for JUMP_IF POP_TOP ...\n if (code[self.prev[next_line_byte]] == self.opc.POP_TOP\n and (code[self.prev[self.prev[next_line_byte]]]\n in self.pop_jump_if)):\n if_offset = self.prev[self.prev[next_line_byte]]\n elif code[self.prev[next_line_byte]] in self.pop_jump_if:\n # Look for POP_JUMP_IF ...\n if_offset = self.prev[next_line_byte]\n if if_offset:\n loop_type = 'while'\n self.ignore_if.add(if_offset)\n if self.version < 2.7 and (\n code[self.prev[jump_back_offset]] == self.opc.RETURN_VALUE):\n self.ignore_if.add(self.prev[jump_back_offset])\n pass\n pass\n else:\n loop_type = 'for'\n setup_target = next_line_byte\n loop_end_offset = jump_back_offset + 3\n else:\n # We have a loop with a jump-back instruction\n if self.get_target(jump_back_offset) >= next_line_byte:\n jump_back_offset = self.last_instr(start, loop_end_offset, self.opc.JUMP_ABSOLUTE, start, False)\n if loop_end_offset > jump_back_offset+4 and code[loop_end_offset] in self.jump_forward:\n if code[jump_back_offset+4] in self.jump_forward:\n if self.get_target(jump_back_offset+4) == self.get_target(loop_end_offset):\n self.fixed_jumps[offset] = jump_back_offset+4\n loop_end_offset = jump_back_offset+4\n elif setup_target < offset:\n self.fixed_jumps[offset] = jump_back_offset+4\n loop_end_offset = jump_back_offset+4\n\n setup_target = self.get_target(jump_back_offset, self.opc.JUMP_ABSOLUTE)\n\n if (self.version > 2.1 and\n code[setup_target] in (self.opc.FOR_ITER, self.opc.GET_ITER)):\n loop_type = 'for'\n else:\n loop_type = 'while'\n # Look for a test condition immediately after the\n # SETUP_LOOP while\n if (self.version < 2.7\n and self.code[self.prev[next_line_byte]] == self.opc.POP_TOP):\n test_op_offset = self.prev[self.prev[next_line_byte]]\n else:\n test_op_offset = self.prev[next_line_byte]\n\n if test_op_offset == offset:\n loop_type = 'while 1'\n elif self.code[test_op_offset] in self.opc.JUMP_OPs:\n test_target = self.get_target(test_op_offset)\n\n self.ignore_if.add(test_op_offset)\n\n if test_target > (jump_back_offset+3):\n jump_back_offset = test_target\n self.not_continue.add(jump_back_offset)\n self.loops.append(setup_target)\n self.structs.append({'type': loop_type + '-loop',\n 'start': setup_target,\n 'end': jump_back_offset})\n if jump_back_offset+3 != loop_end_offset:\n self.structs.append({'type': loop_type + '-else',\n 'start': jump_back_offset+3,\n 'end': loop_end_offset})\n elif op == self.opc.SETUP_EXCEPT:\n start = offset + instruction_size(op, self.opc)\n target = self.get_target(offset, op)\n end_offset = self.restrict_to_parent(target, parent)\n if target != end_offset:\n self.fixed_jumps[offset] = end_offset\n # print target, end, parent\n # Add the try block\n self.structs.append({'type': 'try',\n 'start': start-3,\n 'end': end_offset-4})\n # Now isolate the except and else blocks\n end_else = start_else = self.get_target(self.prev[end_offset])\n\n\n end_finally_offset = end_offset\n setup_except_nest = 0\n while end_finally_offset < len(self.code):\n if self.code[end_finally_offset] == self.opc.END_FINALLY:\n if setup_except_nest == 0:\n break\n else:\n setup_except_nest -= 1\n elif self.code[end_finally_offset] == self.opc.SETUP_EXCEPT:\n setup_except_nest += 1\n end_finally_offset += instruction_size(code[end_finally_offset], self.opc)\n pass\n\n # Add the except blocks\n i = end_offset\n while i < len(self.code) and i < end_finally_offset:\n jmp = self.next_except_jump(i)\n if jmp is None: # check\n i = self.next_stmt[i]\n continue\n if self.code[jmp] == self.opc.RETURN_VALUE:\n self.structs.append({'type': 'except',\n 'start': i,\n 'end': jmp+1})\n i = jmp + 1\n else:\n target = self.get_target(jmp)\n if target != start_else:\n end_else = self.get_target(jmp)\n if self.code[jmp] == self.opc.JUMP_FORWARD:\n if self.version <= 2.6:\n self.fixed_jumps[jmp] = target\n else:\n self.fixed_jumps[jmp] = -1\n self.structs.append({'type': 'except',\n 'start': i,\n 'end': jmp})\n i = jmp + 3\n\n # Add the try-else block\n if end_else != start_else:\n r_end_else = self.restrict_to_parent(end_else, parent)\n # May be able to drop the 2.7 test.\n if self.version == 2.7:\n self.structs.append({'type': 'try-else',\n 'start': i+1,\n 'end': r_end_else})\n self.fixed_jumps[i] = r_end_else\n else:\n self.fixed_jumps[i] = i+1\n\n elif op in self.pop_jump_if:\n target = self.get_target(offset, op)\n rtarget = self.restrict_to_parent(target, parent)\n\n # Do not let jump to go out of parent struct bounds\n if target != rtarget and parent['type'] == 'and/or':\n self.fixed_jumps[offset] = rtarget\n return\n\n jump_if_offset = offset\n\n start = offset+3\n pre = self.prev\n\n # Does this jump to right after another conditional jump that is\n # not myself? If so, it's part of a larger conditional.\n # rocky: if we have a conditional jump to the next instruction, then\n # possibly I am \"skipping over\" a \"pass\" or null statement.\n\n test_target = target\n if self.version < 2.7:\n # Before 2.7 we have to deal with the fact that there is an extra\n # POP_TOP that is logically associated with the JUMP_IF's (even though\n # the instance set is called \"self.pop_jump_if\")\n if code[pre[test_target]] == self.opc.POP_TOP:\n test_target = pre[test_target]\n test_set = self.pop_jump_if\n else:\n test_set = self.pop_jump_if_or_pop | self.pop_jump_if\n\n if ( code[pre[test_target]] in test_set and target > offset ):\n # We have POP_JUMP_IF... target\n # ...\n # pre: POP_JUMP_IF ...\n # target: ...\n #\n # We will take that as either as \"and\" or \"or\".\n self.fixed_jumps[offset] = pre[target]\n self.structs.append({'type': 'and/or',\n 'start': start,\n 'end': pre[target]})\n return\n\n # The instruction offset just before the target jump offset is important\n # in making a determination of what we have. Save that.\n pre_rtarget = pre[rtarget]\n\n # Is it an \"and\" inside an \"if\" or \"while\" block\n if op == self.opc.PJIF:\n\n # Search for other POP_JUMP_IF_...'s targetting the\n # same target, of the current POP_JUMP_... instruction,\n # starting from current offset, and filter everything inside inner 'or'\n # jumps and mid-line ifs\n match = self.rem_or(start, self.next_stmt[offset], self.opc.PJIF, target)\n\n # If we still have any offsets in set, start working on it\n if match:\n if code[pre_rtarget] in self.jump_forward \\\n and pre_rtarget not in self.stmts \\\n and self.restrict_to_parent(self.get_target(pre_rtarget), parent) == rtarget:\n if code[pre[pre_rtarget]] == self.opc.JUMP_ABSOLUTE \\\n and self.remove_mid_line_ifs([offset]) \\\n and target == self.get_target(pre[pre_rtarget]) \\\n and (pre[pre_rtarget] not in self.stmts or self.get_target(pre[pre_rtarget]) > pre[pre_rtarget])\\\n and 1 == len(self.remove_mid_line_ifs(self.rem_or(start, pre[pre_rtarget], self.pop_jump_if, target))):\n pass\n elif code[pre[pre_rtarget]] == self.opc.RETURN_VALUE \\\n and self.remove_mid_line_ifs([offset]) \\\n and 1 == (len(set(self.remove_mid_line_ifs(self.rem_or(start,\n pre[pre_rtarget],\n self.pop_jump_if, target)))\n | set(self.remove_mid_line_ifs(self.rem_or(start, pre[pre_rtarget],\n (self.opc.PJIF, self.opc.PJIT, self.opc.JUMP_ABSOLUTE), pre_rtarget, True))))):\n pass\n else:\n fix = None\n jump_ifs = self.all_instr(start, self.next_stmt[offset], self.opc.PJIF)\n last_jump_good = True\n for j in jump_ifs:\n if target == self.get_target(j):\n if self.lines[j].next == j+3 and last_jump_good:\n fix = j\n break\n else:\n last_jump_good = False\n self.fixed_jumps[offset] = fix or match[-1]\n return\n else:\n if (self.version < 2.7\n and parent['type'] in ('root', 'for-loop', 'if-then',\n 'else', 'try')):\n self.fixed_jumps[offset] = rtarget\n else:\n # note test for < 2.7 might be superflous although informative\n # for 2.7 a different branch is taken and the below code is handled\n # under: elif op in self.pop_jump_if_or_pop\n # below\n self.fixed_jumps[offset] = match[-1]\n return\n else: # op != self.opc.PJIT\n if self.version < 2.7 and code[offset+3] == self.opc.POP_TOP:\n assert_offset = offset + 4\n else:\n assert_offset = offset + 3\n if (assert_offset) in self.load_asserts:\n if code[pre_rtarget] == self.opc.RAISE_VARARGS:\n return\n self.load_asserts.remove(assert_offset)\n\n next = self.next_stmt[offset]\n if pre[next] == offset:\n pass\n elif code[next] in self.jump_forward and target == self.get_target(next):\n if code[pre[next]] == self.opc.PJIF:\n if code[next] == self.opc.JUMP_FORWARD or target != rtarget or code[pre[pre_rtarget]] not in (self.opc.JUMP_ABSOLUTE, self.opc.RETURN_VALUE):\n self.fixed_jumps[offset] = pre[next]\n return\n elif code[next] == self.opc.JUMP_ABSOLUTE and code[target] in self.jump_forward:\n next_target = self.get_target(next)\n if self.get_target(target) == next_target:\n self.fixed_jumps[offset] = pre[next]\n return\n elif code[next_target] in self.jump_forward and self.get_target(next_target) == self.get_target(target):\n self.fixed_jumps[offset] = pre[next]\n return\n\n # don't add a struct for a while test, it's already taken care of\n if offset in self.ignore_if:\n return\n\n if self.version == 2.7:\n if code[pre_rtarget] == self.opc.JUMP_ABSOLUTE and pre_rtarget in self.stmts \\\n and pre_rtarget != offset and pre[pre_rtarget] != offset:\n if code[rtarget] == self.opc.JUMP_ABSOLUTE and code[rtarget+3] == self.opc.POP_BLOCK:\n if code[pre[pre_rtarget]] != self.opc.JUMP_ABSOLUTE:\n pass\n elif self.get_target(pre[pre_rtarget]) != target:\n pass\n else:\n rtarget = pre_rtarget\n else:\n rtarget = pre_rtarget\n pre_rtarget = pre[rtarget]\n\n # Does the \"jump if\" jump beyond a jump op?\n # That is, we have something like:\n # POP_JUMP_IF_FALSE HERE\n # ...\n # JUMP_FORWARD\n # HERE:\n #\n # If so, this can be a block inside an \"if\" statement\n # or a conditional assignment like:\n # x = 1 if x else 2\n #\n # There are other situations we may need to consider, like\n # if the condition jump is to a forward location.\n # Also the existence of a jump to the instruction after \"END_FINALLY\"\n # will distinguish \"try/else\" from \"try\".\n code_pre_rtarget = code[pre_rtarget]\n\n if code_pre_rtarget in self.jump_forward:\n if_end = self.get_target(pre_rtarget)\n\n # Is this a loop and not an \"if\" statment?\n if (if_end < pre_rtarget) and (pre[if_end] in self.setup_loop_targets):\n\n if (if_end > start):\n return\n else:\n # We still have the case in 2.7 that the next instruction\n # is a jump to a SETUP_LOOP target.\n next_offset = target + instruction_size(self.code[target], self.opc)\n next_op = self.code[next_offset]\n if self.op_name(next_op) == 'JUMP_FORWARD':\n jump_target = self.get_target(next_offset, next_op)\n if jump_target in self.setup_loops:\n self.structs.append({'type': 'while-loop',\n 'start': jump_if_offset,\n 'end': jump_target})\n self.fixed_jumps[jump_if_offset] = jump_target\n return\n\n end_offset = self.restrict_to_parent(if_end, parent)\n\n if_then_maybe = None\n\n if 2.2 <= self.version <= 2.6:\n # Take the JUMP_IF target. In an \"if/then\", it will be\n # a POP_TOP instruction and the instruction before it\n # will be a JUMP_FORWARD to just after the POP_TOP.\n # For example:\n # Good:\n # 3 JUMP_IF_FALSE 33 'to 39'\n # ..\n # 36 JUMP_FORWARD 1 'to 40'\n # 39 POP_TOP\n # 40 ...\n # example:\n\n # BAD (is an \"and\"):\n # 28 JUMP_IF_FALSE 4 'to 35'\n # ...\n # 32 JUMP_ABSOLUTE 40 'to 40' # should be 36 or there should\n # # be a COME_FROM at the pop top\n # # before 40 to 35\n # 35 POP_TOP\n # 36 ...\n # 39 POP_TOP\n # 39_0 COME_FROM 3\n # 40 ...\n\n if self.opname_for_offset(jump_if_offset).startswith('JUMP_IF'):\n jump_if_target = code[jump_if_offset+1]\n if self.opname_for_offset(jump_if_target + jump_if_offset + 3) == 'POP_TOP':\n jump_inst = jump_if_target + jump_if_offset\n jump_offset = code[jump_inst+1]\n jump_op = self.opname_for_offset(jump_inst)\n if (jump_op == 'JUMP_FORWARD' and jump_offset == 1):\n self.structs.append({'type': 'if-then',\n 'start': start-3,\n 'end': pre_rtarget})\n self.thens[start] = end_offset\n elif jump_op == 'JUMP_ABSOLUTE':\n if_then_maybe = {'type': 'if-then',\n 'start': start-3,\n 'end': pre_rtarget}\n\n elif self.version == 2.7:\n self.structs.append({'type': 'if-then',\n 'start': start-3,\n 'end': pre_rtarget})\n\n # FIXME: this is yet another case were we need dominators.\n if (pre_rtarget not in self.linestarts\n or self.version < 2.7):\n self.not_continue.add(pre_rtarget)\n\n if rtarget < end_offset:\n # We have an \"else\" block of some kind.\n # Is it associated with \"if_then_maybe\" seen above?\n # These will be linked in this funny way:\n\n # 198 JUMP_IF_FALSE 18 'to 219'\n # 201 POP_TOP\n # ...\n # 216 JUMP_ABSOLUTE 256 'to 256'\n # 219 POP_TOP\n # ...\n # 252 JUMP_FORWARD 1 'to 256'\n # 255 POP_TOP\n # 256\n if if_then_maybe and jump_op == 'JUMP_ABSOLUTE':\n jump_target = self.get_target(jump_inst, code[jump_inst])\n if self.opname_for_offset(end_offset) == 'JUMP_FORWARD':\n end_target = self.get_target(end_offset, code[end_offset])\n if jump_target == end_target:\n self.structs.append(if_then_maybe)\n self.thens[start] = end_offset\n\n self.structs.append({'type': 'else',\n 'start': rtarget,\n 'end': end_offset})\n elif code_pre_rtarget == self.opc.RETURN_VALUE:\n if self.version == 2.7 or pre_rtarget not in self.ignore_if:\n # Below, 10 is exception-match. If there is an exception\n # match in the compare, then this is an exception\n # clause not an if-then clause\n if (self.code[self.prev[offset]] != self.opc.COMPARE_OP or\n self.code[self.prev[offset]+1] != 10):\n self.structs.append({'type': 'if-then',\n 'start': start,\n 'end': rtarget})\n self.thens[start] = rtarget\n if (self.version == 2.7 or\n code[pre_rtarget+1] != self.opc.JUMP_FORWARD):\n # The below is a big hack until we get\n # better control flow analysis: disallow\n # END_IF if the instruction before the\n # END_IF instruction happens to be a jump\n # target. In this case, probably what's\n # gone on is that we messed up on the\n # END_IF location and it should be the\n # instruction before.\n self.fixed_jumps[offset] = rtarget\n if (self.version == 2.7 and\n self.insts[self.offset2inst_index[pre[pre_rtarget]]].is_jump_target):\n self.return_end_ifs.add(pre[pre_rtarget])\n pass\n else:\n self.return_end_ifs.add(pre_rtarget)\n pass\n pass\n pass\n\n elif op in self.pop_jump_if_or_pop:\n target = self.get_target(offset, op)\n self.fixed_jumps[offset] = self.restrict_to_parent(target, parent)"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef find_jump_targets(self, debug):\n code = self.code\n n = len(code)\n self.structs = [{'type': 'root',\n 'start': 0,\n 'end': n-1}]\n # All loop entry points\n self.loops = []\n\n # Map fixed jumps to their real destination\n self.fixed_jumps = {}\n self.ignore_if = set()\n self.build_statement_indices()\n\n # Containers filled by detect_control_flow()\n self.not_continue = set()\n self.return_end_ifs = set()\n self.setup_loop_targets = {} # target given setup_loop offset\n self.setup_loops = {} # setup_loop offset given target\n self.thens = {} # JUMP_IF's that separate the 'then' part of an 'if'\n\n targets = {}\n extended_arg = 0\n for offset in self.op_range(0, n):\n op = code[offset]\n\n if op == self.opc.EXTENDED_ARG:\n arg = code2num(code, offset+1) | extended_arg\n extended_arg += self.extended_arg_val(arg)\n continue\n\n # Determine structures and fix jumps in Python versions\n # since 2.3\n self.detect_control_flow(offset, op, extended_arg)\n\n if op_has_argument(op, self.opc):\n label = self.fixed_jumps.get(offset)\n oparg = self.get_argument(offset)\n\n if label is None:\n if op in self.opc.JREL_OPS and self.op_name(op) != 'FOR_ITER':\n # if (op in self.opc.JREL_OPS and\n # (self.version < 2.0 or op != self.opc.FOR_ITER)):\n label = offset + 3 + oparg\n elif self.version == 2.7 and op in self.opc.JABS_OPS:\n if op in (self.opc.JUMP_IF_FALSE_OR_POP,\n self.opc.JUMP_IF_TRUE_OR_POP):\n if (oparg > offset):\n label = oparg\n pass\n pass\n\n # FIXME FIXME FIXME\n # All the conditions are horrible, and I am not sure I\n # undestand fully what's going l\n # We REALLY REALLY need a better way to handle control flow\n # Expecially for < 2.7\n if label is not None and label != -1:\n if self.version == 2.7:\n # FIXME: rocky: I think we need something like this...\n if label in self.setup_loops:\n source = self.setup_loops[label]\n else:\n source = offset\n targets[label] = targets.get(label, []) + [source]\n elif not (code[label] == self.opc.POP_TOP and\n code[self.prev[label]] == self.opc.RETURN_VALUE):\n # In Python < 2.7, don't add a COME_FROM, for:\n # ~RETURN_VALUE POP_TOP .. END_FINALLY\n # or:\n # ~RETURN_VALUE POP_TOP .. POP_TOP END_FINALLY\n skip_come_from = (code[offset+3] == self.opc.END_FINALLY or\n (code[offset+3] == self.opc.POP_TOP\n and code[offset+4] == self.opc.END_FINALLY))\n\n # The below is for special try/else handling\n if skip_come_from and op == self.opc.JUMP_FORWARD:\n skip_come_from = False\n\n if not skip_come_from:\n # FIXME: rocky: I think we need something like this...\n if offset not in set(self.ignore_if):\n if label in self.setup_loops:\n source = self.setup_loops[label]\n else:\n source = offset\n # FIXME: The grammar for 2.6 and before doesn't\n # handle COME_FROM's from a loop inside if's\n # It probably should.\n if (self.version > 2.6 or\n self.code[source] != self.opc.SETUP_LOOP or\n self.code[label] != self.opc.JUMP_FORWARD):\n targets[label] = targets.get(label, []) + [source]\n pass\n pass\n pass\n pass\n elif op == self.opc.END_FINALLY and offset in self.fixed_jumps and self.version == 2.7:\n label = self.fixed_jumps[offset]\n targets[label] = targets.get(label, []) + [offset]\n pass\n\n extended_arg = 0\n pass # for loop\n\n # DEBUG:\n if debug in ('both', 'after'):\n print(targets)\n import pprint as pp\n pp.pprint(self.structs)\n\n return targets", "response": "Find all jump targets in a byte code."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef rem_or(self, start, end, instr, target=None, include_beyond_target=False):\n\n assert(start >= 0 and end <= len(self.code) and start <= end)\n\n try: None in instr\n except: instr = [instr]\n\n instr_offsets = []\n for i in self.op_range(start, end):\n op = self.code[i]\n if op in instr:\n if target is None:\n instr_offsets.append(i)\n else:\n t = self.get_target(i, op)\n if include_beyond_target and t >= target:\n instr_offsets.append(i)\n elif t == target:\n instr_offsets.append(i)\n\n pjits = self.all_instr(start, end, self.opc.PJIT)\n filtered = []\n for pjit in pjits:\n tgt = self.get_target(pjit)-3\n for i in instr_offsets:\n if i <= pjit or i >= tgt:\n filtered.append(i)\n instr_offsets = filtered\n filtered = []\n return instr_offsets", "response": "Return a list of all instruction offsets that are in the block from start to end."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef call_fn_name(token):\n if token.attr is not None:\n return '%s_%i' % (token.kind, token.attr)\n else:\n return '%s_0' % (token.kind)", "response": "Customize CALL_FUNCTION to add the number of positional arguments"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ncustomizes build_class rule for LOAD_CLASSNAME LOAD_CONST and LOAD_FUNCTION.", "response": "def custom_build_class_rule(self, opname, i, token, tokens, customize):\n '''\n # Should the first rule be somehow folded into the 2nd one?\n build_class ::= LOAD_BUILD_CLASS mkfunc\n LOAD_CLASSNAME {expr}^n-1 CALL_FUNCTION_n\n LOAD_CONST CALL_FUNCTION_n\n build_class ::= LOAD_BUILD_CLASS mkfunc\n expr\n call\n CALL_FUNCTION_3\n '''\n # FIXME: I bet this can be simplified\n # look for next MAKE_FUNCTION\n for i in range(i+1, len(tokens)):\n if tokens[i].kind.startswith('MAKE_FUNCTION'):\n break\n elif tokens[i].kind.startswith('MAKE_CLOSURE'):\n break\n pass\n assert i < len(tokens), \"build_class needs to find MAKE_FUNCTION or MAKE_CLOSURE\"\n assert tokens[i+1].kind == 'LOAD_CONST', \\\n \"build_class expecting CONST after MAKE_FUNCTION/MAKE_CLOSURE\"\n call_fn_tok = None\n for i in range(i, len(tokens)):\n if tokens[i].kind.startswith('CALL_FUNCTION'):\n call_fn_tok = tokens[i]\n break\n if not call_fn_tok:\n raise RuntimeError(\"build_class custom rule for %s needs to find CALL_FUNCTION\"\n % opname)\n\n\n # customize build_class rule\n # FIXME: What's the deal with the two rules? Different Python versions?\n # Different situations? Note that the above rule is based on the CALL_FUNCTION\n # token found, while this one doesn't.\n if self.version < 3.6:\n call_function = self.call_fn_name(call_fn_tok)\n args_pos, args_kw = self.get_pos_kw(call_fn_tok)\n rule = (\"build_class ::= LOAD_BUILD_CLASS mkfunc %s\"\n \"%s\" % (('expr ' * (args_pos - 1) + ('kwarg ' * args_kw)),\n call_function))\n else:\n # 3.6+ handling\n call_function = call_fn_tok.kind\n if call_function.startswith(\"CALL_FUNCTION_KW\"):\n self.addRule(\"classdef ::= build_class_kw store\", nop_func)\n rule = (\"build_class_kw ::= LOAD_BUILD_CLASS mkfunc %sLOAD_CONST %s\"\n % ('expr ' * (call_fn_tok.attr - 1), call_function))\n else:\n call_function = self.call_fn_name(call_fn_tok)\n rule = (\"build_class ::= LOAD_BUILD_CLASS mkfunc %s%s\"\n % ('expr ' * (call_fn_tok.attr - 1), call_function))\n self.addRule(rule, nop_func)\n return"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef custom_classfunc_rule(self, opname, token, customize, next_token):\n args_pos, args_kw = self.get_pos_kw(token)\n\n # Additional exprs for * and ** args:\n # 0 if neither\n # 1 for CALL_FUNCTION_VAR or CALL_FUNCTION_KW\n # 2 for * and ** args (CALL_FUNCTION_VAR_KW).\n # Yes, this computation based on instruction name is a little bit hoaky.\n nak = ( len(opname)-len('CALL_FUNCTION') ) // 3\n\n token.kind = self.call_fn_name(token)\n uniq_param = args_kw + args_pos\n\n # Note: 3.5+ have subclassed this method; so we don't handle\n # 'CALL_FUNCTION_VAR' or 'CALL_FUNCTION_EX' here.\n rule = ('call ::= expr ' +\n ('pos_arg ' * args_pos) +\n ('kwarg ' * args_kw) +\n 'expr ' * nak + token.kind)\n\n self.add_unique_rule(rule, token.kind, uniq_param, customize)\n\n if 'LOAD_BUILD_CLASS' in self.seen_ops:\n if (next_token == 'CALL_FUNCTION' and next_token.attr == 1\n and args_pos > 1):\n rule = ('classdefdeco2 ::= LOAD_BUILD_CLASS mkfunc %s%s_%d'\n % (('expr ' * (args_pos-1)), opname, args_pos))\n self.add_unique_rule(rule, token.kind, uniq_param, customize)", "response": "Customize the classfunc rule for the current class."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef add_make_function_rule(self, rule, opname, attr, customize):\n if self.version >= 3.3:\n new_rule = rule % (('LOAD_CONST ') * 1)\n else:\n new_rule = rule % (('LOAD_CONST ') * 0)\n self.add_unique_rule(new_rule, opname, attr, customize)", "response": "Add a MAKE_FUNCTION rule to the internal rule list."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef customize_grammar_rules(self, tokens, customize):\n\n is_pypy = False\n\n # For a rough break out on the first word. This may\n # include instructions that don't need customization,\n # but we'll do a finer check after the rough breakout.\n customize_instruction_basenames = frozenset(\n ('BUILD', 'CALL', 'CONTINUE', 'DELETE', 'GET',\n 'JUMP', 'LOAD', 'LOOKUP', 'MAKE',\n 'RETURN', 'RAISE', 'SETUP',\n 'UNPACK'))\n\n # Opcode names in the custom_ops_processed set have rules that get added\n # unconditionally and the rules are constant. So they need to be done\n # only once and if we see the opcode a second we don't have to consider\n # adding more rules.\n #\n # Note: BUILD_TUPLE_UNPACK_WITH_CALL gets considered by\n # default because it starts with BUILD. So we'll set to ignore it from\n # the start.\n custom_ops_processed = set(('BUILD_TUPLE_UNPACK_WITH_CALL',))\n\n # A set of instruction operation names that exist in the token stream.\n # We use this customize the grammar that we create.\n # 2.6-compatible set comprehensions\n self.seen_ops = frozenset([t.kind for t in tokens])\n self.seen_op_basenames = frozenset([opname[:opname.rfind('_')] for opname in self.seen_ops])\n\n # Loop over instructions adding custom grammar rules based on\n # a specific instruction seen.\n\n if 'PyPy' in customize:\n is_pypy = True\n self.addRule(\"\"\"\n stmt ::= assign3_pypy\n stmt ::= assign2_pypy\n assign3_pypy ::= expr expr expr store store store\n assign2_pypy ::= expr expr store store\n stmt ::= conditional_lambda\n stmt ::= conditional_not_lambda\n conditional_lambda ::= expr jmp_false expr return_if_lambda\n return_lambda LAMBDA_MARKER\n conditional_not_lambda\n ::= expr jmp_true expr return_if_lambda\n return_lambda LAMBDA_MARKER\n \"\"\", nop_func)\n\n n = len(tokens)\n\n # Determine if we have an iteration CALL_FUNCTION_1.\n has_get_iter_call_function1 = False\n max_branches = 0\n for i, token in enumerate(tokens):\n if token == 'GET_ITER' and i < n-2 and self.call_fn_name(tokens[i+1]) == 'CALL_FUNCTION_1':\n has_get_iter_call_function1 = True\n max_branches += 1\n elif (token == 'GET_AWAITABLE' and i < n-3\n and tokens[i+1] == 'LOAD_CONST' and tokens[i+2] == 'YIELD_FROM'):\n max_branches += 1\n if max_branches > 2:\n break\n\n for i, token in enumerate(tokens):\n opname = token.kind\n\n # Do a quick breakout before testing potentially\n # each of the dozen or so instruction in if elif.\n if (opname[:opname.find('_')] not in customize_instruction_basenames\n or opname in custom_ops_processed):\n continue\n\n opname_base = opname[:opname.rfind('_')]\n # The order of opname listed is roughly sorted below\n if opname_base == 'BUILD_CONST_KEY_MAP':\n # This is in 3.6+\n kvlist_n = 'expr ' * (token.attr)\n rule = \"dict ::= %sLOAD_CONST %s\" % (kvlist_n, opname)\n self.addRule(rule, nop_func)\n elif opname.startswith('BUILD_LIST_UNPACK'):\n v = token.attr\n rule = 'build_list_unpack ::= %s%s' % ('expr ' * v, opname)\n self.addRule(rule, nop_func)\n rule = 'expr ::= build_list_unpack'\n self.addRule(rule, nop_func)\n elif opname_base in ('BUILD_MAP', 'BUILD_MAP_UNPACK'):\n kvlist_n = \"kvlist_%s\" % token.attr\n if opname == 'BUILD_MAP_n':\n # PyPy sometimes has no count. Sigh.\n rule = ('dict_comp_func ::= BUILD_MAP_n LOAD_FAST FOR_ITER store '\n 'comp_iter JUMP_BACK RETURN_VALUE RETURN_LAST')\n self.add_unique_rule(rule, 'dict_comp_func', 1, customize)\n\n kvlist_n = 'kvlist_n'\n rule = 'kvlist_n ::= kvlist_n kv3'\n self.add_unique_rule(rule, 'kvlist_n', 0, customize)\n rule = 'kvlist_n ::='\n self.add_unique_rule(rule, 'kvlist_n', 1, customize)\n rule = \"dict ::= BUILD_MAP_n kvlist_n\"\n elif self.version >= 3.5:\n if not opname.startswith('BUILD_MAP_WITH_CALL'):\n # FIXME: Use the attr\n # so this doesn't run into exponential parsing time.\n if opname.startswith('BUILD_MAP_UNPACK'):\n self.add_unique_rule(rule, opname, token.attr, customize)\n rule = 'dict_entry ::= ' + 'expr ' * (token.attr*2)\n self.add_unique_rule(rule, opname, token.attr, customize)\n\n # FIXME: start here. The LHS should be unmap_dict, not dict.\n # FIXME: really we need a combination of dict_entry-like things.\n # It just so happens the most common case is not to mix\n # dictionary comphensions with dictionary, elements\n if 'LOAD_DICTCOMP' in self.seen_ops:\n rule = 'dict ::= %s%s' % ('dict_comp ' * token.attr, opname)\n self.addRule(rule, nop_func)\n rule = \"\"\"\n expr ::= unmap_dict\n unmap_dict ::= %s%s\n \"\"\" % ('expr ' * token.attr, opname)\n else:\n rule = \"%s ::= %s %s\" % (kvlist_n, 'expr ' * (token.attr*2), opname)\n self.add_unique_rule(rule, opname, token.attr, customize)\n rule = \"dict ::= %s\" % kvlist_n\n else:\n rule = kvlist_n + ' ::= ' + 'expr expr STORE_MAP ' * token.attr\n self.add_unique_rule(rule, opname, token.attr, customize)\n rule = \"dict ::= %s %s\" % (opname, kvlist_n)\n self.add_unique_rule(rule, opname, token.attr, customize)\n elif opname.startswith('BUILD_MAP_UNPACK_WITH_CALL'):\n v = token.attr\n rule = 'build_map_unpack_with_call ::= %s%s' % ('expr ' * v, opname)\n self.addRule(rule, nop_func)\n elif opname.startswith('BUILD_TUPLE_UNPACK_WITH_CALL'):\n v = token.attr\n rule = ('starred ::= %s %s' % ('expr ' * v, opname))\n self.addRule(rule, nop_func)\n\n elif opname_base in ('BUILD_LIST', 'BUILD_SET', 'BUILD_TUPLE',\n 'BUILD_TUPLE_UNPACK'):\n v = token.attr\n\n is_LOAD_CLOSURE = False\n if opname_base == 'BUILD_TUPLE':\n # If is part of a \"load_closure\", then it is not part of a\n # \"list\".\n is_LOAD_CLOSURE = True\n for j in range(v):\n if tokens[i-j-1].kind != 'LOAD_CLOSURE':\n is_LOAD_CLOSURE = False\n break\n if is_LOAD_CLOSURE:\n rule = ('load_closure ::= %s%s' % (('LOAD_CLOSURE ' * v), opname))\n self.add_unique_rule(rule, opname, token.attr, customize)\n if not is_LOAD_CLOSURE or v == 0:\n # We do this complicated test to speed up parsing of\n # pathelogically long literals, especially those over 1024.\n build_count = token.attr\n thousands = (build_count//1024)\n thirty32s = ((build_count//32) % 32)\n if thirty32s > 0:\n rule = \"expr32 ::=%s\" % (' expr' * 32)\n self.add_unique_rule(rule, opname_base, build_count, customize)\n pass\n if thousands > 0:\n self.add_unique_rule(\"expr1024 ::=%s\" % (' expr32' * 32),\n opname_base, build_count, customize)\n pass\n collection = opname_base[opname_base.find('_')+1:].lower()\n rule = (('%s ::= ' % collection) + 'expr1024 '*thousands +\n 'expr32 '*thirty32s + 'expr '*(build_count % 32) + opname)\n self.add_unique_rules([\n \"expr ::= %s\" % collection,\n rule], customize)\n continue\n continue\n elif opname_base == 'BUILD_SLICE':\n if token.attr == 2:\n self.add_unique_rules([\n 'expr ::= build_slice2',\n 'build_slice2 ::= expr expr BUILD_SLICE_2'\n ], customize)\n else:\n assert token.attr == 3, \"BUILD_SLICE value must be 2 or 3; is %s\" % v\n self.add_unique_rules([\n 'expr ::= build_slice3',\n 'build_slice3 ::= expr expr expr BUILD_SLICE_3',\n ], customize)\n elif (opname in frozenset(('CALL_FUNCTION',\n 'CALL_FUNCTION_EX',\n 'CALL_FUNCTION_EX_KW',\n 'CALL_FUNCTION_VAR',\n 'CALL_FUNCTION_VAR_KW'))\n or opname.startswith('CALL_FUNCTION_KW')):\n\n if opname == 'CALL_FUNCTION' and token.attr == 1:\n rule = \"\"\"\n dict_comp ::= LOAD_DICTCOMP LOAD_CONST MAKE_FUNCTION_0 expr\n GET_ITER CALL_FUNCTION_1\n classdefdeco1 ::= expr classdefdeco2 CALL_FUNCTION_1\n \"\"\"\n if self.version < 3.5:\n rule += \"\"\"\n classdefdeco1 ::= expr classdefdeco1 CALL_FUNCTION_1\n \"\"\"\n self.addRule(rule, nop_func)\n\n self.custom_classfunc_rule(opname, token, customize, tokens[i+1])\n # Note: don't add to custom_ops_processed.\n\n elif opname_base == 'CALL_METHOD':\n # PyPy only - DRY with parse2\n\n args_pos, args_kw = self.get_pos_kw(token)\n\n # number of apply equiv arguments:\n nak = ( len(opname_base)-len('CALL_METHOD') ) // 3\n rule = ('call ::= expr ' +\n ('pos_arg ' * args_pos) +\n ('kwarg ' * args_kw) +\n 'expr ' * nak + opname)\n self.add_unique_rule(rule, opname, token.attr, customize)\n elif opname == 'CONTINUE':\n self.addRule('continue ::= CONTINUE', nop_func)\n custom_ops_processed.add(opname)\n elif opname == 'CONTINUE_LOOP':\n self.addRule('continue ::= CONTINUE_LOOP', nop_func)\n custom_ops_processed.add(opname)\n elif opname == 'DELETE_ATTR':\n self.addRule('del_stmt ::= expr DELETE_ATTR', nop_func)\n custom_ops_processed.add(opname)\n elif opname == 'DELETE_DEREF':\n self.addRule(\"\"\"\n stmt ::= del_deref_stmt\n del_deref_stmt ::= DELETE_DEREF\n \"\"\", nop_func)\n custom_ops_processed.add(opname)\n elif opname == 'DELETE_SUBSCR':\n self.addRule(\"\"\"\n del_stmt ::= delete_subscr\n delete_subscr ::= expr expr DELETE_SUBSCR\n \"\"\", nop_func)\n custom_ops_processed.add(opname)\n elif opname == 'GET_ITER':\n self.addRule(\"\"\"\n expr ::= get_iter\n attribute ::= expr GET_ITER\n \"\"\", nop_func)\n custom_ops_processed.add(opname)\n elif opname == 'JUMP_IF_NOT_DEBUG':\n v = token.attr\n self.addRule(\"\"\"\n stmt ::= assert_pypy\n stmt ::= assert2_pypy\", nop_func)\n assert_pypy ::= JUMP_IF_NOT_DEBUG assert_expr jmp_true\n LOAD_ASSERT RAISE_VARARGS_1 COME_FROM\n assert2_pypy ::= JUMP_IF_NOT_DEBUG assert_expr jmp_true\n LOAD_ASSERT expr CALL_FUNCTION_1\n RAISE_VARARGS_1 COME_FROM\n assert2_pypy ::= JUMP_IF_NOT_DEBUG assert_expr jmp_true\n LOAD_ASSERT expr CALL_FUNCTION_1\n RAISE_VARARGS_1 COME_FROM,\n \"\"\", nop_func)\n custom_ops_processed.add(opname)\n elif opname == 'LOAD_BUILD_CLASS':\n self.custom_build_class_rule(opname, i, token, tokens, customize)\n # Note: don't add to custom_ops_processed.\n elif opname == 'LOAD_CLASSDEREF':\n # Python 3.4+\n self.addRule(\"expr ::= LOAD_CLASSDEREF\", nop_func)\n custom_ops_processed.add(opname)\n elif opname == 'LOAD_CLASSNAME':\n self.addRule(\"expr ::= LOAD_CLASSNAME\", nop_func)\n custom_ops_processed.add(opname)\n elif opname == 'LOAD_DICTCOMP':\n if has_get_iter_call_function1:\n rule_pat = (\"dict_comp ::= LOAD_DICTCOMP %sMAKE_FUNCTION_0 expr \"\n \"GET_ITER CALL_FUNCTION_1\")\n self.add_make_function_rule(rule_pat, opname, token.attr, customize)\n pass\n custom_ops_processed.add(opname)\n elif opname == 'LOAD_ATTR':\n self.addRule(\"\"\"\n expr ::= attribute\n attribute ::= expr LOAD_ATTR\n \"\"\", nop_func)\n custom_ops_processed.add(opname)\n elif opname == 'LOAD_LISTCOMP':\n self.add_unique_rule(\"expr ::= listcomp\", opname, token.attr, customize)\n custom_ops_processed.add(opname)\n elif opname == 'LOAD_SETCOMP':\n # Should this be generalized and put under MAKE_FUNCTION?\n if has_get_iter_call_function1:\n self.addRule(\"expr ::= set_comp\", nop_func)\n rule_pat = (\"set_comp ::= LOAD_SETCOMP %sMAKE_FUNCTION_0 expr \"\n \"GET_ITER CALL_FUNCTION_1\")\n self.add_make_function_rule(rule_pat, opname, token.attr, customize)\n pass\n custom_ops_processed.add(opname)\n elif opname == 'LOOKUP_METHOD':\n # A PyPy speciality - DRY with parse3\n self.addRule(\"\"\"\n expr ::= attribute\n attribute ::= expr LOOKUP_METHOD\n \"\"\",\n nop_func)\n custom_ops_processed.add(opname)\n elif opname.startswith('MAKE_CLOSURE'):\n # DRY with MAKE_FUNCTION\n # Note: this probably doesn't handle kwargs proprerly\n\n if opname == 'MAKE_CLOSURE_0' and 'LOAD_DICTCOMP' in self.seen_ops:\n # Is there something general going on here?\n # Note that 3.6+ doesn't do this, but we'll remove\n # this rule in parse36.py\n rule = \"\"\"\n dict_comp ::= load_closure LOAD_DICTCOMP LOAD_CONST\n MAKE_CLOSURE_0 expr\n GET_ITER CALL_FUNCTION_1\n \"\"\"\n self.addRule(rule, nop_func)\n\n args_pos, args_kw, annotate_args = token.attr\n\n # FIXME: Fold test into add_make_function_rule\n if self.version < 3.3:\n j = 1\n else:\n j = 2\n if is_pypy or (i >= j and tokens[i-j] == 'LOAD_LAMBDA'):\n rule_pat = ('mklambda ::= %sload_closure LOAD_LAMBDA %%s%s' %\n ('pos_arg ' * args_pos, opname))\n self.add_make_function_rule(rule_pat, opname, token.attr, customize)\n\n if has_get_iter_call_function1:\n rule_pat = (\"generator_exp ::= %sload_closure load_genexpr %%s%s expr \"\n \"GET_ITER CALL_FUNCTION_1\" % ('pos_arg ' * args_pos, opname))\n self.add_make_function_rule(rule_pat, opname, token.attr, customize)\n\n if has_get_iter_call_function1:\n if (is_pypy or (i >= j and tokens[i-j] == 'LOAD_LISTCOMP')):\n # In the tokens we saw:\n # LOAD_LISTCOMP LOAD_CONST MAKE_FUNCTION (>= 3.3) or\n # LOAD_LISTCOMP MAKE_FUNCTION (< 3.3) or\n # and have GET_ITER CALL_FUNCTION_1\n # Todo: For Pypy we need to modify this slightly\n rule_pat = ('listcomp ::= %sload_closure LOAD_LISTCOMP %%s%s expr '\n 'GET_ITER CALL_FUNCTION_1' % ('pos_arg ' * args_pos, opname))\n self.add_make_function_rule(rule_pat, opname, token.attr, customize)\n if (is_pypy or (i >= j and tokens[i-j] == 'LOAD_SETCOMP')):\n rule_pat = ('set_comp ::= %sload_closure LOAD_SETCOMP %%s%s expr '\n 'GET_ITER CALL_FUNCTION_1' % ('pos_arg ' * args_pos, opname))\n self.add_make_function_rule(rule_pat, opname, token.attr, customize)\n if (is_pypy or (i >= j and tokens[i-j] == 'LOAD_DICTCOMP')):\n self.add_unique_rule('dict_comp ::= %sload_closure LOAD_DICTCOMP %s '\n 'expr GET_ITER CALL_FUNCTION_1' %\n ('pos_arg ' * args_pos, opname),\n opname, token.attr, customize)\n\n if args_kw > 0:\n kwargs_str = 'kwargs '\n else:\n kwargs_str = ''\n\n # Note order of kwargs and pos args changed between 3.3-3.4\n if self.version <= 3.2:\n rule = ('mkfunc ::= %s%sload_closure LOAD_CONST %s'\n % (kwargs_str, 'expr ' * args_pos, opname))\n elif self.version == 3.3:\n rule = ('mkfunc ::= %s%sload_closure LOAD_CONST LOAD_CONST %s'\n % (kwargs_str, 'expr ' * args_pos, opname))\n elif self.version >= 3.4:\n rule = ('mkfunc ::= %s%s load_closure LOAD_CONST LOAD_CONST %s'\n % ('expr ' * args_pos, kwargs_str, opname))\n\n self.add_unique_rule(rule, opname, token.attr, customize)\n\n if args_kw == 0:\n rule = ('mkfunc ::= %sload_closure load_genexpr %s'\n % ('pos_arg ' * args_pos, opname))\n self.add_unique_rule(rule, opname, token.attr, customize)\n\n if self.version < 3.4:\n rule = ('mkfunc ::= %sload_closure LOAD_CONST %s'\n % ('expr ' * args_pos, opname))\n self.add_unique_rule(rule, opname, token.attr, customize)\n\n pass\n elif opname_base.startswith('MAKE_FUNCTION'):\n # DRY with MAKE_CLOSURE\n if self.version >= 3.6:\n # The semantics of MAKE_FUNCTION in 3.6 are totally different from\n # before.\n args_pos, args_kw, annotate_args, closure = token.attr\n stack_count = args_pos + args_kw + annotate_args\n if closure:\n if args_pos:\n rule = ('mklambda ::= %s%s%s%s' %\n ('expr ' * stack_count,\n 'load_closure ' * closure,\n 'BUILD_TUPLE_1 LOAD_LAMBDA LOAD_CONST ',\n opname))\n else:\n rule = ('mklambda ::= %s%s%s' %\n ('load_closure ' * closure,\n 'LOAD_LAMBDA LOAD_CONST ',\n opname))\n self.add_unique_rule(rule, opname, token.attr, customize)\n\n else:\n rule = ('mklambda ::= %sLOAD_LAMBDA LOAD_CONST %s' %\n (('expr ' * stack_count), opname))\n self.add_unique_rule(rule, opname, token.attr, customize)\n\n\n rule = ('mkfunc ::= %s%s%s%s' %\n ('expr ' * stack_count,\n 'load_closure ' * closure,\n 'LOAD_CONST ' * 2,\n opname))\n self.add_unique_rule(rule, opname, token.attr, customize)\n\n if has_get_iter_call_function1:\n rule_pat = (\"generator_exp ::= %sload_genexpr %%s%s expr \"\n \"GET_ITER CALL_FUNCTION_1\" % ('pos_arg ' * args_pos, opname))\n self.add_make_function_rule(rule_pat, opname, token.attr, customize)\n rule_pat = (\"generator_exp ::= %sload_closure load_genexpr %%s%s expr \"\n \"GET_ITER CALL_FUNCTION_1\" % ('pos_arg ' * args_pos, opname))\n self.add_make_function_rule(rule_pat, opname, token.attr, customize)\n if is_pypy or (i >= 2 and tokens[i-2] == 'LOAD_LISTCOMP'):\n if self.version >= 3.6:\n # 3.6+ sometimes bundles all of the\n # 'exprs' in the rule above into a\n # tuple.\n rule_pat = (\"listcomp ::= load_closure LOAD_LISTCOMP %%s%s \"\n \"expr GET_ITER CALL_FUNCTION_1\" % (opname,))\n self.add_make_function_rule(rule_pat, opname, token.attr, customize)\n rule_pat = (\"listcomp ::= %sLOAD_LISTCOMP %%s%s expr \"\n \"GET_ITER CALL_FUNCTION_1\" % ('expr ' * args_pos, opname))\n self.add_make_function_rule(rule_pat, opname, token.attr, customize)\n\n if is_pypy or (i >= 2 and tokens[i-2] == 'LOAD_LAMBDA'):\n rule_pat = ('mklambda ::= %s%sLOAD_LAMBDA %%s%s' %\n (('pos_arg ' * args_pos),\n ('kwarg ' * args_kw),\n opname))\n self.add_make_function_rule(rule_pat, opname, token.attr, customize)\n continue\n\n if self.version < 3.6:\n args_pos, args_kw, annotate_args = token.attr\n else:\n args_pos, args_kw, annotate_args, closure = token.attr\n\n if self.version < 3.3:\n j = 1\n else:\n j = 2\n\n if has_get_iter_call_function1:\n rule_pat = (\"generator_exp ::= %sload_genexpr %%s%s expr \"\n \"GET_ITER CALL_FUNCTION_1\" % ('pos_arg ' * args_pos, opname))\n self.add_make_function_rule(rule_pat, opname, token.attr, customize)\n\n if is_pypy or (i >= j and tokens[i-j] == 'LOAD_LISTCOMP'):\n # In the tokens we saw:\n # LOAD_LISTCOMP LOAD_CONST MAKE_FUNCTION (>= 3.3) or\n # LOAD_LISTCOMP MAKE_FUNCTION (< 3.3) or\n # and have GET_ITER CALL_FUNCTION_1\n # Todo: For Pypy we need to modify this slightly\n rule_pat = (\"listcomp ::= %sLOAD_LISTCOMP %%s%s expr \"\n \"GET_ITER CALL_FUNCTION_1\" % ('expr ' * args_pos, opname))\n self.add_make_function_rule(rule_pat, opname, token.attr, customize)\n\n # FIXME: Fold test into add_make_function_rule\n if is_pypy or (i >= j and tokens[i-j] == 'LOAD_LAMBDA'):\n rule_pat = ('mklambda ::= %s%sLOAD_LAMBDA %%s%s' %\n (('pos_arg ' * args_pos),\n ('kwarg ' * args_kw),\n opname))\n self.add_make_function_rule(rule_pat, opname, token.attr, customize)\n\n if args_kw == 0:\n kwargs = 'no_kwargs'\n self.add_unique_rule(\"no_kwargs ::=\", opname, token.attr, customize)\n else:\n kwargs = 'kwargs'\n\n if self.version < 3.3:\n # positional args after keyword args\n rule = ('mkfunc ::= %s %s%s%s' %\n (kwargs, 'pos_arg ' * args_pos, 'LOAD_CONST ',\n opname))\n self.add_unique_rule(rule, opname, token.attr, customize)\n rule = ('mkfunc ::= %s%s%s' %\n ('pos_arg ' * args_pos, 'LOAD_CONST ',\n opname))\n elif self.version == 3.3:\n # positional args after keyword args\n rule = ('mkfunc ::= %s %s%s%s' %\n (kwargs, 'pos_arg ' * args_pos, 'LOAD_CONST '*2,\n opname))\n elif self.version > 3.5:\n # positional args before keyword args\n rule = ('mkfunc ::= %s%s %s%s' %\n ('pos_arg ' * args_pos, kwargs, 'LOAD_CONST '*2,\n opname))\n elif self.version > 3.3:\n # positional args before keyword args\n rule = ('mkfunc ::= %s%s %s%s' %\n ('pos_arg ' * args_pos, kwargs, 'LOAD_CONST '*2,\n opname))\n else:\n rule = ('mkfunc ::= %s%sexpr %s' %\n (kwargs, 'pos_arg ' * args_pos, opname))\n self.add_unique_rule(rule, opname, token.attr, customize)\n\n if opname.startswith('MAKE_FUNCTION_A'):\n if self.version >= 3.6:\n rule = ('mkfunc_annotate ::= %s%sannotate_tuple LOAD_CONST LOAD_CONST %s' %\n (('pos_arg ' * (args_pos)),\n ('call ' * (annotate_args-1)), opname))\n self.add_unique_rule(rule, opname, token.attr, customize)\n rule = ('mkfunc_annotate ::= %s%sannotate_tuple LOAD_CONST LOAD_CONST %s' %\n (('pos_arg ' * (args_pos)),\n ('annotate_arg ' * (annotate_args-1)), opname))\n if self.version >= 3.3:\n # Normally we remove EXTENDED_ARG from the opcodes, but in the case of\n # annotated functions can use the EXTENDED_ARG tuple to signal we have an annotated function.\n # Yes this is a little hacky\n rule = ('mkfunc_annotate ::= %s%sannotate_tuple LOAD_CONST LOAD_CONST EXTENDED_ARG %s' %\n (('pos_arg ' * (args_pos)),\n ('call ' * (annotate_args-1)), opname))\n self.add_unique_rule(rule, opname, token.attr, customize)\n rule = ('mkfunc_annotate ::= %s%sannotate_tuple LOAD_CONST LOAD_CONST EXTENDED_ARG %s' %\n (('pos_arg ' * (args_pos)),\n ('annotate_arg ' * (annotate_args-1)), opname))\n else:\n # See above comment about use of EXTENDED_ARG\n rule = ('mkfunc_annotate ::= %s%sannotate_tuple LOAD_CONST EXTENDED_ARG %s' %\n (('pos_arg ' * (args_pos)),\n ('annotate_arg ' * (annotate_args-1)), opname))\n self.add_unique_rule(rule, opname, token.attr, customize)\n rule = ('mkfunc_annotate ::= %s%sannotate_tuple LOAD_CONST EXTENDED_ARG %s' %\n (('pos_arg ' * (args_pos)),\n ('call ' * (annotate_args-1)), opname))\n self.addRule(rule, nop_func)\n elif opname == 'RETURN_VALUE_LAMBDA':\n self.addRule(\"\"\"\n return_lambda ::= ret_expr RETURN_VALUE_LAMBDA\n \"\"\", nop_func)\n custom_ops_processed.add(opname)\n elif opname == 'RAISE_VARARGS_0':\n self.addRule(\"\"\"\n stmt ::= raise_stmt0\n raise_stmt0 ::= RAISE_VARARGS_0\n \"\"\", nop_func)\n custom_ops_processed.add(opname)\n elif opname == 'RAISE_VARARGS_1':\n self.addRule(\"\"\"\n stmt ::= raise_stmt1\n raise_stmt1 ::= expr RAISE_VARARGS_1\n \"\"\", nop_func)\n custom_ops_processed.add(opname)\n elif opname == 'RAISE_VARARGS_2':\n self.addRule(\"\"\"\n stmt ::= raise_stmt2\n raise_stmt2 ::= expr expr RAISE_VARARGS_2\n \"\"\", nop_func)\n custom_ops_processed.add(opname)\n elif opname == 'SETUP_EXCEPT':\n self.addRule(\"\"\"\n try_except ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK\n except_handler opt_come_from_except\n\n tryelsestmt ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK\n except_handler else_suite come_from_except_clauses\n\n tryelsestmt ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK\n except_handler else_suite come_froms\n\n tryelsestmtl ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK\n except_handler else_suitel come_from_except_clauses\n\n stmt ::= tryelsestmtl3\n tryelsestmtl3 ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK\n except_handler COME_FROM else_suitel\n opt_come_from_except\n \"\"\", nop_func)\n custom_ops_processed.add(opname)\n elif opname_base in ('UNPACK_EX',):\n before_count, after_count = token.attr\n rule = 'unpack ::= ' + opname + ' store' * (before_count + after_count + 1)\n self.addRule(rule, nop_func)\n elif opname_base in ('UNPACK_TUPLE', 'UNPACK_SEQUENCE'):\n rule = 'unpack ::= ' + opname + ' store' * token.attr\n self.addRule(rule, nop_func)\n elif opname_base == 'UNPACK_LIST':\n rule = 'unpack_list ::= ' + opname + ' store' * token.attr\n self.addRule(rule, nop_func)\n custom_ops_processed.add(opname)\n pass\n pass\n\n self.check_reduce['aug_assign1'] = 'AST'\n self.check_reduce['aug_assign2'] = 'AST'\n self.check_reduce['while1stmt'] = 'noAST'\n self.check_reduce['while1elsestmt'] = 'noAST'\n self.check_reduce['ifelsestmt'] = 'AST'\n self.check_reduce['annotate_tuple'] = 'noAST'\n self.check_reduce['kwarg'] = 'noAST'\n if self.version < 3.6:\n # 3.6+ can remove a JUMP_FORWARD which messes up our testing here\n self.check_reduce['try_except'] = 'AST'\n\n # FIXME: remove parser errors caused by the below\n # self.check_reduce['while1elsestmt'] = 'noAST'\n return", "response": "Customize the grammar rules for a Python version."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ngive a code block and a Python bytecode version decompile it into a single code block.", "response": "def decompile(\n bytecode_version, co, out=None, showasm=None, showast=False,\n timestamp=None, showgrammar=False, code_objects={},\n source_size=None, is_pypy=None, magic_int=None,\n mapstream=None, do_fragments=False):\n \"\"\"\n ingests and deparses a given code block 'co'\n\n if `bytecode_version` is None, use the current Python intepreter\n version.\n\n Caller is responsible for closing `out` and `mapstream`\n \"\"\"\n if bytecode_version is None:\n bytecode_version = sysinfo2float()\n\n # store final output stream for case of error\n real_out = out or sys.stdout\n\n def write(s):\n s += '\\n'\n real_out.write(s)\n\n assert iscode(co)\n\n co_pypy_str = 'PyPy ' if is_pypy else ''\n run_pypy_str = 'PyPy ' if IS_PYPY else ''\n sys_version_lines = sys.version.split('\\n')\n write('# uncompyle6 version %s\\n'\n '# %sPython bytecode %s%s\\n# Decompiled from: %sPython %s' %\n (VERSION, co_pypy_str, bytecode_version,\n \" (%s)\" % str(magic_int) if magic_int else \"\",\n run_pypy_str, '\\n# '.join(sys_version_lines)))\n if co.co_filename:\n write('# Embedded file name: %s' % co.co_filename,)\n if timestamp:\n write('# Compiled at: %s' % datetime.datetime.fromtimestamp(timestamp))\n if source_size:\n write('# Size of source mod 2**32: %d bytes' % source_size)\n\n debug_opts = {\n 'asm': showasm,\n 'ast': showast,\n 'grammar': showgrammar\n }\n\n try:\n if mapstream:\n if isinstance(mapstream, str):\n mapstream = _get_outstream(mapstream)\n\n deparsed = deparse_code_with_map(bytecode_version, co, out, showasm, showast,\n showgrammar,\n code_objects = code_objects,\n is_pypy = is_pypy,\n )\n header_count = 3+len(sys_version_lines)\n linemap = [(line_no, deparsed.source_linemap[line_no]+header_count)\n for line_no in\n sorted(deparsed.source_linemap.keys())]\n mapstream.write(\"\\n\\n# %s\\n\" % linemap)\n else:\n if do_fragments:\n deparse_fn = code_deparse_fragments\n else:\n deparse_fn = code_deparse\n deparsed = deparse_fn(co, out, bytecode_version,\n debug_opts = debug_opts,\n is_pypy=is_pypy)\n pass\n return deparsed\n except pysource.SourceWalkerError as e:\n # deparsing failed\n raise pysource.SourceWalkerError(str(e))"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef main(in_base, out_base, compiled_files, source_files, outfile=None,\n showasm=None, showast=False, do_verify=False,\n showgrammar=False, raise_on_error=False,\n do_linemaps=False, do_fragments=False):\n \"\"\"\n in_base\tbase directory for input files\n out_base\tbase directory for output files (ignored when\n files\tlist of filenames to be uncompyled (relative to in_base)\n outfile\twrite output to this filename (overwrites out_base)\n\n For redirecting output to\n - \t\toutfile= (out_base is ignored)\n - files below out_base\tout_base=...\n - stdout\t\t\tout_base=None, outfile=None\n \"\"\"\n tot_files = okay_files = failed_files = verify_failed_files = 0\n current_outfile = outfile\n linemap_stream = None\n\n for source_path in source_files:\n compiled_files.append(compile_file(source_path))\n\n for filename in compiled_files:\n infile = os.path.join(in_base, filename)\n # print(\"XXX\", infile)\n if not os.path.exists(infile):\n sys.stderr.write(\"File '%s' doesn't exist. Skipped\\n\"\n % infile)\n continue\n\n if do_linemaps:\n linemap_stream = infile + '.pymap'\n pass\n\n # print (infile, file=sys.stderr)\n\n if outfile: # outfile was given as parameter\n outstream = _get_outstream(outfile)\n elif out_base is None:\n outstream = sys.stdout\n if do_linemaps:\n linemap_stream = sys.stdout\n if do_verify:\n prefix = os.path.basename(filename) + '-'\n if prefix.endswith('.py'):\n prefix = prefix[:-len('.py')]\n\n # Unbuffer output if possible\n buffering = -1 if sys.stdout.isatty() else 0\n if PYTHON_VERSION >= 3.5:\n t = tempfile.NamedTemporaryFile(mode='w+b',\n buffering=buffering,\n suffix='.py',\n prefix=prefix)\n else:\n t = tempfile.NamedTemporaryFile(mode='w+b',\n suffix='.py',\n prefix=prefix)\n current_outfile = t.name\n sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', buffering)\n tee = subprocess.Popen([\"tee\", current_outfile],\n stdin=subprocess.PIPE)\n os.dup2(tee.stdin.fileno(), sys.stdout.fileno())\n os.dup2(tee.stdin.fileno(), sys.stderr.fileno())\n else:\n if filename.endswith('.pyc'):\n current_outfile = os.path.join(out_base, filename[0:-1])\n else:\n current_outfile = os.path.join(out_base, filename) + '_dis'\n pass\n pass\n\n outstream = _get_outstream(current_outfile)\n\n # print(current_outfile, file=sys.stderr)\n\n # Try to uncompile the input file\n try:\n deparsed = decompile_file(infile, outstream, showasm, showast, showgrammar,\n linemap_stream, do_fragments)\n if do_fragments:\n for d in deparsed:\n last_mod = None\n offsets = d.offsets\n for e in sorted([k for k in offsets.keys() if isinstance(k[1], int)]):\n if e[0] != last_mod:\n line = '=' * len(e[0])\n outstream.write(\"%s\\n%s\\n%s\\n\" % (line, e[0], line))\n last_mod = e[0]\n info = offsets[e]\n extractInfo = d.extract_node_info(info)\n outstream.write(\"%s\" % info.node.format().strip() + \"\\n\")\n outstream.write(extractInfo.selectedLine + \"\\n\")\n outstream.write(extractInfo.markerLine + \"\\n\\n\")\n pass\n pass\n tot_files += 1\n except (ValueError, SyntaxError, ParserError, pysource.SourceWalkerError) as e:\n sys.stdout.write(\"\\n\")\n sys.stderr.write(\"\\n# file %s\\n# %s\\n\" % (infile, e))\n failed_files += 1\n tot_files += 1\n except KeyboardInterrupt:\n if outfile:\n outstream.close()\n os.remove(outfile)\n sys.stdout.write(\"\\n\")\n sys.stderr.write(\"\\nLast file: %s \" % (infile))\n raise\n # except:\n # failed_files += 1\n # if current_outfile:\n # outstream.close()\n # os.rename(current_outfile, current_outfile + '_failed')\n # else:\n # sys.stderr.write(\"\\n# %s\" % sys.exc_info()[1])\n # sys.stderr.write(\"\\n# Can't uncompile %s\\n\" % infile)\n else: # uncompile successful\n if current_outfile:\n outstream.close()\n\n if do_verify:\n try:\n msg = verify.compare_code_with_srcfile(infile,\n current_outfile,\n do_verify)\n if not current_outfile:\n if not msg:\n print('\\n# okay decompiling %s' % infile)\n okay_files += 1\n else:\n verify_failed_files += 1\n print('\\n# %s\\n\\t%s', infile, msg)\n pass\n else:\n okay_files += 1\n pass\n except verify.VerifyCmpError as e:\n print(e)\n verify_failed_files += 1\n os.rename(current_outfile, current_outfile + '_unverified')\n sys.stderr.write(\"### Error Verifying %s\\n\" % filename)\n sys.stderr.write(str(e) + \"\\n\")\n if not outfile:\n if raise_on_error:\n raise\n pass\n pass\n pass\n else:\n okay_files += 1\n pass\n elif do_verify:\n sys.stderr.write(\"\\n### uncompile successful, but no file to compare against\\n\")\n pass\n else:\n okay_files += 1\n if not current_outfile:\n mess = '\\n# okay decompiling'\n # mem_usage = __memUsage()\n print(mess, infile)\n if current_outfile:\n sys.stdout.write(\"%s\\r\" %\n status_msg(do_verify, tot_files, okay_files, failed_files,\n verify_failed_files, do_verify))\n try:\n # FIXME: Something is weird with Pypy here\n sys.stdout.flush()\n except:\n pass\n if current_outfile:\n sys.stdout.write(\"\\n\")\n try:\n # FIXME: Something is weird with Pypy here\n sys.stdout.flush()\n except:\n pass\n pass\n return (tot_files, okay_files, failed_files, verify_failed_files)", "response": "This function is the main entry point for the main function of the main function of the main function of the main function of the main function of the main function of the main function of the main function of the main function."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ndumping function defintion, doc string, and function body. This code is specialied for Python 2.", "response": "def make_function2(self, node, is_lambda, nested=1, code_node=None):\n \"\"\"\n Dump function defintion, doc string, and function body.\n This code is specialied for Python 2.\n \"\"\"\n\n # FIXME: call make_function3 if we are self.version >= 3.0\n # and then simplify the below.\n\n def build_param(ast, name, default):\n \"\"\"build parameters:\n - handle defaults\n - handle format tuple parameters\n \"\"\"\n # if formal parameter is a tuple, the paramater name\n # starts with a dot (eg. '.1', '.2')\n if name.startswith('.'):\n # replace the name with the tuple-string\n name = self.get_tuple_parameter(ast, name)\n pass\n\n if default:\n value = self.traverse(default, indent='')\n maybe_show_tree_param_default(self.showast, name, value)\n result = '%s=%s' % (name, value)\n if result[-2:] == '= ':\t# default was 'LOAD_CONST None'\n result += 'None'\n return result\n else:\n return name\n\n # MAKE_FUNCTION_... or MAKE_CLOSURE_...\n assert node[-1].kind.startswith('MAKE_')\n\n args_node = node[-1]\n if isinstance(args_node.attr, tuple):\n # positional args are after kwargs\n defparams = node[1:args_node.attr[0]+1]\n pos_args, kw_args, annotate_argc = args_node.attr\n else:\n defparams = node[:args_node.attr]\n kw_args = 0\n pass\n\n lambda_index = None\n\n if lambda_index and is_lambda and iscode(node[lambda_index].attr):\n assert node[lambda_index].kind == 'LOAD_LAMBDA'\n code = node[lambda_index].attr\n else:\n code = code_node.attr\n\n assert iscode(code)\n code = Code(code, self.scanner, self.currentclass)\n\n # add defaults values to parameter names\n argc = code.co_argcount\n paramnames = list(code.co_varnames[:argc])\n\n # defaults are for last n parameters, thus reverse\n paramnames.reverse(); defparams.reverse()\n\n try:\n ast = self.build_ast(code._tokens,\n code._customize,\n is_lambda = is_lambda,\n noneInNames = ('None' in code.co_names))\n except (ParserError, ParserError2) as p:\n self.write(str(p))\n if not self.tolerate_errors:\n self.ERROR = p\n return\n\n kw_pairs = 0\n indent = self.indent\n\n # build parameters\n params = [build_param(ast, name, default) for\n name, default in zip_longest(paramnames, defparams, fillvalue=None)]\n params.reverse() # back to correct order\n\n if code_has_star_arg(code):\n params.append('*%s' % code.co_varnames[argc])\n argc += 1\n\n # dump parameter list (with default values)\n if is_lambda:\n self.write(\"lambda \", \", \".join(params))\n # If the last statement is None (which is the\n # same thing as \"return None\" in a lambda) and the\n # next to last statement is a \"yield\". Then we want to\n # drop the (return) None since that was just put there\n # to have something to after the yield finishes.\n # FIXME: this is a bit hoaky and not general\n if (len(ast) > 1 and\n self.traverse(ast[-1]) == 'None' and\n self.traverse(ast[-2]).strip().startswith('yield')):\n del ast[-1]\n # Now pick out the expr part of the last statement\n ast_expr = ast[-1]\n while ast_expr.kind != 'expr':\n ast_expr = ast_expr[0]\n ast[-1] = ast_expr\n pass\n else:\n self.write(\"(\", \", \".join(params))\n\n if kw_args > 0:\n if not (4 & code.co_flags):\n if argc > 0:\n self.write(\", *, \")\n else:\n self.write(\"*, \")\n pass\n else:\n self.write(\", \")\n\n for n in node:\n if n == 'pos_arg':\n continue\n else:\n self.preorder(n)\n break\n pass\n\n if code_has_star_star_arg(code):\n if argc > 0:\n self.write(', ')\n self.write('**%s' % code.co_varnames[argc + kw_pairs])\n\n if is_lambda:\n self.write(\": \")\n else:\n self.println(\"):\")\n\n if len(code.co_consts) > 0 and code.co_consts[0] is not None and not is_lambda: # ugly\n # docstring exists, dump it\n print_docstring(self, indent, code.co_consts[0])\n\n code._tokens = None # save memory\n if not is_lambda:\n assert ast == 'stmts'\n\n all_globals = find_all_globals(ast, set())\n\n globals, nonlocals = find_globals_and_nonlocals(ast, set(), set(),\n code, self.version)\n\n # Python 2 doesn't support the \"nonlocal\" statement\n assert self.version >= 3.0 or not nonlocals\n\n for g in sorted((all_globals & self.mod_globs) | globals):\n self.println(self.indent, 'global ', g)\n self.mod_globs -= all_globals\n has_none = 'None' in code.co_names\n rn = has_none and not find_none(ast)\n self.gen_source(ast, code.co_name, code._customize, is_lambda=is_lambda,\n returnNone=rn)\n code._tokens = None; code._customize = None"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef make_function3(self, node, is_lambda, nested=1, code_node=None):\n\n # For Python 3.3, the evaluation stack in MAKE_FUNCTION is:\n\n # * default argument objects in positional order\n # * pairs of name and default argument, with the name just below\n # the object on the stack, for keyword-only parameters\n # * parameter annotation objects\n # * a tuple listing the parameter names for the annotations\n # (only if there are ony annotation objects)\n # * the code associated with the function (at TOS1)\n # * the qualified name of the function (at TOS)\n\n # For Python 3.0 .. 3.2 the evaluation stack is:\n # The function object is defined to have argc default parameters,\n # which are found below TOS.\n # * first come positional args in the order they are given in the source,\n # * next come the keyword args in the order they given in the source,\n # * finally is the code associated with the function (at TOS)\n #\n # Note: There is no qualified name at TOS\n\n # MAKE_CLOSURE adds an additional closure slot\n\n # In Python 3.6 stack entries change again. I understand\n # 3.7 changes some of those changes. Yes, it is hard to follow\n # and I am sure I haven't been able to keep up.\n\n # Thank you, Python.\n\n def build_param(ast, name, default):\n \"\"\"build parameters:\n - handle defaults\n - handle format tuple parameters\n \"\"\"\n if self.version >= 3.6:\n value = default\n else:\n value = self.traverse(default, indent='')\n maybe_show_tree_param_default(self.showast, name, value)\n result = '%s=%s' % (name, value)\n\n # The below can probably be removed. This is probably\n # a holdover from days when LOAD_CONST erroneously\n # didn't handle LOAD_CONST None properly\n if result[-2:] == '= ':\t# default was 'LOAD_CONST None'\n result += 'None'\n\n return result\n\n # MAKE_FUNCTION_... or MAKE_CLOSURE_...\n assert node[-1].kind.startswith('MAKE_')\n\n # Python 3.3+ adds a qualified name at TOS (-1)\n # moving down the LOAD_LAMBDA instruction\n if 3.0 <= self.version <= 3.2:\n lambda_index = -2\n elif 3.03 <= self.version:\n lambda_index = -3\n else:\n lambda_index = None\n\n args_node = node[-1]\n\n annotate_dict = {}\n\n # Get a list of tree nodes that constitute the values for the \"default\n # parameters\"; these are default values that appear before any *, and are\n # not to be confused with keyword parameters which may appear after *.\n args_attr = args_node.attr\n\n if isinstance(args_attr, tuple) or (self.version >= 3.6 and isinstance(args_attr, list)):\n if len(args_attr) == 3:\n pos_args, kw_args, annotate_argc = args_attr\n else:\n pos_args, kw_args, annotate_argc, closure = args_attr\n\n i = -4\n kw_pairs = 0\n if closure:\n # FIXME: fill in\n i -= 1\n if annotate_argc:\n # Turn into subroutine and DRY with other use\n annotate_node = node[i]\n if annotate_node == 'expr':\n annotate_node = annotate_node[0]\n annotate_name_node = annotate_node[-1]\n if annotate_node == 'dict' and annotate_name_node.kind.startswith('BUILD_CONST_KEY_MAP'):\n types = [self.traverse(n, indent='') for n in annotate_node[:-2]]\n names = annotate_node[-2].attr\n l = len(types)\n assert l == len(names)\n for i in range(l): annotate_dict[names[i]] = types[i]\n pass\n pass\n i -= 1\n if kw_args:\n kw_node = node[i]\n if kw_node == 'expr':\n kw_node = kw_node[0]\n if kw_node == 'dict':\n kw_pairs = kw_node[-1].attr\n\n\n # FIXME: there is probably a better way to classify this.\n have_kwargs = node[0].kind.startswith('kwarg') or node[0] == 'no_kwargs'\n if len(node) >= 4:\n lc_index = -4\n else:\n lc_index = -3\n pass\n\n if (3.0 <= self.version <= 3.3 and len(node) > 2 and\n node[lambda_index] != 'LOAD_LAMBDA' and\n (have_kwargs or node[lc_index].kind != 'load_closure')):\n\n # Find the index in \"node\" where the first default\n # parameter value is located. Note this is in contrast to\n # key-word arguments, pairs of (name, value), which appear after \"*\".\n # \"default_values_start\" is this location.\n default_values_start = 0\n if node[0] == 'no_kwargs':\n default_values_start += 1\n # args are after kwargs; kwargs are bundled as one node\n if node[default_values_start] == 'kwargs':\n default_values_start += 1\n defparams = node[default_values_start:default_values_start+args_node.attr[0]]\n else:\n if self.version < 3.6:\n defparams = node[:args_node.attr[0]]\n kw_args = 0\n else:\n defparams = []\n # FIXME: DRY with code below\n default, kw_args, annotate_argc = args_node.attr[0:3]\n if default:\n expr_node = node[0]\n if node[0] == 'pos_arg':\n expr_node = expr_node[0]\n assert expr_node == 'expr', \"expecting mkfunc default node to be an expr\"\n if (expr_node[0] == 'LOAD_CONST' and\n isinstance(expr_node[0].attr, tuple)):\n defparams = [repr(a) for a in expr_node[0].attr]\n elif expr_node[0] in frozenset(('list', 'tuple', 'dict', 'set')):\n defparams = [self.traverse(n, indent='') for n in expr_node[0][:-1]]\n else:\n defparams = []\n pass\n else:\n if self.version < 3.6:\n defparams = node[:args_node.attr]\n kw_args = 0\n else:\n default, kw_args, annotate, closure = args_node.attr\n if default:\n expr_node = node[0]\n if node[0] == 'pos_arg':\n expr_node = expr_node[0]\n assert expr_node == 'expr', \"expecting mkfunc default node to be an expr\"\n if (expr_node[0] == 'LOAD_CONST' and\n isinstance(expr_node[0].attr, tuple)):\n defparams = [repr(a) for a in expr_node[0].attr]\n elif expr_node[0] in frozenset(('list', 'tuple', 'dict', 'set')):\n defparams = [self.traverse(n, indent='') for n in expr_node[0][:-1]]\n else:\n defparams = []\n\n i = -4\n kw_pairs = 0\n if closure:\n # FIXME: fill in\n annotate = node[i]\n i -= 1\n if annotate_argc:\n # Turn into subroutine and DRY with other use\n annotate_node = node[i]\n if annotate_node == 'expr':\n annotate_node = annotate_node[0]\n annotate_name_node = annotate_node[-1]\n if annotate_node == 'dict' and annotate_name_node.kind.startswith('BUILD_CONST_KEY_MAP'):\n types = [self.traverse(n, indent='') for n in annotate_node[:-2]]\n names = annotate_node[-2].attr\n l = len(types)\n assert l == len(names)\n for i in range(l): annotate_dict[names[i]] = types[i]\n pass\n pass\n i -= 1\n if kw_args:\n kw_node = node[i]\n if kw_node == 'expr':\n kw_node = kw_node[0]\n if kw_node == 'dict':\n kw_pairs = kw_node[-1].attr\n pass\n\n if lambda_index and is_lambda and iscode(node[lambda_index].attr):\n assert node[lambda_index].kind == 'LOAD_LAMBDA'\n code = node[lambda_index].attr\n else:\n code = code_node.attr\n\n assert iscode(code)\n scanner_code = Code(code, self.scanner, self.currentclass)\n\n # add defaults values to parameter names\n argc = code.co_argcount\n paramnames = list(scanner_code.co_varnames[:argc])\n\n # defaults are for last n parameters, thus reverse\n paramnames.reverse();\n defparams.reverse()\n\n try:\n ast = self.build_ast(scanner_code._tokens,\n scanner_code._customize,\n is_lambda = is_lambda,\n noneInNames = ('None' in code.co_names))\n except (ParserError, ParserError2) as p:\n self.write(str(p))\n if not self.tolerate_errors:\n self.ERROR = p\n return\n\n if self.version >= 3.0:\n if self.version < 3.6:\n kw_pairs = args_node.attr[1]\n else:\n kw_pairs = 0\n\n # build parameters\n params = []\n if defparams:\n for i, defparam in enumerate(defparams):\n params.append(build_param(ast, paramnames[i], defparam))\n\n params += paramnames[i+1:]\n else:\n params = paramnames\n\n params.reverse() # back to correct order\n\n if code_has_star_arg(code):\n if self.version > 3.0:\n params.append('*%s' % code.co_varnames[argc + kw_pairs])\n else:\n params.append('*%s' % code.co_varnames[argc])\n argc += 1\n\n # dump parameter list (with default values)\n if is_lambda:\n self.write(\"lambda \", \", \".join(params))\n # If the last statement is None (which is the\n # same thing as \"return None\" in a lambda) and the\n # next to last statement is a \"yield\". Then we want to\n # drop the (return) None since that was just put there\n # to have something to after the yield finishes.\n # FIXME: this is a bit hoaky and not general\n if (len(ast) > 1 and\n self.traverse(ast[-1]) == 'None' and\n self.traverse(ast[-2]).strip().startswith('yield')):\n del ast[-1]\n # Now pick out the expr part of the last statement\n ast_expr = ast[-1]\n while ast_expr.kind != 'expr':\n ast_expr = ast_expr[0]\n ast[-1] = ast_expr\n pass\n else:\n # FIXME: add annotations here\n self.write(\"(\", \", \".join(params))\n # self.println(indent, '#flags:\\t', int(code.co_flags))\n\n ends_in_comma = False\n if kw_args > 0:\n if not (4 & code.co_flags):\n if argc > 0:\n self.write(\", *, \")\n else:\n self.write(\"*, \")\n pass\n else:\n self.write(\", \")\n ends_in_comma = True\n\n # FIXME: this is not correct for 3.5. or 3.6 (which works different)\n # and 3.7?\n if 3.0 <= self.version <= 3.2:\n kwargs = node[0]\n last = len(kwargs)-1\n i = 0\n for n in node[0]:\n if n == 'kwarg':\n self.write('%s=' % n[0].pattr)\n self.preorder(n[1])\n if i < last:\n self.write(', ')\n ends_in_comma = True\n pass\n else:\n ends_in_comma = False\n pass\n i += 1\n pass\n pass\n elif self.version <= 3.5:\n # FIXME this is not qute right for 3.5\n for n in node:\n if n == 'pos_arg':\n continue\n elif self.version >= 3.4 and not (n.kind in ('kwargs', 'no_kwargs', 'kwarg')):\n continue\n else:\n self.preorder(n)\n ends_in_comma = False\n break\n elif self.version >= 3.6:\n # argc = node[-1].attr\n # co = node[-3].attr\n # argcount = co.co_argcount\n # kwonlyargcount = co.co_kwonlyargcount\n\n free_tup = annotate_dict = kw_dict = default_tup = None\n fn_bits = node[-1].attr\n index = -4 # Skip over:\n # MAKE_FUNCTION,\n # LOAD_CONST qualified name,\n # LOAD_CONST code object\n if fn_bits[-1]:\n free_tup = node[index]\n index -= 1\n if fn_bits[-2]:\n annotate_dict = node[index]\n index -= 1\n if fn_bits[-3]:\n kw_dict = node[index]\n index -= 1\n if fn_bits[-4]:\n default_tup = node[index]\n\n if kw_dict == 'expr':\n kw_dict = kw_dict[0]\n\n # FIXME: handle free_tup, annotate_dict, and default_tup\n if kw_dict:\n assert kw_dict == 'dict'\n defaults = [self.traverse(n, indent='') for n in kw_dict[:-2]]\n names = eval(self.traverse(kw_dict[-2]))\n assert len(defaults) == len(names)\n sep = ''\n # FIXME: possibly handle line breaks\n for i, n in enumerate(names):\n self.write(sep)\n self.write(\"%s=%s\" % (n, defaults[i]))\n sep = ', '\n ends_in_comma = False\n pass\n pass\n pass\n\n if code_has_star_star_arg(code):\n if argc > 0 and not ends_in_comma:\n self.write(', ')\n self.write('**%s' % code.co_varnames[argc + kw_pairs])\n\n if is_lambda:\n self.write(\": \")\n else:\n self.write(')')\n if annotate_dict and 'return' in annotate_dict:\n self.write(' -> %s' % annotate_dict['return'])\n self.println(\":\")\n\n if len(code.co_consts) > 0 and code.co_consts[0] is not None and not is_lambda: # ugly\n # docstring exists, dump it\n print_docstring(self, self.indent, code.co_consts[0])\n\n scanner_code._tokens = None # save memory\n assert ast == 'stmts'\n\n all_globals = find_all_globals(ast, set())\n globals, nonlocals = find_globals_and_nonlocals(ast, set(),\n set(), code, self.version)\n\n for g in sorted((all_globals & self.mod_globs) | globals):\n self.println(self.indent, 'global ', g)\n\n for nl in sorted(nonlocals):\n self.println(self.indent, 'nonlocal ', nl)\n\n self.mod_globs -= all_globals\n has_none = 'None' in code.co_names\n rn = has_none and not find_none(ast)\n self.gen_source(ast, code.co_name, scanner_code._customize, is_lambda=is_lambda,\n returnNone=rn)\n scanner_code._tokens = None; scanner_code._customize = None", "response": "Create a function definition in Python 3. 0 and above a function body."} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncomparing two code - objects.", "response": "def cmp_code_objects(version, is_pypy, code_obj1, code_obj2, verify,\n name=''):\n \"\"\"\n Compare two code-objects.\n\n This is the main part of this module.\n \"\"\"\n # print code_obj1, type(code_obj2)\n assert iscode(code_obj1), \\\n \"cmp_code_object first object type is %s, not code\" % type(code_obj1)\n assert iscode(code_obj2), \\\n \"cmp_code_object second object type is %s, not code\" % type(code_obj2)\n # print dir(code_obj1)\n if isinstance(code_obj1, object):\n # new style classes (Python 2.2)\n # assume _both_ code objects to be new stle classes\n assert dir(code_obj1) == dir(code_obj2)\n else:\n # old style classes\n assert dir(code_obj1) == code_obj1.__members__\n assert dir(code_obj2) == code_obj2.__members__\n assert code_obj1.__members__ == code_obj2.__members__\n\n if name == '__main__':\n name = code_obj1.co_name\n else:\n name = '%s.%s' % (name, code_obj1.co_name)\n if name == '.?': name = '__main__'\n\n if isinstance(code_obj1, object) and code_equal(code_obj1, code_obj2):\n # use the new style code-classes' __cmp__ method, which\n # should be faster and more sophisticated\n # if this compare fails, we use the old routine to\n # find out, what exactly is nor equal\n # if this compare succeds, simply return\n # return\n pass\n\n if isinstance(code_obj1, object):\n members = [x for x in dir(code_obj1) if x.startswith('co_')]\n else:\n members = dir(code_obj1)\n members.sort() # ; members.reverse()\n\n tokens1 = None\n for member in members:\n if member in __IGNORE_CODE_MEMBERS__ or verify != 'verify':\n pass\n elif member == 'co_code':\n if verify != 'strong':\n continue\n scanner = get_scanner(version, is_pypy, show_asm=False)\n\n global JUMP_OPS\n JUMP_OPS = list(scan.JUMP_OPS) + ['JUMP_BACK']\n\n # use changed Token class\n # We (re)set this here to save exception handling,\n # which would get confusing.\n scanner.setTokenClass(Token)\n try:\n # ingest both code-objects\n tokens1, customize = scanner.ingest(code_obj1)\n del customize # save memory\n tokens2, customize = scanner.ingest(code_obj2)\n del customize # save memory\n finally:\n scanner.resetTokenClass() # restore Token class\n\n targets1 = dis.findlabels(code_obj1.co_code)\n tokens1 = [t for t in tokens1 if t.kind != 'COME_FROM']\n tokens2 = [t for t in tokens2 if t.kind != 'COME_FROM']\n\n i1 = 0; i2 = 0\n offset_map = {}; check_jumps = {}\n while i1 < len(tokens1):\n if i2 >= len(tokens2):\n if len(tokens1) == len(tokens2) + 2 \\\n and tokens1[-1].kind == 'RETURN_VALUE' \\\n and tokens1[-2].kind == 'LOAD_CONST' \\\n and tokens1[-2].pattr is None \\\n and tokens1[-3].kind == 'RETURN_VALUE':\n break\n else:\n raise CmpErrorCodeLen(name, tokens1, tokens2)\n\n offset_map[tokens1[i1].offset] = tokens2[i2].offset\n\n for idx1, idx2, offset2 in check_jumps.get(tokens1[i1].offset, []):\n if offset2 != tokens2[i2].offset:\n raise CmpErrorCode(name, tokens1[idx1].offset, tokens1[idx1],\n tokens2[idx2], tokens1, tokens2)\n\n if tokens1[i1].kind != tokens2[i2].kind:\n if tokens1[i1].kind == 'LOAD_CONST' == tokens2[i2].kind:\n i = 1\n while tokens1[i1+i].kind == 'LOAD_CONST':\n i += 1\n if tokens1[i1+i].kind.startswith(('BUILD_TUPLE', 'BUILD_LIST')) \\\n and i == int(tokens1[i1+i].kind.split('_')[-1]):\n t = tuple([ elem.pattr for elem in tokens1[i1:i1+i] ])\n if t != tokens2[i2].pattr:\n raise CmpErrorCode(name, tokens1[i1].offset, tokens1[i1],\n tokens2[i2], tokens1, tokens2)\n i1 += i + 1\n i2 += 1\n continue\n elif i == 2 and tokens1[i1+i].kind == 'ROT_TWO' and tokens2[i2+1].kind == 'UNPACK_SEQUENCE_2':\n i1 += 3\n i2 += 2\n continue\n elif i == 2 and tokens1[i1+i].kind in BIN_OP_FUNCS:\n f = BIN_OP_FUNCS[tokens1[i1+i].kind]\n if f(tokens1[i1].pattr, tokens1[i1+1].pattr) == tokens2[i2].pattr:\n i1 += 3\n i2 += 1\n continue\n elif tokens1[i1].kind == 'UNARY_NOT':\n if tokens2[i2].kind == 'POP_JUMP_IF_TRUE':\n if tokens1[i1+1].kind == 'POP_JUMP_IF_FALSE':\n i1 += 2\n i2 += 1\n continue\n elif tokens2[i2].kind == 'POP_JUMP_IF_FALSE':\n if tokens1[i1+1].kind == 'POP_JUMP_IF_TRUE':\n i1 += 2\n i2 += 1\n continue\n elif tokens1[i1].kind in ('JUMP_FORWARD', 'JUMP_BACK') \\\n and tokens1[i1-1].kind == 'RETURN_VALUE' \\\n and tokens2[i2-1].kind in ('RETURN_VALUE', 'RETURN_END_IF') \\\n and int(tokens1[i1].offset) not in targets1:\n i1 += 1\n continue\n elif tokens1[i1].kind == 'JUMP_BACK' and tokens2[i2].kind == 'CONTINUE':\n # FIXME: should make sure that offset is inside loop, not outside of it\n i1 += 2\n i2 += 2\n continue\n elif tokens1[i1].kind == 'JUMP_FORWARD' and tokens2[i2].kind == 'JUMP_BACK' \\\n and tokens1[i1+1].kind == 'JUMP_BACK' and tokens2[i2+1].kind == 'JUMP_BACK' \\\n and int(tokens1[i1].pattr) == int(tokens1[i1].offset) + 3:\n if int(tokens1[i1].pattr) == int(tokens1[i1+1].offset):\n i1 += 2\n i2 += 2\n continue\n elif tokens1[i1].kind == 'LOAD_NAME' and tokens2[i2].kind == 'LOAD_CONST' \\\n and tokens1[i1].pattr == 'None' and tokens2[i2].pattr is None:\n pass\n elif tokens1[i1].kind == 'LOAD_GLOBAL' and tokens2[i2].kind == 'LOAD_NAME' \\\n and tokens1[i1].pattr == tokens2[i2].pattr:\n pass\n elif tokens1[i1].kind == 'LOAD_ASSERT' and tokens2[i2].kind == 'LOAD_NAME' \\\n and tokens1[i1].pattr == tokens2[i2].pattr:\n pass\n elif (tokens1[i1].kind == 'RETURN_VALUE' and\n tokens2[i2].kind == 'RETURN_END_IF'):\n pass\n elif (tokens1[i1].kind == 'BUILD_TUPLE_0' and\n tokens2[i2].pattr == ()):\n pass\n else:\n raise CmpErrorCode(name, tokens1[i1].offset, tokens1[i1],\n tokens2[i2], tokens1, tokens2)\n elif tokens1[i1].kind in JUMP_OPS and tokens1[i1].pattr != tokens2[i2].pattr:\n if tokens1[i1].kind == 'JUMP_BACK':\n dest1 = int(tokens1[i1].pattr)\n dest2 = int(tokens2[i2].pattr)\n if offset_map[dest1] != dest2:\n raise CmpErrorCode(name, tokens1[i1].offset, tokens1[i1],\n tokens2[i2], tokens1, tokens2)\n else:\n # import pdb; pdb.set_trace()\n try:\n dest1 = int(tokens1[i1].pattr)\n if dest1 in check_jumps:\n check_jumps[dest1].append((i1, i2, dest2))\n else:\n check_jumps[dest1] = [(i1, i2, dest2)]\n except:\n pass\n\n i1 += 1\n i2 += 1\n del tokens1, tokens2 # save memory\n elif member == 'co_consts':\n # partial optimization can make the co_consts look different,\n # so we'll just compare the code consts\n codes1 = ( c for c in code_obj1.co_consts if hasattr(c, 'co_consts') )\n codes2 = ( c for c in code_obj2.co_consts if hasattr(c, 'co_consts') )\n\n for c1, c2 in zip(codes1, codes2):\n cmp_code_objects(version, is_pypy, c1, c2, verify,\n name=name)\n elif member == 'co_flags':\n flags1 = code_obj1.co_flags\n flags2 = code_obj2.co_flags\n if is_pypy:\n # For PYPY for now we don't care about PYPY_SOURCE_IS_UTF8:\n flags2 &= ~0x0100 # PYPY_SOURCE_IS_UTF8\n # We also don't care about COROUTINE or GENERATOR for now\n flags1 &= ~0x000000a0\n flags2 &= ~0x000000a0\n if flags1 != flags2:\n raise CmpErrorMember(name, 'co_flags',\n pretty_flags(flags1),\n pretty_flags(flags2))\n else:\n # all other members must be equal\n if getattr(code_obj1, member) != getattr(code_obj2, member):\n raise CmpErrorMember(name, member,\n getattr(code_obj1, member),\n getattr(code_obj2, member))"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncompare a. pyc file with a source code file.", "response": "def compare_code_with_srcfile(pyc_filename, src_filename, verify):\n \"\"\"Compare a .pyc with a source code file. If everything is okay, None\n is returned. Otherwise a string message describing the mismatch is returned.\n \"\"\"\n (version, timestamp, magic_int, code_obj1, is_pypy,\n source_size) = load_module(pyc_filename)\n if magic_int != PYTHON_MAGIC_INT:\n msg = (\"Can't compare code - Python is running with magic %s, but code is magic %s \"\n % (PYTHON_MAGIC_INT, magic_int))\n return msg\n try:\n code_obj2 = load_file(src_filename)\n except SyntaxError as e:\n # src_filename can be the first of a group sometimes\n return str(e).replace(src_filename, pyc_filename)\n cmp_code_objects(version, is_pypy, code_obj1, code_obj2, verify)\n if verify == 'verify-run':\n try:\n retcode = call(\"%s %s\" % (sys.executable, src_filename), shell=True)\n if retcode != 0:\n return \"Child was terminated by signal %d\" % retcode\n pass\n except OSError as e:\n return \"Execution failed: %s\" % e\n pass\n return None"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncompare two. pyc files.", "response": "def compare_files(pyc_filename1, pyc_filename2, verify):\n \"\"\"Compare two .pyc files.\"\"\"\n (version1, timestamp, magic_int1, code_obj1, is_pypy,\n source_size) = uncompyle6.load_module(pyc_filename1)\n (version2, timestamp, magic_int2, code_obj2, is_pypy,\n source_size) = uncompyle6.load_module(pyc_filename2)\n if (magic_int1 != magic_int2) and verify == 'verify':\n verify = 'weak_verify'\n cmp_code_objects(version1, is_pypy, code_obj1, code_obj2, verify)"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nconverting a code object co into a python source fragment.", "response": "def code_deparse(co, out=StringIO(), version=None, is_pypy=None,\n debug_opts=DEFAULT_DEBUG_OPTS,\n code_objects={}, compile_mode='exec',\n walker=FragmentsWalker):\n \"\"\"\n Convert the code object co into a python source fragment.\n\n :param version: The python version this code is from as a float, for\n example 2.6, 2.7, 3.2, 3.3, 3.4, 3.5 etc.\n :param co: The code object to parse.\n :param out: File like object to write the output to.\n :param debug_opts: A dictionary with keys\n 'asm': value determines whether to show\n mangled bytecode disdassembly\n 'ast': value determines whether to show\n 'grammar': boolean determining whether to show\n grammar reduction rules.\n If value is a file-like object, output that object's write method will\n be used rather than sys.stdout\n\n :return: The deparsed source fragment.\n \"\"\"\n\n assert iscode(co)\n\n if version is None:\n version = sysinfo2float()\n if is_pypy is None:\n is_pypy = IS_PYPY\n\n scanner = get_scanner(version, is_pypy=is_pypy)\n\n show_asm = debug_opts.get('asm', None)\n tokens, customize = scanner.ingest(co, code_objects=code_objects,\n show_asm=show_asm)\n\n tokens, customize = scanner.ingest(co)\n maybe_show_asm(show_asm, tokens)\n\n debug_parser = dict(PARSER_DEFAULT_DEBUG)\n show_grammar = debug_opts.get('grammar', None)\n if show_grammar:\n debug_parser['reduce'] = show_grammar\n debug_parser['errorstack'] = True\n\n # Build Syntax Tree from tokenized and massaged disassembly.\n # deparsed = pysource.FragmentsWalker(out, scanner, showast=showast)\n show_ast = debug_opts.get('ast', None)\n deparsed = walker(version, scanner, showast=show_ast,\n debug_parser=debug_parser, compile_mode=compile_mode,\n is_pypy=is_pypy)\n\n deparsed.ast = deparsed.build_ast(tokens, customize)\n\n assert deparsed.ast == 'stmts', 'Should have parsed grammar start'\n\n del tokens # save memory\n\n # convert leading '__doc__ = \"...\" into doc string\n assert deparsed.ast == 'stmts'\n (deparsed.mod_globs,\n nonlocals) = (pysource\n .find_globals_and_nonlocals(deparsed.ast,\n set(), set(),\n co, version))\n\n # Just when you think we've forgotten about what we\n # were supposed to to: Generate source from the Syntax ree!\n deparsed.gen_source(deparsed.ast, co.co_name, customize)\n\n deparsed.set_pos_info(deparsed.ast, 0, len(deparsed.text))\n deparsed.fixup_parents(deparsed.ast, None)\n\n for g in sorted(deparsed.mod_globs):\n deparsed.write('# global %s ## Warning: Unused global\\n' % g)\n\n if deparsed.ast_errors:\n deparsed.write(\"# NOTE: have decompilation errors.\\n\")\n deparsed.write(\"# Use -t option to show full context.\")\n for err in deparsed.ast_errors:\n deparsed.write(err)\n deparsed.ERROR = True\n\n if deparsed.ERROR:\n raise deparsed.ERROR\n\n # To keep the API consistent with previous releases, convert\n # deparse.offset values into NodeInfo items\n for tup, node in deparsed.offsets.items():\n deparsed.offsets[tup] = NodeInfo(node = node, start = node.start,\n finish = node.finish)\n\n deparsed.scanner = scanner\n return deparsed"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nliking deparse_code(), but given a function/module name and offset, finds the node closest to offset. If offset is not an instruction boundary, we raise an IndexError.", "response": "def code_deparse_around_offset(name, offset, co, out=StringIO(),\n version=None, is_pypy=None,\n debug_opts=DEFAULT_DEBUG_OPTS):\n \"\"\"\n Like deparse_code(), but given a function/module name and\n offset, finds the node closest to offset. If offset is not an instruction boundary,\n we raise an IndexError.\n \"\"\"\n assert iscode(co)\n\n if version is None:\n version = sysinfo2float()\n if is_pypy is None:\n is_pypy = IS_PYPY\n\n deparsed = code_deparse(co, out, version, is_pypy, debug_opts)\n if (name, offset) in deparsed.offsets.keys():\n # This is the easy case\n return deparsed\n\n valid_offsets = [t for t in deparsed.offsets if isinstance(t[1], int)]\n offset_list = sorted([t[1] for t in valid_offsets if t[0] == name])\n\n # FIXME: should check for branching?\n found_offset = find_gt(offset_list, offset)\n deparsed.offsets[name, offset] = deparsed.offsets[name, found_offset]\n return deparsed"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef deparsed_find(tup, deparsed, code):\n nodeInfo = None\n name, last_i = tup\n if (name, last_i) in deparsed.offsets.keys():\n nodeInfo = deparsed.offsets[name, last_i]\n else:\n from uncompyle6.scanner import get_scanner\n scanner = get_scanner(deparsed.version)\n co = code.co_code\n if op_at_code_loc(co, last_i, scanner.opc) == 'DUP_TOP':\n offset = deparsed.scanner.next_offset(co[last_i], last_i)\n if (name, offset) in deparsed.offsets:\n nodeInfo = deparsed.offsets[name, offset]\n\n return nodeInfo", "response": "Return a NodeInfo nametuple for a fragment - deparsed object at tup."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef table_r_node(self, node):\n start = len(self.f.getvalue())\n try:\n self.default(node)\n except GenericASTTraversalPruningException:\n final = len(self.f.getvalue())\n self.set_pos_info(node, start, final)\n self.set_pos_info(node[-1], start, final)\n raise GenericASTTraversalPruningException", "response": "General pattern where the last node should be should\n get the text span attributes of the entire tree"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef n_exec_stmt(self, node):\n start = len(self.f.getvalue()) + len(self.indent)\n try:\n super(FragmentsWalker, self).n_exec_stmt(node)\n except GenericASTTraversalPruningException:\n pass\n self.set_pos_info(node, start, len(self.f.getvalue()))\n self.set_pos_info(node[-1], start, len(self.f.getvalue()))\n self.prune()", "response": "A function that walks the tree for the EXEC_STMT node and sets the node s position and prune the tree."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nwalks the tree for list comprehensions.", "response": "def comprehension_walk3(self, node, iter_index, code_index=-5):\n \"\"\"\n List comprehensions the way they are done in Python3.\n They're more other comprehensions, e.g. set comprehensions\n See if we can combine code.\n \"\"\"\n p = self.prec\n self.prec = 27\n code = node[code_index].attr\n\n assert iscode(code), node[code_index]\n code_name = code.co_name\n code = Code(code, self.scanner, self.currentclass)\n\n ast = self.build_ast(code._tokens, code._customize)\n\n self.customize(code._customize)\n if ast[0] == 'sstmt':\n ast = ast[0]\n\n # skip over stmt return ret_expr\n ast = ast[0][0][0]\n store = None\n if ast in ['set_comp_func', 'dict_comp_func']:\n # Offset 0: BUILD_SET should have the span\n # of '{'\n self.gen_source(ast, code_name, {})\n for k in ast:\n if k == 'comp_iter':\n n = k\n elif k == 'store':\n store = k\n pass\n pass\n pass\n else:\n ast = ast[0][0]\n n = ast[iter_index]\n assert n == 'list_iter', n\n\n # FIXME: I'm not totally sure this is right.\n\n # Find the list comprehension body. It is the inner-most\n # node that is not list_.. .\n if_node = None\n comp_for = None\n comp_store = None\n if n == 'comp_iter':\n comp_for = n\n comp_store = ast[3]\n\n have_not = False\n while n in ('list_iter', 'comp_iter'):\n n = n[0] # iterate one nesting deeper\n if n in ('list_for', 'comp_for'):\n if n[2] == 'store':\n store = n[2]\n n = n[3]\n elif n in ('list_if', 'list_if_not', 'comp_if', 'comp_ifnot'):\n have_not = n in ('list_if_not', 'comp_ifnot')\n if_node = n[0]\n if n[1] == 'store':\n store = n[1]\n n = n[2]\n pass\n pass\n\n # Python 2.7+ starts including set_comp_body\n # Python 3.5+ starts including set_comp_func\n assert n.kind in ('lc_body', 'comp_body', 'set_comp_func', 'set_comp_body'), ast\n assert store, \"Couldn't find store in list/set comprehension\"\n\n old_name = self.name\n self.name = code_name\n\n # Issue created with later Python code generation is that there\n # is a lamda set up with a dummy argument name that is then called\n # So we can't just translate that as is but need to replace the\n # dummy name. Below we are picking out the variable name as seen\n # in the code. And trying to generate code for the other parts\n # that don't have the dummy argument name in it.\n # Another approach might be to be able to pass in the source name\n # for the dummy argument.\n\n self.preorder(n[0])\n gen_start = len(self.f.getvalue()) + 1\n self.write(' for ')\n start = len(self.f.getvalue())\n if comp_store:\n self.preorder(comp_store)\n else:\n self.preorder(store)\n\n self.set_pos_info(store, start, len(self.f.getvalue()))\n\n # FIXME this is all merely approximate\n # from trepan.api import debug; debug()\n self.write(' in ')\n start = len(self.f.getvalue())\n node[-3].parent = node\n self.preorder(node[-3])\n fin = len(self.f.getvalue())\n self.set_pos_info(node[-3], start, fin, old_name)\n\n if ast == 'list_comp':\n list_iter = ast[1]\n assert list_iter == 'list_iter'\n if list_iter == 'list_for':\n self.preorder(list_iter[3])\n self.prec = p\n return\n pass\n\n if comp_store:\n self.preorder(comp_for)\n elif if_node:\n self.write(' if ')\n if have_not:\n self.write('not ')\n self.preorder(if_node)\n pass\n self.prec = p\n self.name = old_name\n if node[-1].kind.startswith('CALL_FUNCTION'):\n self.set_pos_info(node[-1], gen_start, len(self.f.getvalue()))"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef setcomprehension_walk3(self, node, collection_index):\n p = self.prec\n self.prec = 27\n\n code = Code(node[1].attr, self.scanner, self.currentclass)\n ast = self.build_ast(code._tokens, code._customize)\n self.customize(code._customize)\n ast = ast[0][0][0]\n store = ast[3]\n collection = node[collection_index]\n\n n = ast[4]\n list_if = None\n assert n == 'comp_iter'\n\n # find innermost node\n while n == 'comp_iter':\n n = n[0] # recurse one step\n # FIXME: adjust for set comprehension\n if n == 'list_for':\n store = n[2]\n n = n[3]\n elif n in ('list_if', 'list_if_not', 'comp_if', 'comp_if_not'):\n # FIXME: just a guess\n if n[0].kind == 'expr':\n list_if = n\n else:\n list_if = n[1]\n n = n[2]\n pass\n pass\n\n assert n == 'comp_body', ast\n\n self.preorder(n[0])\n self.write(' for ')\n start = len(self.f.getvalue())\n self.preorder(store)\n self.set_pos_info(store, start, len(self.f.getvalue()))\n self.write(' in ')\n start = len(self.f.getvalue())\n self.preorder(collection)\n self.set_pos_info(collection, start, len(self.f.getvalue()))\n if list_if:\n start = len(self.f.getvalue())\n self.preorder(list_if)\n self.set_pos_info(list_if, start, len(self.f.getvalue()))\n self.prec = p", "response": "Walks the tree for list comprehensions and sets the comprehension_index to the index of the node."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nmaking sure each node has a parent.", "response": "def fixup_parents(self, node, parent):\n \"\"\"Make sure each node has a parent\"\"\"\n start, finish = 0, self.last_finish\n # We assume anything with a start has a finish.\n needs_range = not hasattr(node, 'start')\n\n if not hasattr(node, 'parent'):\n node.parent = parent\n\n for n in node:\n if needs_range and hasattr(n, 'start'):\n if n.start < start: start = n.start\n if n.finish > finish: finish = n.finish\n\n if hasattr(n, 'offset') and not hasattr(n, 'parent'):\n n.parent = node\n else:\n self.fixup_parents(n, node)\n pass\n pass\n if needs_range:\n node.start, node.finish = start, finish\n\n return"} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nadjust all offsets under node", "response": "def fixup_offsets(self, new_start, node):\n \"\"\"Adjust all offsets under node\"\"\"\n if hasattr(node, 'start'):\n node.start += new_start\n node.finish += new_start\n for n in node:\n if hasattr(n, 'offset'):\n if hasattr(n, 'start'):\n n.start += new_start\n n.finish += new_start\n else:\n self.fixup_offsets(new_start, n)\n return"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef set_pos_info_recurse(self, node, start, finish, parent=None):\n self.set_pos_info(node, start, finish)\n if parent is None:\n parent = node\n for n in node:\n n.parent = parent\n if hasattr(n, 'offset'):\n self.set_pos_info(n, start, finish)\n else:\n n.start = start\n n.finish = finish\n self.set_pos_info_recurse(n, start, finish, parent)\n return", "response": "Set positions under node recursively."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef write(self, *data):\n for l in data:\n ## print(\"XXX write: '%s'\" % l)\n for i in str(l):\n if i == '\\n':\n self.current_line_number += 1\n pass\n pass\n pass\n return super(LineMapWalker, self).write(*data)", "response": "Augment write routine to keep track of current line number"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\naugments write default routine to record line number changes", "response": "def default(self, node):\n \"\"\"Augment write default routine to record line number changes\"\"\"\n if hasattr(node, 'linestart'):\n if node.linestart:\n self.source_linemap[self.current_line_number] = node.linestart\n return super(LineMapWalker, self).default(node)"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef ingest(self, co, classname=None, code_objects={}, show_asm=None):\n\n if not show_asm:\n show_asm = self.show_asm\n\n bytecode = self.build_instructions(co)\n\n # show_asm = 'after'\n if show_asm in ('both', 'before'):\n for instr in bytecode.get_instructions(co):\n print(instr.disassemble())\n\n # Container for tokens\n tokens = []\n\n customize = {}\n if self.is_pypy:\n customize['PyPy'] = 1\n\n codelen = len(self.code)\n\n free, names, varnames = self.unmangle_code_names(co, classname)\n self.names = names\n\n # Scan for assertions. Later we will\n # turn 'LOAD_GLOBAL' to 'LOAD_ASSERT'.\n # 'LOAD_ASSERT' is used in assert statements.\n self.load_asserts = set()\n for i in self.op_range(0, codelen):\n # We need to detect the difference between:\n # raise AssertionError\n # and\n # assert ...\n if (self.code[i] == self.opc.JUMP_IF_TRUE and\n i + 4 < codelen and\n self.code[i+3] == self.opc.POP_TOP and\n self.code[i+4] == self.opc.LOAD_GLOBAL):\n if names[self.get_argument(i+4)] == 'AssertionError':\n self.load_asserts.add(i+4)\n\n jump_targets = self.find_jump_targets(show_asm)\n # contains (code, [addrRefToCode])\n\n last_stmt = self.next_stmt[0]\n i = self.next_stmt[last_stmt]\n replace = {}\n while i < codelen - 1:\n if self.lines[last_stmt].next > i:\n # Distinguish \"print ...\" from \"print ...,\"\n if self.code[last_stmt] == self.opc.PRINT_ITEM:\n if self.code[i] == self.opc.PRINT_ITEM:\n replace[i] = 'PRINT_ITEM_CONT'\n elif self.code[i] == self.opc.PRINT_NEWLINE:\n replace[i] = 'PRINT_NEWLINE_CONT'\n last_stmt = i\n i = self.next_stmt[i]\n\n extended_arg = 0\n for offset in self.op_range(0, codelen):\n op = self.code[offset]\n op_name = self.opname[op]\n oparg = None; pattr = None\n\n if offset in jump_targets:\n jump_idx = 0\n # We want to process COME_FROMs to the same offset to be in *descending*\n # offset order so we have the larger range or biggest instruction interval\n # last. (I think they are sorted in increasing order, but for safety\n # we sort them). That way, specific COME_FROM tags will match up\n # properly. For example, a \"loop\" with an \"if\" nested in it should have the\n # \"loop\" tag last so the grammar rule matches that properly.\n last_jump_offset = -1\n for jump_offset in sorted(jump_targets[offset], reverse=True):\n if jump_offset != last_jump_offset:\n tokens.append(Token(\n 'COME_FROM', jump_offset, repr(jump_offset),\n offset=\"%s_%d\" % (offset, jump_idx),\n has_arg = True))\n jump_idx += 1\n last_jump_offset = jump_offset\n elif offset in self.thens:\n tokens.append(Token(\n 'THEN', None, self.thens[offset],\n offset=\"%s_0\" % offset,\n has_arg = True))\n\n has_arg = (op >= self.opc.HAVE_ARGUMENT)\n if has_arg:\n oparg = self.get_argument(offset) + extended_arg\n extended_arg = 0\n if op == self.opc.EXTENDED_ARG:\n extended_arg = oparg * L65536\n continue\n if op in self.opc.CONST_OPS:\n const = co.co_consts[oparg]\n # We can't use inspect.iscode() because we may be\n # using a different version of Python than the\n # one that this was byte-compiled on. So the code\n # types may mismatch.\n if hasattr(const, 'co_name'):\n oparg = const\n if const.co_name == '':\n assert op_name == 'LOAD_CONST'\n op_name = 'LOAD_LAMBDA'\n elif const.co_name == self.genexpr_name:\n op_name = 'LOAD_GENEXPR'\n elif const.co_name == '':\n op_name = 'LOAD_DICTCOMP'\n elif const.co_name == '':\n op_name = 'LOAD_SETCOMP'\n # verify uses 'pattr' for comparison, since 'attr'\n # now holds Code(const) and thus can not be used\n # for comparison (todo: think about changing this)\n # pattr = 'code_object @ 0x%x %s->%s' % \\\n # (id(const), const.co_filename, const.co_name)\n pattr = ''\n else:\n if oparg < len(co.co_consts):\n argval, _ = _get_const_info(oparg, co.co_consts)\n # Why don't we use _ above for \"pattr\" rather than \"const\"?\n # This *is* a little hoaky, but we have to coordinate with\n # other parts like n_LOAD_CONST in pysource.py for example.\n pattr = const\n pass\n elif op in self.opc.NAME_OPS:\n pattr = names[oparg]\n elif op in self.opc.JREL_OPS:\n pattr = repr(offset + 3 + oparg)\n if op == self.opc.JUMP_FORWARD:\n target = self.get_target(offset)\n # FIXME: this is a hack to catch stuff like:\n # if x: continue\n # the \"continue\" is not on a new line.\n if len(tokens) and tokens[-1].kind == 'JUMP_BACK':\n tokens[-1].kind = intern('CONTINUE')\n\n elif op in self.opc.JABS_OPS:\n pattr = repr(oparg)\n elif op in self.opc.LOCAL_OPS:\n pattr = varnames[oparg]\n elif op in self.opc.COMPARE_OPS:\n pattr = self.opc.cmp_op[oparg]\n elif op in self.opc.FREE_OPS:\n pattr = free[oparg]\n if op in self.varargs_ops:\n # CE - Hack for >= 2.5\n # Now all values loaded via LOAD_CLOSURE are packed into\n # a tuple before calling MAKE_CLOSURE.\n if (self.version >= 2.5 and op == self.opc.BUILD_TUPLE and\n self.code[self.prev[offset]] == self.opc.LOAD_CLOSURE):\n continue\n else:\n op_name = '%s_%d' % (op_name, oparg)\n customize[op_name] = oparg\n elif self.version > 2.0 and op == self.opc.CONTINUE_LOOP:\n customize[op_name] = 0\n elif op_name in \"\"\"\n CONTINUE_LOOP EXEC_STMT LOAD_LISTCOMP LOAD_SETCOMP\n \"\"\".split():\n customize[op_name] = 0\n elif op == self.opc.JUMP_ABSOLUTE:\n # Further classify JUMP_ABSOLUTE into backward jumps\n # which are used in loops, and \"CONTINUE\" jumps which\n # may appear in a \"continue\" statement. The loop-type\n # and continue-type jumps will help us classify loop\n # boundaries The continue-type jumps help us get\n # \"continue\" statements with would otherwise be turned\n # into a \"pass\" statement because JUMPs are sometimes\n # ignored in rules as just boundary overhead. In\n # comprehensions we might sometimes classify JUMP_BACK\n # as CONTINUE, but that's okay since we add a grammar\n # rule for that.\n target = self.get_target(offset)\n if target <= offset:\n op_name = 'JUMP_BACK'\n if (offset in self.stmts\n and self.code[offset+3] not in (self.opc.END_FINALLY,\n self.opc.POP_BLOCK)):\n if ((offset in self.linestarts and\n tokens[-1].kind == 'JUMP_BACK')\n or offset not in self.not_continue):\n op_name = 'CONTINUE'\n else:\n # FIXME: this is a hack to catch stuff like:\n # if x: continue\n # the \"continue\" is not on a new line.\n if tokens[-1].kind == 'JUMP_BACK':\n # We need 'intern' since we have\n # already have processed the previous\n # token.\n tokens[-1].kind = intern('CONTINUE')\n\n elif op == self.opc.LOAD_GLOBAL:\n if offset in self.load_asserts:\n op_name = 'LOAD_ASSERT'\n elif op == self.opc.RETURN_VALUE:\n if offset in self.return_end_ifs:\n op_name = 'RETURN_END_IF'\n\n linestart = self.linestarts.get(offset, None)\n\n if offset not in replace:\n tokens.append(Token(\n op_name, oparg, pattr, offset, linestart, op,\n has_arg, self.opc))\n else:\n tokens.append(Token(\n replace[offset], oparg, pattr, offset, linestart, op,\n has_arg, self.opc))\n pass\n pass\n\n if show_asm in ('both', 'after'):\n for t in tokens:\n print(t.format(line_prefix='L.'))\n print()\n return tokens, customize", "response": "Ingests a code object into a list of tokens and returns a list of tokens and their associated custom grammars."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nsearching Syntax Tree node to find variable names that are global.", "response": "def find_all_globals(node, globs):\n \"\"\"Search Syntax Tree node to find variable names that are global.\"\"\"\n for n in node:\n if isinstance(n, SyntaxTree):\n globs = find_all_globals(n, globs)\n elif n.kind in read_write_global_ops:\n globs.add(n.pattr)\n return globs"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nsearching a node of parse tree to find variable names that need a global or nonlocal statements added.", "response": "def find_globals_and_nonlocals(node, globs, nonlocals, code, version):\n \"\"\"search a node of parse tree to find variable names that need a\n either 'global' or 'nonlocal' statements added.\"\"\"\n for n in node:\n if isinstance(n, SyntaxTree):\n globs, nonlocals = find_globals_and_nonlocals(n, globs, nonlocals,\n code, version)\n elif n.kind in read_global_ops:\n globs.add(n.pattr)\n elif (version >= 3.0\n and n.kind in nonglobal_ops\n and n.pattr in code.co_freevars\n and n.pattr != code.co_name\n and code.co_name != ''):\n nonlocals.add(n.pattr)\n return globs, nonlocals"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef flatten_list(node):\n flat_elems = []\n for elem in node:\n if elem == 'expr1024':\n for subelem in elem:\n assert subelem == 'expr32'\n for subsubelem in subelem:\n flat_elems.append(subsubelem)\n elif elem == 'expr32':\n for subelem in elem:\n assert subelem == 'expr'\n flat_elems.append(subelem)\n else:\n flat_elems.append(elem)\n pass\n pass\n return flat_elems", "response": "flattens a list of expressions into a single list of 32 and 1024 items."} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ntake a string or py3 bytes or a string", "response": "def encode_to_py3bytes_or_py2str(s):\n \"\"\" takes anything and attempts to return a py2 string or py3 bytes. this\n is typically used when creating command + arguments to be executed via\n os.exec* \"\"\"\n\n fallback_encoding = \"utf8\"\n\n if IS_PY3:\n # if we're already bytes, do nothing\n if isinstance(s, bytes):\n pass\n else:\n s = str(s)\n try:\n s = bytes(s, DEFAULT_ENCODING)\n except UnicodeEncodeError:\n s = bytes(s, fallback_encoding)\n else:\n # attempt to convert the thing to unicode from the system's encoding\n try:\n s = unicode(s, DEFAULT_ENCODING)\n # if the thing is already unicode, or it's a number, it can't be\n # coerced to unicode with an encoding argument, but if we leave out\n # the encoding argument, it will convert it to a string, then to unicode\n except TypeError:\n s = unicode(s)\n\n # now that we have guaranteed unicode, encode to our system encoding,\n # but attempt to fall back to something\n try:\n s = s.encode(DEFAULT_ENCODING)\n except:\n s = s.encode(fallback_encoding, \"replace\")\n return s"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_exc_from_name(name):\n\n exc = None\n try:\n return rc_exc_cache[name]\n except KeyError:\n m = rc_exc_regex.match(name)\n if m:\n base = m.group(1)\n rc_or_sig_name = m.group(2)\n\n if base == \"SignalException\":\n try:\n rc = -int(rc_or_sig_name)\n except ValueError:\n rc = -getattr(signal, rc_or_sig_name)\n else:\n rc = int(rc_or_sig_name)\n\n exc = get_rc_exc(rc)\n return exc", "response": "returns the corresponding exception from a given exception name"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn an exception for a given exit code or negative signal number", "response": "def get_rc_exc(rc):\n \"\"\" takes a exit code or negative signal number and produces an exception\n that corresponds to that return code. positive return codes yield\n ErrorReturnCode exception, negative return codes yield SignalException\n\n we also cache the generated exception so that only one signal of that type\n exists, preserving identity \"\"\"\n\n try:\n return rc_exc_cache[rc]\n except KeyError:\n pass\n\n if rc > 0:\n name = \"ErrorReturnCode_%d\" % rc\n base = ErrorReturnCode\n else:\n signame = SIGNAL_MAPPING[abs(rc)]\n name = \"SignalException_\" + signame\n base = SignalException\n\n exc = ErrorReturnCodeMeta(name, (base,), {\"exit_code\": rc})\n rc_exc_cache[rc] = exc\n return exc"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns the full path of the executable that is found in the PATH environment", "response": "def which(program, paths=None):\n \"\"\" takes a program name or full path, plus an optional collection of search\n paths, and returns the full path of the requested executable. if paths is\n specified, it is the entire list of search paths, and the PATH env is not\n used at all. otherwise, PATH env is used to look for the program \"\"\"\n\n def is_exe(fpath):\n return (os.path.exists(fpath) and\n os.access(fpath, os.X_OK) and\n os.path.isfile(os.path.realpath(fpath)))\n\n found_path = None\n fpath, fname = os.path.split(program)\n\n # if there's a path component, then we've specified a path to the program,\n # and we should just test if that program is executable. if it is, return\n if fpath:\n program = os.path.abspath(os.path.expanduser(program))\n if is_exe(program):\n found_path = program\n\n # otherwise, we've just passed in the program name, and we need to search\n # the paths to find where it actually lives\n else:\n paths_to_search = []\n\n if isinstance(paths, (tuple, list)):\n paths_to_search.extend(paths)\n else:\n env_paths = os.environ.get(\"PATH\", \"\").split(os.pathsep)\n paths_to_search.extend(env_paths)\n\n for path in paths_to_search:\n exe_file = os.path.join(path, program)\n if is_exe(exe_file):\n found_path = exe_file\n break\n\n return found_path"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nchecking if an object is a tty.", "response": "def ob_is_tty(ob):\n \"\"\" checks if an object (like a file-like object) is a tty. \"\"\"\n fileno = get_fileno(ob)\n is_tty = False\n if fileno:\n is_tty = os.isatty(fileno)\n return is_tty"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef compile_args(args, kwargs, sep, prefix):\n processed_args = []\n encode = encode_to_py3bytes_or_py2str\n\n # aggregate positional args\n for arg in args:\n if isinstance(arg, (list, tuple)):\n if isinstance(arg, GlobResults) and not arg:\n arg = [arg.path]\n\n for sub_arg in arg:\n processed_args.append(encode(sub_arg))\n elif isinstance(arg, dict):\n processed_args += aggregate_keywords(arg, sep, prefix, raw=True)\n else:\n processed_args.append(encode(arg))\n\n # aggregate the keyword arguments\n processed_args += aggregate_keywords(kwargs, sep, prefix)\n\n return processed_args", "response": "Takes args and kwargs and returns a list of all the arguments that can be executed into the command instance that can be executed by the command instance."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef aggregate_keywords(keywords, sep, prefix, raw=False):\n\n processed = []\n encode = encode_to_py3bytes_or_py2str\n\n for k, v in keywords.items():\n # we're passing a short arg as a kwarg, example:\n # cut(d=\"\\t\")\n if len(k) == 1:\n if v is not False:\n processed.append(encode(\"-\" + k))\n if v is not True:\n processed.append(encode(v))\n\n # we're doing a long arg\n else:\n if not raw:\n k = k.replace(\"_\", \"-\")\n\n if v is True:\n processed.append(encode(\"--\" + k))\n elif v is False:\n pass\n elif sep is None or sep == \" \":\n processed.append(encode(prefix + k))\n processed.append(encode(v))\n else:\n arg = encode(\"%s%s%s%s\" % (prefix, k, sep, v))\n processed.append(arg)\n\n return processed", "response": "Takes a dictionary of keywords and a separator and returns a list of the names of the a\n archive."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nsets the terminal size of a tty file descriptor.", "response": "def setwinsize(fd, rows_cols):\n \"\"\" set the terminal size of a tty file descriptor. borrowed logic\n from pexpect.py \"\"\"\n rows, cols = rows_cols\n TIOCSWINSZ = getattr(termios, 'TIOCSWINSZ', -2146929561)\n\n s = struct.pack('HHHH', rows, cols, 0, 0)\n fcntl.ioctl(fd, TIOCSWINSZ, s)"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nconstruct a closure for our streamreader callback", "response": "def construct_streamreader_callback(process, handler):\n \"\"\" here we're constructing a closure for our streamreader callback. this\n is used in the case that we pass a callback into _out or _err, meaning we\n want to our callback to handle each bit of output\n\n we construct the closure based on how many arguments it takes. the reason\n for this is to make it as easy as possible for people to use, without\n limiting them. a new user will assume the callback takes 1 argument (the\n data). as they get more advanced, they may want to terminate the process,\n or pass some stdin back, and will realize that they can pass a callback of\n more args \"\"\"\n\n\n # implied arg refers to the \"self\" that methods will pass in. we need to\n # account for this implied arg when figuring out what function the user\n # passed in based on number of args\n implied_arg = 0\n\n partial_args = 0\n handler_to_inspect = handler\n\n if isinstance(handler, partial):\n partial_args = len(handler.args)\n handler_to_inspect = handler.func\n\n if inspect.ismethod(handler_to_inspect):\n implied_arg = 1\n num_args = get_num_args(handler_to_inspect)\n\n else:\n if inspect.isfunction(handler_to_inspect):\n num_args = get_num_args(handler_to_inspect)\n\n # is an object instance with __call__ method\n else:\n implied_arg = 1\n num_args = get_num_args(handler_to_inspect.__call__)\n\n\n net_args = num_args - implied_arg - partial_args\n\n handler_args = ()\n\n # just the chunk\n if net_args == 1:\n handler_args = ()\n\n # chunk, stdin\n if net_args == 2:\n handler_args = (process.stdin,)\n\n # chunk, stdin, process\n elif net_args == 3:\n # notice we're only storing a weakref, to prevent cyclic references\n # (where the process holds a streamreader, and a streamreader holds a\n # handler-closure with a reference to the process\n handler_args = (process.stdin, weakref.ref(process))\n\n def fn(chunk):\n # this is pretty ugly, but we're evaluating the process at call-time,\n # because it's a weakref\n args = handler_args\n if len(args) == 2:\n args = (handler_args[0], handler_args[1]())\n return handler(chunk, *args)\n\n return fn"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef handle_process_exit_code(exit_code):\n # if we exited from a signal, let our exit code reflect that\n if os.WIFSIGNALED(exit_code):\n exit_code = -os.WTERMSIG(exit_code)\n # otherwise just give us a normal exit code\n elif os.WIFEXITED(exit_code):\n exit_code = os.WEXITSTATUS(exit_code)\n else:\n raise RuntimeError(\"Unknown child exit status!\")\n\n return exit_code", "response": "This function handles the process exit code of the child process."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef no_interrupt(syscall, *args, **kwargs):\n ret = None\n\n while True:\n try:\n ret = syscall(*args, **kwargs)\n except OSError as e:\n if e.errno == errno.EINTR:\n continue\n else:\n raise\n else:\n break\n\n return ret", "response": "a helper for making system calls immune to EINTR"} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nhandle the timeout logic", "response": "def background_thread(timeout_fn, timeout_event, handle_exit_code, is_alive,\n quit):\n \"\"\" handles the timeout logic \"\"\"\n\n # if there's a timeout event, loop \n if timeout_event:\n while not quit.is_set():\n timed_out = event_wait(timeout_event, 0.1)\n if timed_out:\n timeout_fn()\n break\n\n # handle_exit_code will be a function ONLY if our command was NOT waited on\n # as part of its spawning. in other words, it's probably a background\n # command\n #\n # this reports the exit code exception in our thread. it's purely for the\n # user's awareness, and cannot be caught or used in any way, so it's ok to\n # suppress this during the tests\n if handle_exit_code and not RUNNING_TESTS: # pragma: no cover\n alive = True\n while alive:\n quit.wait(1)\n alive, exit_code = is_alive()\n\n handle_exit_code(exit_code)"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef determine_how_to_read_input(input_obj):\n\n get_chunk = None\n\n if isinstance(input_obj, Queue):\n log_msg = \"queue\"\n get_chunk = get_queue_chunk_reader(input_obj)\n\n elif callable(input_obj):\n log_msg = \"callable\"\n get_chunk = get_callable_chunk_reader(input_obj)\n\n # also handles stringio\n elif hasattr(input_obj, \"read\"):\n log_msg = \"file descriptor\"\n get_chunk = get_file_chunk_reader(input_obj)\n\n elif isinstance(input_obj, basestring):\n log_msg = \"string\"\n get_chunk = get_iter_string_reader(input_obj)\n\n elif isinstance(input_obj, bytes):\n log_msg = \"bytes\"\n get_chunk = get_iter_string_reader(input_obj)\n\n elif isinstance(input_obj, GeneratorType):\n log_msg = \"generator\"\n get_chunk = get_iter_chunk_reader(iter(input_obj))\n\n else:\n try:\n it = iter(input_obj)\n except TypeError:\n raise Exception(\"unknown input object\")\n else:\n log_msg = \"general iterable\"\n get_chunk = get_iter_chunk_reader(it)\n\n return get_chunk, log_msg", "response": "return a function that knows how to read chunks of that input object"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_iter_string_reader(stdin):\n bufsize = 1024\n iter_str = (stdin[i:i + bufsize] for i in range(0, len(stdin), bufsize))\n return get_iter_chunk_reader(iter_str)", "response": "return an iterator that returns a chunk of a string every time it is\n called."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef pushd(path):\n orig_path = os.getcwd()\n os.chdir(path)\n try:\n yield\n finally:\n os.chdir(orig_path)", "response": "pushd changes the actual working directory for the duration of the\n context"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nallow us to temporarily override all the special keyword parameters in a with context", "response": "def args(**kwargs):\n \"\"\" allows us to temporarily override all the special keyword parameters in\n a with context \"\"\"\n\n kwargs_str = \",\".join([\"%s=%r\" % (k,v) for k,v in kwargs.items()])\n\n raise DeprecationWarning(\"\"\"\n\nsh.args() has been deprecated because it was never thread safe. use the\nfollowing instead:\n\n sh2 = sh({kwargs})\n sh2.your_command()\n\nor\n\n sh2 = sh({kwargs})\n from sh2 import your_command\n your_command()\n\n\"\"\".format(kwargs=kwargs_str))"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef register_importer():\n\n def test(importer):\n return importer.__class__.__name__ == ModuleImporterFromVariables.__name__\n already_registered = any([True for i in sys.meta_path if test(i)])\n\n if not already_registered:\n importer = ModuleImporterFromVariables(\n restrict_to=[\"SelfWrapper\"],\n )\n sys.meta_path.insert(0, importer)\n\n return not already_registered", "response": "registers our fancy importer that can let us import from a module name"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nwaiting for the running command to finish", "response": "def wait(self):\n \"\"\" waits for the running command to finish. this is called on all\n running commands, eventually, except for ones that run in the background\n \"\"\"\n if not self._process_completed:\n self._process_completed = True\n\n exit_code = self.process.wait()\n if self.process.timed_out:\n # if we timed out, our exit code represents a signal, which is\n # negative, so let's make it positive to store in our\n # TimeoutException\n raise TimeoutException(-exit_code)\n\n else:\n self.handle_command_exit_code(exit_code)\n \n # if an iterable command is using an instance of OProc for its stdin,\n # wait on it. the process is probably set to \"piped\", which means it\n # won't be waited on, which means exceptions won't propagate up to the\n # main thread. this allows them to bubble up\n if self.process._stdin_process:\n self.process._stdin_process.command.wait()\n\n self.log.info(\"process completed\")\n return self"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef handle_command_exit_code(self, code):\n ca = self.call_args\n exc_class = get_exc_exit_code_would_raise(code, ca[\"ok_code\"],\n ca[\"piped\"])\n if exc_class:\n exc = exc_class(self.ran, self.process.stdout, self.process.stderr,\n ca[\"truncate_exc\"])\n raise exc", "response": "handle_command_exit_code - handle the exit code of a command"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nallowing us to iterate over the output of our command", "response": "def next(self):\n \"\"\" allow us to iterate over the output of our command \"\"\"\n\n if self._stopped_iteration:\n raise StopIteration()\n\n # we do this because if get blocks, we can't catch a KeyboardInterrupt\n # so the slight timeout allows for that.\n while True:\n try:\n chunk = self.process._pipe_queue.get(True, 0.001)\n except Empty:\n if self.call_args[\"iter_noblock\"]:\n return errno.EWOULDBLOCK\n else:\n if chunk is None:\n self.wait()\n self._stopped_iteration = True\n raise StopIteration()\n try:\n return chunk.decode(self.call_args[\"encoding\"],\n self.call_args[\"decode_errors\"])\n except UnicodeDecodeError:\n return chunk"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _extract_call_args(kwargs):\n\n kwargs = kwargs.copy()\n call_args = {}\n for parg, default in Command._call_args.items():\n key = \"_\" + parg\n\n if key in kwargs:\n call_args[parg] = kwargs[key]\n del kwargs[key]\n\n invalid_kwargs = special_kwarg_validator(call_args,\n Command._kwarg_validators)\n\n if invalid_kwargs:\n exc_msg = []\n for args, error_msg in invalid_kwargs:\n exc_msg.append(\" %r: %s\" % (args, error_msg))\n exc_msg = \"\\n\".join(exc_msg)\n raise TypeError(\"Invalid special arguments:\\n\\n%s\\n\" % exc_msg)\n\n return call_args, kwargs", "response": "Extracts the arguments that were passed to a command s __call__ and returns a tuple of special keyword arguments args and kwargs that will go to the execd command."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef is_alive(self):\n\n if self.exit_code is not None:\n return False, self.exit_code\n\n # what we're doing here essentially is making sure that the main thread\n # (or another thread), isn't calling .wait() on the process. because\n # .wait() calls os.waitpid(self.pid, 0), we can't do an os.waitpid\n # here...because if we did, and the process exited while in this\n # thread, the main thread's os.waitpid(self.pid, 0) would raise OSError\n # (because the process ended in another thread).\n #\n # so essentially what we're doing is, using this lock, checking if\n # we're calling .wait(), and if we are, let .wait() get the exit code\n # and handle the status, otherwise let us do it.\n acquired = self._wait_lock.acquire(False)\n if not acquired:\n if self.exit_code is not None:\n return False, self.exit_code\n return True, self.exit_code\n\n try:\n # WNOHANG is just that...we're calling waitpid without hanging...\n # essentially polling the process. the return result is (0, 0) if\n # there's no process status, so we check that pid == self.pid below\n # in order to determine how to proceed\n pid, exit_code = no_interrupt(os.waitpid, self.pid, os.WNOHANG)\n if pid == self.pid:\n self.exit_code = handle_process_exit_code(exit_code)\n self._process_just_ended()\n\n return False, self.exit_code\n\n # no child process\n except OSError:\n return False, self.exit_code\n else:\n return True, self.exit_code\n finally:\n self._wait_lock.release()", "response": "Checks if our child process is still alive."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nwait for the process to complete returns the exit code", "response": "def wait(self):\n \"\"\" waits for the process to complete, handles the exit code \"\"\"\n\n self.log.debug(\"acquiring wait lock to wait for completion\")\n # using the lock in a with-context blocks, which is what we want if\n # we're running wait()\n with self._wait_lock:\n self.log.debug(\"got wait lock\")\n witnessed_end = False\n\n if self.exit_code is None:\n self.log.debug(\"exit code not set, waiting on pid\")\n pid, exit_code = no_interrupt(os.waitpid, self.pid, 0) # blocks\n self.exit_code = handle_process_exit_code(exit_code)\n witnessed_end = True\n\n else:\n self.log.debug(\"exit code already set (%d), no need to wait\",\n self.exit_code)\n\n self._quit_threads.set()\n\n # we may not have a thread for stdin, if the pipe has been connected\n # via _piped=\"direct\"\n if self._input_thread:\n self._input_thread.join()\n\n # wait, then signal to our output thread that the child process is\n # done, and we should have finished reading all the stdout/stderr\n # data that we can by now\n timer = threading.Timer(2.0, self._stop_output_event.set)\n timer.start()\n\n # wait for our stdout and stderr streamreaders to finish reading and\n # aggregating the process output\n self._output_thread.join()\n timer.cancel()\n\n self._background_thread.join()\n\n if witnessed_end:\n self._process_just_ended()\n\n return self.exit_code"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef write(self):\n\n # get_chunk may sometimes return bytes, and sometimes return strings\n # because of the nature of the different types of STDIN objects we\n # support\n try:\n chunk = self.get_chunk()\n if chunk is None:\n raise DoneReadingForever\n\n except DoneReadingForever:\n self.log.debug(\"done reading\")\n\n if self.tty_in:\n # EOF time\n try:\n char = termios.tcgetattr(self.stream)[6][termios.VEOF]\n except:\n char = chr(4).encode()\n\n # normally, one EOF should be enough to signal to an program\n # that is read()ing, to return 0 and be on your way. however,\n # some programs are misbehaved, like python3.1 and python3.2.\n # they don't stop reading sometimes after read() returns 0.\n # this can be demonstrated with the following program:\n #\n # import sys\n # sys.stdout.write(sys.stdin.read())\n #\n # then type 'a' followed by ctrl-d 3 times. in python\n # 2.6,2.7,3.3,3.4,3.5,3.6, it only takes 2 ctrl-d to terminate.\n # however, in python 3.1 and 3.2, it takes all 3.\n #\n # so here we send an extra EOF along, just in case. i don't\n # believe it can hurt anything\n os.write(self.stream, char)\n os.write(self.stream, char)\n\n return True\n\n except NotYetReadyToRead:\n self.log.debug(\"received no data\")\n return False\n\n # if we're not bytes, make us bytes\n if IS_PY3 and hasattr(chunk, \"encode\"):\n chunk = chunk.encode(self.encoding)\n\n for proc_chunk in self.stream_bufferer.process(chunk):\n self.log.debug(\"got chunk size %d: %r\", len(proc_chunk),\n proc_chunk[:30])\n\n self.log.debug(\"writing chunk to process\")\n try:\n os.write(self.stream, proc_chunk)\n except OSError:\n self.log.debug(\"OSError writing stdin chunk\")\n return True", "response": "This method attempts to get a chunk of data from our child process s stdin then write it to our child process s stdout then return True if we are done writing forever?"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef find_module(self, mod_fullname, path=None):\n\n parent_frame = inspect.currentframe().f_back\n while in_importlib(parent_frame):\n parent_frame = parent_frame.f_back\n\n # this line is saying \"hey, does mod_fullname exist as a name we've\n # defind previously?\" the purpose of this is to ensure that\n # mod_fullname is really a thing we've defined. if we haven't defined\n # it before, then we \"can't\" import from it\n module = fetch_module_from_frame(mod_fullname, parent_frame)\n if not module:\n return None\n\n # make sure it's a class we're allowed to import from\n if module.__class__.__name__ not in self.restrict_to:\n return None\n\n return self", "response": "find the module that contains the given module name"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nnormalizes semi - structured JSON data into a flat table.", "response": "def json_normalize(data, record_path=None, meta=None,\n meta_prefix=None,\n record_prefix=None,\n errors='raise',\n sep='.'):\n \"\"\"\n \"Normalize\" semi-structured JSON data into a flat table\n\n Parameters\n ----------\n data : dict or list of dicts\n Unserialized JSON objects\n record_path : string or list of strings, default None\n Path in each object to list of records. If not passed, data will be\n assumed to be an array of records\n meta : list of paths (string or list of strings), default None\n Fields to use as metadata for each record in resulting table\n record_prefix : string, default None\n If True, prefix records with dotted (?) path, e.g. foo.bar.field if\n path to records is ['foo', 'bar']\n meta_prefix : string, default None\n errors : {'raise', 'ignore'}, default 'raise'\n\n * 'ignore' : will ignore KeyError if keys listed in meta are not\n always present\n * 'raise' : will raise KeyError if keys listed in meta are not\n always present\n\n .. versionadded:: 0.20.0\n\n sep : string, default '.'\n Nested records will generate names separated by sep,\n e.g., for sep='.', { 'foo' : { 'bar' : 0 } } -> foo.bar\n\n .. versionadded:: 0.20.0\n\n\n Returns\n -------\n frame : DataFrame\n\n Examples\n --------\n\n >>> data = [{'state': 'Florida',\n ... 'shortname': 'FL',\n ... 'info': {\n ... 'governor': 'Rick Scott'\n ... },\n ... 'counties': [{'name': 'Dade', 'population': 12345},\n ... {'name': 'Broward', 'population': 40000},\n ... {'name': 'Palm Beach', 'population': 60000}]},\n ... {'state': 'Ohio',\n ... 'shortname': 'OH',\n ... 'info': {\n ... 'governor': 'John Kasich'\n ... },\n ... 'counties': [{'name': 'Summit', 'population': 1234},\n ... {'name': 'Cuyahoga', 'population': 1337}]}]\n >>> from pandas.io.json import json_normalize\n >>> result = json_normalize(data, 'counties', ['state', 'shortname',\n ... ['info', 'governor']])\n >>> result\n name population info.governor state shortname\n 0 Dade 12345 Rick Scott Florida FL\n 1 Broward 40000 Rick Scott Florida FL\n 2 Palm Beach 60000 Rick Scott Florida FL\n 3 Summit 1234 John Kasich Ohio OH\n 4 Cuyahoga 1337 John Kasich Ohio OH\n\n \"\"\"\n def _pull_field(js, spec):\n result = js\n if isinstance(spec, list):\n for field in spec:\n result = result[field]\n else:\n result = result[spec]\n\n return result\n\n if isinstance(data, list) and len(data) is 0:\n return DataFrame()\n\n # A bit of a hackjob\n if isinstance(data, dict):\n data = [data]\n\n if record_path is None:\n if any([isinstance(x, dict) for x in compat.itervalues(data[0])]):\n # naive normalization, this is idempotent for flat records\n # and potentially will inflate the data considerably for\n # deeply nested structures:\n # {VeryLong: { b: 1,c:2}} -> {VeryLong.b:1 ,VeryLong.c:@}\n #\n # TODO: handle record value which are lists, at least error\n # reasonably\n data = nested_to_record(data, sep=sep)\n return DataFrame(data)\n elif not isinstance(record_path, list):\n record_path = [record_path]\n\n if meta is None:\n meta = []\n elif not isinstance(meta, list):\n meta = [meta]\n\n for i, x in enumerate(meta):\n if not isinstance(x, list):\n meta[i] = [x]\n\n # Disastrously inefficient for now\n records = []\n lengths = []\n\n meta_vals = defaultdict(list)\n if not isinstance(sep, compat.string_types):\n sep = str(sep)\n meta_keys = [sep.join(val) for val in meta]\n\n def _recursive_extract(data, path, seen_meta, level=0):\n if len(path) > 1:\n for obj in data:\n for val, key in zip(meta, meta_keys):\n if level + 1 == len(val):\n seen_meta[key] = _pull_field(obj, val[-1])\n\n _recursive_extract(obj[path[0]], path[1:],\n seen_meta, level=level + 1)\n else:\n for obj in data:\n recs = _pull_field(obj, path[0])\n\n # For repeating the metadata later\n lengths.append(len(recs))\n\n for val, key in zip(meta, meta_keys):\n if level + 1 > len(val):\n meta_val = seen_meta[key]\n else:\n try:\n meta_val = _pull_field(obj, val[level:])\n except KeyError as e:\n if errors == 'ignore':\n meta_val = np.nan\n else:\n raise \\\n KeyError(\"Try running with \"\n \"errors='ignore' as key \"\n \"%s is not always present\", e)\n meta_vals[key].append(meta_val)\n\n records.extend(recs)\n\n _recursive_extract(data, record_path, {}, level=0)\n\n result = DataFrame(records)\n\n if record_prefix is not None:\n result.rename(columns=lambda x: record_prefix + x, inplace=True)\n\n # Data types, a problem\n for k, v in compat.iteritems(meta_vals):\n if meta_prefix is not None:\n k = meta_prefix + k\n\n if k in result:\n raise ValueError('Conflicting metadata name %s, '\n 'need distinguishing prefix ' % k)\n\n result[k] = np.array(v).repeat(lengths)\n\n return result"} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nconvert a NumPy dtype to its corresponding json_table.", "response": "def as_json_table_type(x):\n \"\"\"\n Convert a NumPy / pandas type to its corresponding json_table.\n\n Parameters\n ----------\n x : array or dtype\n\n Returns\n -------\n t : str\n the Table Schema data types\n\n Notes\n -----\n This table shows the relationship between NumPy / pandas dtypes,\n and Table Schema dtypes.\n\n ============== =================\n Pandas type Table Schema type\n ============== =================\n int64 integer\n float64 number\n bool boolean\n datetime64[ns] datetime\n timedelta64[ns] duration\n object str\n categorical any\n =============== =================\n \"\"\"\n if is_integer_dtype(x):\n return 'integer'\n elif is_bool_dtype(x):\n return 'boolean'\n elif is_numeric_dtype(x):\n return 'number'\n elif (is_datetime64_dtype(x) or is_datetime64tz_dtype(x)):\n return 'datetime'\n elif is_timedelta64_dtype(x):\n return 'duration'\n elif is_categorical_dtype(x):\n return 'any'\n elif is_string_dtype(x):\n return 'string'\n else:\n return 'any'"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nset index names to index or level_x for regular or Multi", "response": "def set_default_names(data):\n \"\"\"Sets index names to 'index' for regular, or 'level_x' for Multi\"\"\"\n if all(name is not None for name in data.index.names):\n return data\n\n data = data.copy()\n if data.index.nlevels > 1:\n names = [name if name is not None else 'level_{}'.format(i)\n for i, name in enumerate(data.index.names)]\n data.index.names = names\n else:\n data.index.name = data.index.name or 'index'\n return data"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef read_json(path_or_buf=None, orient=None, typ='frame', dtype=True,\n convert_axes=True, convert_dates=True, keep_default_dates=True,\n numpy=False, precise_float=False, date_unit=None, encoding=None,\n lines=False):\n \"\"\"\n Convert a JSON string to pandas object\n\n Parameters\n ----------\n path_or_buf : a valid JSON string or file-like, default: None\n The string could be a URL. Valid URL schemes include http, ftp, s3, and\n file. For file URLs, a host is expected. For instance, a local file\n could be ``file://localhost/path/to/table.json``\n\n orient : string,\n Indication of expected JSON string format.\n Compatible JSON strings can be produced by ``to_json()`` with a\n corresponding orient value.\n The set of possible orients is:\n\n - ``'split'`` : dict like\n ``{index -> [index], columns -> [columns], data -> [values]}``\n - ``'records'`` : list like\n ``[{column -> value}, ... , {column -> value}]``\n - ``'index'`` : dict like ``{index -> {column -> value}}``\n - ``'columns'`` : dict like ``{column -> {index -> value}}``\n - ``'values'`` : just the values array\n\n The allowed and default values depend on the value\n of the `typ` parameter.\n\n * when ``typ == 'series'``,\n\n - allowed orients are ``{'split','records','index'}``\n - default is ``'index'``\n - The Series index must be unique for orient ``'index'``.\n\n * when ``typ == 'frame'``,\n\n - allowed orients are ``{'split','records','index',\n 'columns','values'}``\n - default is ``'columns'``\n - The DataFrame index must be unique for orients ``'index'`` and\n ``'columns'``.\n - The DataFrame columns must be unique for orients ``'index'``,\n ``'columns'``, and ``'records'``.\n\n typ : type of object to recover (series or frame), default 'frame'\n dtype : boolean or dict, default True\n If True, infer dtypes, if a dict of column to dtype, then use those,\n if False, then don't infer dtypes at all, applies only to the data.\n convert_axes : boolean, default True\n Try to convert the axes to the proper dtypes.\n convert_dates : boolean, default True\n List of columns to parse for dates; If True, then try to parse\n datelike columns default is True; a column label is datelike if\n\n * it ends with ``'_at'``,\n\n * it ends with ``'_time'``,\n\n * it begins with ``'timestamp'``,\n\n * it is ``'modified'``, or\n\n * it is ``'date'``\n\n keep_default_dates : boolean, default True\n If parsing dates, then parse the default datelike columns\n numpy : boolean, default False\n Direct decoding to numpy arrays. Supports numeric data only, but\n non-numeric column and index labels are supported. Note also that the\n JSON ordering MUST be the same for each term if numpy=True.\n precise_float : boolean, default False\n Set to enable usage of higher precision (strtod) function when\n decoding string to double values. Default (False) is to use fast but\n less precise builtin functionality\n date_unit : string, default None\n The timestamp unit to detect if converting dates. The default behaviour\n is to try and detect the correct precision, but if this is not desired\n then pass one of 's', 'ms', 'us' or 'ns' to force parsing only seconds,\n milliseconds, microseconds or nanoseconds respectively.\n lines : boolean, default False\n Read the file as a json object per line.\n\n .. versionadded:: 0.19.0\n\n encoding : str, default is 'utf-8'\n The encoding to use to decode py3 bytes.\n\n .. versionadded:: 0.19.0\n\n Returns\n -------\n result : Series or DataFrame, depending on the value of `typ`.\n\n See Also\n --------\n DataFrame.to_json\n\n Examples\n --------\n\n >>> df = pd.DataFrame([['a', 'b'], ['c', 'd']],\n ... index=['row 1', 'row 2'],\n ... columns=['col 1', 'col 2'])\n\n Encoding/decoding a Dataframe using ``'split'`` formatted JSON:\n\n >>> df.to_json(orient='split')\n '{\"columns\":[\"col 1\",\"col 2\"],\n \"index\":[\"row 1\",\"row 2\"],\n \"data\":[[\"a\",\"b\"],[\"c\",\"d\"]]}'\n >>> pd.read_json(_, orient='split')\n col 1 col 2\n row 1 a b\n row 2 c d\n\n Encoding/decoding a Dataframe using ``'index'`` formatted JSON:\n\n >>> df.to_json(orient='index')\n '{\"row 1\":{\"col 1\":\"a\",\"col 2\":\"b\"},\"row 2\":{\"col 1\":\"c\",\"col 2\":\"d\"}}'\n >>> pd.read_json(_, orient='index')\n col 1 col 2\n row 1 a b\n row 2 c d\n\n Encoding/decoding a Dataframe using ``'records'`` formatted JSON.\n Note that index labels are not preserved with this encoding.\n\n >>> df.to_json(orient='records')\n '[{\"col 1\":\"a\",\"col 2\":\"b\"},{\"col 1\":\"c\",\"col 2\":\"d\"}]'\n >>> pd.read_json(_, orient='records')\n col 1 col 2\n 0 a b\n 1 c d\n\n Encoding with Table Schema\n\n >>> df.to_json(orient='table')\n '{\"schema\": {\"fields\": [{\"name\": \"index\", \"type\": \"string\"},\n {\"name\": \"col 1\", \"type\": \"string\"},\n {\"name\": \"col 2\", \"type\": \"string\"}],\n \"primaryKey\": \"index\",\n \"pandas_version\": \"0.20.0\"},\n \"data\": [{\"index\": \"row 1\", \"col 1\": \"a\", \"col 2\": \"b\"},\n {\"index\": \"row 2\", \"col 1\": \"c\", \"col 2\": \"d\"}]}'\n \"\"\"\n\n filepath_or_buffer, _, _ = get_filepath_or_buffer(path_or_buf,\n encoding=encoding)\n if isinstance(filepath_or_buffer, compat.string_types):\n try:\n exists = os.path.exists(filepath_or_buffer)\n\n # if the filepath is too long will raise here\n # 5874\n except (TypeError, ValueError):\n exists = False\n\n if exists:\n fh, handles = _get_handle(filepath_or_buffer, 'r',\n encoding=encoding)\n json = fh.read()\n fh.close()\n else:\n json = filepath_or_buffer\n elif hasattr(filepath_or_buffer, 'read'):\n json = filepath_or_buffer.read()\n else:\n json = filepath_or_buffer\n\n if lines:\n # If given a json lines file, we break the string into lines, add\n # commas and put it in a json list to make a valid json object.\n lines = list(StringIO(json.strip()))\n json = '[' + ','.join(lines) + ']'\n\n obj = None\n if typ == 'frame':\n obj = FrameParser(json, orient, dtype, convert_axes, convert_dates,\n keep_default_dates, numpy, precise_float,\n date_unit).parse()\n\n if typ == 'series' or obj is None:\n if not isinstance(dtype, bool):\n dtype = dict(data=dtype)\n obj = SeriesParser(json, orient, dtype, convert_axes, convert_dates,\n keep_default_dates, numpy, precise_float,\n date_unit).parse()\n\n return obj", "response": "Read a JSON string or file - like object into pandas. Series."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _try_convert_data(self, name, data, use_dtypes=True,\n convert_dates=True):\n \"\"\" try to parse a ndarray like into a column by inferring dtype \"\"\"\n\n # don't try to coerce, unless a force conversion\n if use_dtypes:\n if self.dtype is False:\n return data, False\n elif self.dtype is True:\n pass\n\n else:\n\n # dtype to force\n dtype = (self.dtype.get(name)\n if isinstance(self.dtype, dict) else self.dtype)\n if dtype is not None:\n try:\n dtype = np.dtype(dtype)\n return data.astype(dtype), True\n except:\n return data, False\n\n if convert_dates:\n new_data, result = self._try_convert_to_date(data)\n if result:\n return new_data, True\n\n result = False\n\n if data.dtype == 'object':\n\n # try float\n try:\n data = data.astype('float64')\n result = True\n except:\n pass\n\n if data.dtype.kind == 'f':\n\n if data.dtype != 'float64':\n\n # coerce floats to 64\n try:\n data = data.astype('float64')\n result = True\n except:\n pass\n\n # do't coerce 0-len data\n if len(data) and (data.dtype == 'float' or data.dtype == 'object'):\n\n # coerce ints if we can\n try:\n new_data = data.astype('int64')\n if (new_data == data).all():\n data = new_data\n result = True\n except:\n pass\n\n # coerce ints to 64\n if data.dtype == 'int':\n\n # coerce floats to 64\n try:\n data = data.astype('int64')\n result = True\n except:\n pass\n\n return data, result", "response": "try to convert a data array like into a column by inferring dtype"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ntry to parse a numpy array like into a date column", "response": "def _try_convert_to_date(self, data):\n \"\"\" try to parse a ndarray like into a date column\n try to coerce object in epoch/iso formats and\n integer/float in epcoh formats, return a boolean if parsing\n was successful \"\"\"\n\n # no conversion on empty\n if not len(data):\n return data, False\n\n new_data = data\n if new_data.dtype == 'object':\n try:\n new_data = data.astype('int64')\n except:\n pass\n\n # ignore numbers that are out of range\n if issubclass(new_data.dtype.type, np.number):\n in_range = (isnull(new_data.values) | (new_data > self.min_stamp) |\n (new_data.values == iNaT))\n if not in_range.all():\n return data, False\n\n date_units = (self.date_unit,) if self.date_unit else self._STAMP_UNITS\n for date_unit in date_units:\n try:\n new_data = to_datetime(new_data, errors='raise',\n unit=date_unit)\n except ValueError:\n continue\n except:\n break\n return new_data, True\n return data, False"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nsets the default options for the current kernel.", "response": "def set_defaults(show_toolbar=None,\n precision=None,\n grid_options=None,\n column_options=None):\n \"\"\"\n Set the default qgrid options. The options that you can set here are the\n same ones that you can pass into ``QgridWidget`` constructor, with the\n exception of the ``df`` option, for which a default value wouldn't be\n particularly useful (since the purpose of qgrid is to display a DataFrame).\n\n See the documentation for ``QgridWidget`` for more information.\n\n Notes\n -----\n This function will be useful to you if you find yourself\n setting the same options every time you create a QgridWidget. Calling\n this ``set_defaults`` function once sets the options for the lifetime of\n the kernel, so you won't have to include the same options every time you\n instantiate a ``QgridWidget``.\n\n See Also\n --------\n QgridWidget :\n The widget whose default behavior is changed by ``set_defaults``.\n \"\"\"\n defaults.set_defaults(show_toolbar=show_toolbar,\n precision=precision,\n grid_options=grid_options,\n column_options=column_options)"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef enable(dataframe=True, series=True):\n try:\n from IPython.core.getipython import get_ipython\n except ImportError:\n raise ImportError('This feature requires IPython 1.0+')\n\n ip = get_ipython()\n ip_formatter = ip.display_formatter.ipython_display_formatter\n\n if dataframe:\n ip_formatter.for_type(pd.DataFrame, _display_as_qgrid)\n else:\n ip_formatter.type_printers.pop(pd.DataFrame, None)\n\n if series:\n ip_formatter.for_type(pd.Series, _display_as_qgrid)\n else:\n ip_formatter.type_printers.pop(pd.Series)", "response": "Enables or disables the display of the current node."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef show_grid(data_frame,\n show_toolbar=None,\n precision=None,\n grid_options=None,\n column_options=None,\n column_definitions=None,\n row_edit_callback=None):\n \"\"\"\n Renders a DataFrame or Series as an interactive qgrid, represented by\n an instance of the ``QgridWidget`` class. The ``QgridWidget`` instance\n is constructed using the options passed in to this function. The\n ``data_frame`` argument to this function is used as the ``df`` kwarg in\n call to the QgridWidget constructor, and the rest of the parameters\n are passed through as is.\n\n If the ``data_frame`` argument is a Series, it will be converted to a\n DataFrame before being passed in to the QgridWidget constructor as the\n ``df`` kwarg.\n\n :rtype: QgridWidget\n\n Parameters\n ----------\n data_frame : DataFrame\n The DataFrame that will be displayed by this instance of\n QgridWidget.\n grid_options : dict\n Options to use when creating the SlickGrid control (i.e. the\n interactive grid). See the Notes section below for more information\n on the available options, as well as the default options that this\n widget uses.\n precision : integer\n The number of digits of precision to display for floating-point\n values. If unset, we use the value of\n `pandas.get_option('display.precision')`.\n show_toolbar : bool\n Whether to show a toolbar with options for adding/removing rows.\n Adding/removing rows is an experimental feature which only works\n with DataFrames that have an integer index.\n column_options : dict\n Column options that are to be applied to every column. See the\n Notes section below for more information on the available options,\n as well as the default options that this widget uses.\n column_definitions : dict\n Column options that are to be applied to individual\n columns. The keys of the dict should be the column names, and each\n value should be the column options for a particular column,\n represented as a dict. The available options for each column are the\n same options that are available to be set for all columns via the\n ``column_options`` parameter. See the Notes section below for more\n information on those options.\n row_edit_callback : callable\n A callable that is called to determine whether a particular row\n should be editable or not. Its signature should be\n ``callable(row)``, where ``row`` is a dictionary which contains a\n particular row's values, keyed by column name. The callback should\n return True if the provided row should be editable, and False\n otherwise.\n\n\n Notes\n -----\n The following dictionary is used for ``grid_options`` if none are\n provided explicitly::\n\n {\n # SlickGrid options\n 'fullWidthRows': True,\n 'syncColumnCellResize': True,\n 'forceFitColumns': True,\n 'defaultColumnWidth': 150,\n 'rowHeight': 28,\n 'enableColumnReorder': False,\n 'enableTextSelectionOnCells': True,\n 'editable': True,\n 'autoEdit': False,\n 'explicitInitialization': True,\n\n # Qgrid options\n 'maxVisibleRows': 15,\n 'minVisibleRows': 8,\n 'sortable': True,\n 'filterable': True,\n 'highlightSelectedCell': False,\n 'highlightSelectedRow': True\n }\n\n The first group of options are SlickGrid \"grid options\" which are\n described in the `SlickGrid documentation\n `_.\n\n The second group of option are options that were added specifically\n for Qgrid and therefore are not documented in the SlickGrid documentation.\n The following bullet points describe these options.\n\n * **maxVisibleRows** The maximum number of rows that Qgrid will show.\n * **minVisibleRows** The minimum number of rows that Qgrid will show\n * **sortable** Whether the Qgrid instance will allow the user to sort\n columns by clicking the column headers. When this is set to ``False``,\n nothing will happen when users click the column headers.\n * **filterable** Whether the Qgrid instance will allow the user to filter\n the grid. When this is set to ``False`` the filter icons won't be shown\n for any columns.\n * **highlightSelectedCell** If you set this to True, the selected cell\n will be given a light blue border.\n * **highlightSelectedRow** If you set this to False, the light blue\n background that's shown by default for selected rows will be hidden.\n\n The following dictionary is used for ``column_options`` if none are\n provided explicitly::\n\n {\n # SlickGrid column options\n 'defaultSortAsc': True,\n 'maxWidth': None,\n 'minWidth': 30,\n 'resizable': True,\n 'sortable': True,\n 'toolTip': \"\",\n 'width': None\n\n # Qgrid column options\n 'editable': True,\n }\n\n The first group of options are SlickGrid \"column options\" which are\n described in the `SlickGrid documentation\n `_.\n\n The ``editable`` option was added specifically for Qgrid and therefore is\n not documented in the SlickGrid documentation. This option specifies\n whether a column should be editable or not.\n\n See Also\n --------\n set_defaults : Permanently set global defaults for the parameters\n of ``show_grid``, with the exception of the ``data_frame``\n and ``column_definitions`` parameters, since those\n depend on the particular set of data being shown by an\n instance, and therefore aren't parameters we would want\n to set for all QgridWidet instances.\n set_grid_option : Permanently set global defaults for individual\n grid options. Does so by changing the defaults\n that the ``show_grid`` method uses for the\n ``grid_options`` parameter.\n QgridWidget : The widget class that is instantiated and returned by this\n method.\n\n \"\"\"\n\n if show_toolbar is None:\n show_toolbar = defaults.show_toolbar\n if precision is None:\n precision = defaults.precision\n if not isinstance(precision, Integral):\n raise TypeError(\"precision must be int, not %s\" % type(precision))\n if column_options is None:\n column_options = defaults.column_options\n else:\n options = defaults.column_options.copy()\n options.update(column_options)\n column_options = options\n if grid_options is None:\n grid_options = defaults.grid_options\n else:\n options = defaults.grid_options.copy()\n options.update(grid_options)\n grid_options = options\n if not isinstance(grid_options, dict):\n raise TypeError(\n \"grid_options must be dict, not %s\" % type(grid_options)\n )\n\n # if a Series is passed in, convert it to a DataFrame\n if isinstance(data_frame, pd.Series):\n data_frame = pd.DataFrame(data_frame)\n elif not isinstance(data_frame, pd.DataFrame):\n raise TypeError(\n \"data_frame must be DataFrame or Series, not %s\" % type(data_frame)\n )\n\n column_definitions = (column_definitions or {})\n\n # create a visualization for the dataframe\n return QgridWidget(df=data_frame, precision=precision,\n grid_options=grid_options,\n column_options=column_options,\n column_definitions=column_definitions,\n row_edit_callback=row_edit_callback,\n show_toolbar=show_toolbar)", "response": "Renders a Qgrid widget with the specified options and options."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_themes():\n styles_dir = os.path.join(package_dir, 'styles')\n themes = [os.path.basename(theme).replace('.less', '')\n for theme in glob('{0}/*.less'.format(styles_dir))]\n return themes", "response": "Returns a list of available themes"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ninstall theme to jupyter_customcss with specified font fontsize and md layout and toolbar pref", "response": "def install_theme(theme=None,\n monofont=None,\n monosize=11,\n nbfont=None,\n nbfontsize=13,\n tcfont=None,\n tcfontsize=13,\n dffontsize=93,\n outfontsize=85,\n mathfontsize=100,\n margins='auto',\n cellwidth='980',\n lineheight=170,\n cursorwidth=2,\n cursorcolor='default',\n altprompt=False,\n altmd=False,\n altout=False,\n hideprompt=False,\n vimext=False,\n toolbar=False,\n nbname=False,\n kernellogo=False,\n dfonts=False):\n\n \"\"\" Install theme to jupyter_customcss with specified font, fontsize,\n md layout, and toolbar pref\n \"\"\"\n # get working directory\n wkdir = os.path.abspath('./')\n\n stylefx.reset_default(False)\n stylefx.check_directories()\n\n doc = '\\nConcatenated font imports, .less styles, & custom variables\\n'\n s = '*' * 65\n style_less = '\\n'.join(['/*', s, s, doc, s, s, '*/'])\n style_less += '\\n\\n\\n'\n style_less += '/* Import Notebook, Markdown, & Code Fonts */\\n'\n\n # initialize style_less & style_css\n style_less = stylefx.set_font_properties(\n style_less=style_less,\n monofont=monofont,\n monosize=monosize,\n nbfont=nbfont,\n nbfontsize=nbfontsize,\n tcfont=tcfont,\n tcfontsize=tcfontsize,\n dffontsize=dffontsize,\n outfontsize=outfontsize,\n mathfontsize=mathfontsize,\n dfonts=dfonts)\n\n if theme is not None:\n # define some vars for cell layout\n cursorcolor = stylefx.get_colors(theme=theme, c=cursorcolor)\n style_less = stylefx.style_layout(\n style_less,\n theme=theme,\n cellwidth=cellwidth,\n margins=margins,\n lineheight=lineheight,\n altprompt=altprompt,\n altmd=altmd,\n altout=altout,\n hideprompt=hideprompt,\n cursorwidth=cursorwidth,\n cursorcolor=cursorcolor,\n vimext=vimext,\n toolbar=toolbar,\n nbname=nbname,\n kernellogo=kernellogo)\n\n # compile tempfile.less to css code and append to style_css\n style_css = stylefx.less_to_css(style_less)\n\n # append mathjax css & script to style_css\n style_css = stylefx.set_mathjax_style(style_css, mathfontsize)\n\n # install style_css to .jupyter/custom/custom.css\n stylefx.write_final_css(style_css)\n\n # change back to original working directory\n os.chdir(wkdir)"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ninferring theme from css file", "response": "def infer_theme():\n \"\"\" checks jupyter_config_dir() for text file containing theme name\n (updated whenever user installs a new theme)\n \"\"\"\n themes = [os.path.basename(theme).replace('.less', '')\n for theme in glob('{0}/*.less'.format(styles_dir))]\n if os.path.exists(theme_name_file):\n with open(theme_name_file) as f:\n theme = f.readlines()[0]\n if theme not in themes:\n theme = 'default'\n else:\n theme = 'default'\n\n return theme"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nset the style of the current context.", "response": "def set_style(rcdict, theme=None, grid=True, gridlines=u'-', ticks=False, spines=True):\n \"\"\"\n This code has been modified from seaborn.rcmod.set_style()\n ::Arguments::\n rcdict (str): dict of \"context\" properties (filled by set_context())\n theme (str): name of theme to use when setting color properties\n grid (bool): turns off axis grid if False (default: True)\n ticks (bool): removes x,y axis ticks if True (default: False)\n spines (bool): removes axis spines if False (default: True)\n \"\"\"\n\n # extract style and color info for theme\n styleMap, clist = get_theme_style(theme)\n\n # extract style variables\n figureFace = styleMap['figureFace']\n axisFace = styleMap['axisFace']\n textColor = styleMap['textColor']\n edgeColor = styleMap['edgeColor']\n gridColor = styleMap['gridColor']\n\n if not spines:\n edgeColor = 'none'\n\n style_dict = {\n 'figure.edgecolor': figureFace,\n 'figure.facecolor': figureFace,\n 'axes.facecolor': axisFace,\n 'axes.edgecolor': edgeColor,\n 'axes.labelcolor': textColor,\n 'axes.grid': grid,\n 'grid.linestyle': gridlines,\n 'grid.color': gridColor,\n 'text.color': textColor,\n 'xtick.color': textColor,\n 'ytick.color': textColor,\n 'patch.edgecolor': axisFace,\n 'patch.facecolor': gridColor,\n 'savefig.facecolor': figureFace,\n 'savefig.edgecolor': figureFace}\n\n # update rcdict with style params\n rcdict.update(style_dict)\n\n # Show or hide the axes ticks\n if ticks:\n rcdict.update({\n \"xtick.major.size\": 6,\n \"ytick.major.size\": 6,\n \"xtick.minor.size\": 3,\n \"ytick.minor.size\": 3})\n\n base_style.update(rcdict)\n\n # update matplotlib with rcdict (incl. context, font, & style)\n mpl.rcParams.update(rcdict)\n\n # update seaborn with rcdict (incl. context, font, & style)\n try:\n import seaborn as sns\n sns.set_style(rc=rcdict)\n except Exception:\n pass\n\n try:\n from cycler import cycler\n # set color cycle to jt-style color list\n mpl.rcParams['axes.prop_cycle'] = cycler(color=clist)\n except Exception:\n pass\n\n # replace default blue, green, etc. with jt colors\n for code, color in zip(\"bgrmyck\", clist[:7]):\n rgb = mpl.colors.colorConverter.to_rgb(color)\n mpl.colors.colorConverter.colors[code] = rgb\n mpl.colors.colorConverter.cache[code] = rgb"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef set_context(context='paper', fscale=1., figsize=(8., 7.)):\n # scale all the parameters by the same factor depending on the context\n scaling = dict(paper=.8, notebook=1, talk=1.3, poster=1.6)[context]\n context_dict = {k: v * scaling for k, v in base_context.items()}\n\n # scale default figsize\n figX, figY = figsize\n context_dict[\"figure.figsize\"] = (figX*scaling, figY*scaling)\n\n # independently scale the fonts\n font_dict = {k: v * fscale for k, v in base_font.items()}\n font_dict[\"font.family\"] = [\"sans-serif\"]\n font_dict[\"font.sans-serif\"] = [\"Helvetica\", \"Helvetica Neue\", \"Arial\",\n \"DejaVu Sans\", \"Liberation Sans\", \"sans-serif\"]\n context_dict.update(font_dict)\n return context_dict", "response": "Sets the context of the current page."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nread - in theme style info and populate styleMap and clist", "response": "def get_theme_style(theme):\n \"\"\"\n read-in theme style info and populate styleMap (dict of with mpl.rcParams)\n and clist (list of hex codes passed to color cylcler)\n ::Arguments::\n theme (str): theme name\n ::Returns::\n styleMap (dict): dict containing theme-specific colors for figure properties\n clist (list): list of colors to replace mpl's default color_cycle\n \"\"\"\n styleMap, clist = get_default_jtstyle()\n if theme == 'default':\n return styleMap, clist\n\n syntaxVars = ['@yellow:', '@orange:', '@red:', '@magenta:', '@violet:', '@blue:', '@cyan:', '@green:']\n\n get_hex_code = lambda line: line.split(':')[-1].split(';')[0][-7:]\n\n themeFile = os.path.join(styles_dir, theme+'.less')\n with open(themeFile) as f:\n for line in f:\n for k, v in styleMap.items():\n if k in line.strip():\n styleMap[k] = get_hex_code(line)\n for c in syntaxVars:\n if c in line.strip():\n syntaxVars[syntaxVars.index(c)] = get_hex_code(line)\n\n # remove duplicate hexcolors\n syntaxVars = list(set(syntaxVars))\n clist.extend(syntaxVars)\n clist = remove_non_colors(clist)\n return styleMap, clist"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef reset():\n colors = [(0., 0., 1.), (0., .5, 0.), (1., 0., 0.), (.75, .75, 0.),\n (.75, .75, 0.), (0., .75, .75), (0., 0., 0.)]\n for code, color in zip(\"bgrmyck\", colors):\n rgb = mpl.colors.colorConverter.to_rgb(color)\n mpl.colors.colorConverter.colors[code] = rgb\n mpl.colors.colorConverter.cache[code] = rgb\n mpl.rcParams.update(mpl.rcParamsDefault)\n mpl.rcParams['figure.facecolor'] = 'white'\n mpl.rcParams['axes.facecolor'] = 'white'", "response": "full reset of matplotlib default style and colors\n "} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nwrite less - compiled css file to jupyter_customcss in jupyter_dir", "response": "def less_to_css(style_less):\n \"\"\" write less-compiled css file to jupyter_customcss in jupyter_dir\n \"\"\"\n with fileOpen(tempfile, 'w') as f:\n f.write(style_less)\n os.chdir(package_dir)\n style_css = lesscpy.compile(tempfile)\n style_css += '\\n\\n'\n return style_css"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef set_font_properties(style_less,\n nbfont=None,\n tcfont=None,\n monofont=None,\n monosize=11,\n tcfontsize=13,\n nbfontsize=13,\n prfontsize=95,\n dffontsize=93,\n outfontsize=85,\n mathfontsize=100,\n dfonts=False):\n \"\"\"Parent function for setting notebook, text/md, and\n codecell font-properties\n \"\"\"\n\n fontsizes = [monosize, nbfontsize, tcfontsize, prfontsize, dffontsize, outfontsize]\n monosize, nbfontsize, tcfontsize, prfontsize, dffontsize, outfontsize = convert_fontsizes(fontsizes)\n if dfonts==True:\n monofont, tcfont, nbfont = ['monospace', 'sans-serif', 'sans-serif']\n else:\n if monofont is not None:\n monofont, monofpath = stored_font_dicts(monofont)\n style_less = import_fonts(style_less, monofont, monofpath)\n else:\n monofont='monospace'\n if tcfont is not None:\n tcfont, tcfontpath = stored_font_dicts(tcfont)\n style_less = import_fonts(style_less, tcfont, tcfontpath)\n else:\n tcfont='sans-serif'\n if nbfont is not None:\n if nbfont == 'proxima':\n nbfont, tcfont = [\"'Proxima Nova'\"]*2\n style_less = proxima_nova_imports(style_less)\n else:\n nbfont, nbfontpath = stored_font_dicts(nbfont)\n style_less = import_fonts(style_less, nbfont, nbfontpath)\n else:\n nbfont='sans-serif'\n\n style_less += '/* Set Font-Type and Font-Size Variables */\\n'\n # font names and fontfamily info for codecells, notebook & textcells\n style_less += '@monofont: {}; \\n'.format(monofont)\n style_less += '@notebook-fontfamily: {}; \\n'.format(nbfont)\n style_less += '@text-cell-fontfamily: {}; \\n'.format(tcfont)\n # font size for codecells, main notebook, notebook-sub, & textcells\n style_less += '@monofontsize: {}pt; \\n'.format(monosize)\n style_less += '@monofontsize-sub: {}pt; \\n'.format(float(monosize) - 1)\n style_less += '@nb-fontsize: {}pt; \\n'.format(nbfontsize)\n style_less += '@nb-fontsize-sub: {}pt; \\n'.format(float(nbfontsize) - 1)\n style_less += '@text-cell-fontsize: {}pt; \\n'.format(tcfontsize)\n style_less += '@df-header-fontsize: {}pt; \\n'.format(float(dffontsize) + 1)\n style_less += '@df-fontsize: {}pt; \\n'.format(dffontsize)\n style_less += '@output-font-size: {}pt; \\n'.format(outfontsize)\n style_less += '@prompt-fontsize: {}pt; \\n'.format(prfontsize)\n style_less += '@mathfontsize: {}%; \\n'.format(mathfontsize)\n style_less += '\\n\\n'\n style_less += '/* Import Theme Colors and Define Layout Variables */\\n'\n return style_less", "response": "Private function to set font properties for a notebook and codecell."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef import_fonts(style_less, fontname, font_subdir):\n\n ftype_dict = {'woff2': 'woff2',\n 'woff': 'woff',\n 'ttf': 'truetype',\n 'otf': 'opentype',\n 'svg': 'svg'}\n\n define_font = (\n \"@font-face {{font-family: {fontname};\\n\\tfont-weight:\"\n \"{weight};\\n\\tfont-style: {style};\\n\\tsrc: local('{fontname}'),\"\n \"\\n\\turl('fonts{sepp}{fontfile}') format('{ftype}');}}\\n\")\n fontname = fontname.split(',')[0]\n fontpath = os.path.join(fonts_dir, font_subdir)\n for fontfile in os.listdir(fontpath):\n if '.txt' in fontfile or 'DS_' in fontfile:\n continue\n weight = 'normal'\n style = 'normal'\n if 'medium' in fontfile:\n weight = 'medium'\n elif 'ital' in fontfile:\n style = 'italic'\n ft = ftype_dict[fontfile.split('.')[-1]]\n style_less += define_font.format(\n fontname=fontname,\n weight=weight,\n style=style,\n sepp='/',\n fontfile=fontfile,\n ftype=ft)\n send_fonts_to_jupyter(os.path.join(fontpath, fontfile))\n\n return style_less", "response": "Copy all custom fonts to ~/.jupyter / custom / fonts and write import statements to style_less"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef style_layout(style_less,\n theme='grade3',\n cursorwidth=2,\n cursorcolor='default',\n cellwidth='980',\n lineheight=170,\n margins='auto',\n vimext=False,\n toolbar=False,\n nbname=False,\n kernellogo=False,\n altprompt=False,\n altmd=False,\n altout=False,\n hideprompt=False):\n \"\"\"Set general layout and style properties of text and code cells\"\"\"\n\n # write theme name to ~/.jupyter/custom/ (referenced by jtplot.py)\n with fileOpen(theme_name_file, 'w') as f:\n f.write(theme)\n\n if (os.path.isdir(styles_dir_user) and\n '{}.less'.format(theme) in os.listdir(styles_dir_user)):\n theme_relpath = os.path.relpath(\n os.path.join(styles_dir_user, theme), package_dir)\n else:\n theme_relpath = os.path.relpath(\n os.path.join(styles_dir, theme), package_dir)\n\n style_less += '@import \"{}\";\\n'.format(theme_relpath)\n\n textcell_bg = '@cc-input-bg'\n promptText = '@input-prompt'\n promptBG = '@cc-input-bg'\n promptPadding = '.25em'\n promptBorder = '2px solid @prompt-line'\n tcPromptBorder = '2px solid @tc-prompt-std'\n promptMinWidth = 11.5\n outpromptMinWidth = promptMinWidth # remove + 3 since it will overlay output print() text\n tcPromptWidth = promptMinWidth + 3\n tcPromptFontsize = \"@prompt-fontsize\"\n ccOutputBG = '@cc-output-bg-default'\n\n if theme == 'grade3':\n textcell_bg = '@notebook-bg'\n if altprompt:\n promptPadding = '.1em'\n promptMinWidth = 8\n outpromptMinWidth = promptMinWidth + 3\n tcPromptWidth = promptMinWidth + 3\n promptText = 'transparent'\n tcPromptBorder = '2px solid transparent'\n if altmd:\n textcell_bg = '@notebook-bg'\n tcPromptBorder = '2px dotted @tc-border-selected'\n if altout:\n ccOutputBG = '@notebook-bg'\n if margins != 'auto':\n margins = '{}px'.format(margins)\n if '%' not in cellwidth:\n cellwidth = str(cellwidth) + 'px'\n\n style_less += '@container-margins: {};\\n'.format(margins)\n style_less += '@cell-width: {}; \\n'.format(cellwidth)\n style_less += '@cc-line-height: {}%; \\n'.format(lineheight)\n style_less += '@text-cell-bg: {}; \\n'.format(textcell_bg)\n style_less += '@cc-prompt-width: {}ex; \\n'.format(promptMinWidth)\n style_less += '@cc-prompt-bg: {}; \\n'.format(promptBG)\n style_less += '@cc-output-bg: {}; \\n'.format(ccOutputBG)\n style_less += '@prompt-text: {}; \\n'.format(promptText)\n style_less += '@prompt-padding: {}; \\n'.format(promptPadding)\n style_less += '@prompt-border: {}; \\n'.format(promptBorder)\n style_less += '@prompt-min-width: {}ex; \\n'.format(promptMinWidth)\n style_less += '@out-prompt-min-width: {}ex; \\n'.format(outpromptMinWidth)\n style_less += '@tc-prompt-width: {}ex; \\n'.format(tcPromptWidth)\n style_less += '@tc-prompt-border: {}; \\n'.format(tcPromptBorder)\n style_less += '@cursor-width: {}px; \\n'.format(cursorwidth)\n style_less += '@cursor-info: @cursor-width solid {}; \\n'.format(\n cursorcolor)\n style_less += '@tc-prompt-fontsize: {}; \\n'.format(tcPromptFontsize)\n style_less += '\\n\\n'\n\n # read-in notebook.less (general nb style)\n with fileOpen(nb_style, 'r') as notebook:\n style_less += notebook.read() + '\\n'\n\n # read-in cells.less (cell layout)\n with fileOpen(cl_style, 'r') as cells:\n style_less += cells.read() + '\\n'\n\n # read-in extras.less (misc layout)\n with fileOpen(ex_style, 'r') as extras:\n style_less += extras.read() + '\\n'\n\n # read-in codemirror.less (syntax-highlighting)\n with fileOpen(cm_style, 'r') as codemirror:\n style_less += codemirror.read() + '\\n'\n with fileOpen(comp_style, 'r') as codemirror:\n style_less += codemirror.read() + '\\n'\n\n style_less += toggle_settings(\n toolbar, nbname, hideprompt, kernellogo) + '\\n'\n if vimext:\n set_vim_style(theme)\n\n return style_less", "response": "Set general layout and style properties of text and code cells."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef toggle_settings(\n toolbar=False, nbname=False, hideprompt=False, kernellogo=False):\n \"\"\"Toggle main notebook toolbar (e.g., buttons), filename,\n and kernel logo.\"\"\"\n\n toggle = ''\n if toolbar:\n toggle += 'div#maintoolbar {margin-left: 8px !important;}\\n'\n toggle += '.toolbar.container {width: 100% !important;}\\n'\n else:\n toggle += 'div#maintoolbar {display: none !important;}\\n'\n if nbname:\n toggle += ('span.save_widget span.filename {margin-left: 8px; height: initial;'\n 'font-size: 100%; color: @nb-name-fg; background-color:'\n '@cc-input-bg;}\\n')\n toggle += ('span.save_widget span.filename:hover {color:'\n '@nb-name-hover; background-color: @cc-input-bg;}\\n')\n toggle += ('#menubar {padding-top: 4px; background-color:'\n '@notebook-bg;}\\n')\n else:\n toggle += '#header-container {display: none !important;}\\n'\n if hideprompt:\n toggle += 'div.prompt.input_prompt {display: none !important;}\\n'\n toggle += 'div.prompt.output_prompt {width: 5ex !important;}\\n'\n toggle += 'div.out_prompt_overlay.prompt:hover {width: 5ex !important; min-width: 5ex !important;}\\n'\n toggle += (\n '.CodeMirror-gutters, .cm-s-ipython .CodeMirror-gutters'\n '{ position: absolute; left: 0; top: 0; z-index: 3; width: 2em; '\n 'display: inline-block !important; }\\n')\n toggle += ('div.cell.code_cell .input { border-left: 5px solid @cm-gutters !important; border-bottom-left-radius: 5px; border-top-left-radius: 5px; }\\n')\n if kernellogo:\n toggle += '@kernel-logo-display: block;'\n else:\n toggle += '@kernel-logo-display: none;'\n\n return toggle", "response": "Toggle main notebook toolbar filename and kernel logo."} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nwrite mathjax settings set math fontsize", "response": "def set_mathjax_style(style_css, mathfontsize):\n \"\"\"Write mathjax settings, set math fontsize\n \"\"\"\n\n jax_style = \"\"\"\n \"\"\" % (int(mathfontsize), '\"{}%\"'.format(str(mathfontsize)))\n\n style_css += jax_style\n return style_css"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef set_vim_style(theme):\n\n vim_jupyter_nbext = os.path.join(jupyter_nbext, 'vim_binding')\n\n if not os.path.isdir(vim_jupyter_nbext):\n os.makedirs(vim_jupyter_nbext)\n\n vim_less = '@import \"styles{}\";\\n'.format(''.join([os.sep, theme]))\n\n with open(vim_style, 'r') as vimstyle:\n vim_less += vimstyle.read() + '\\n'\n with open(vimtemp, 'w') as vtemp:\n vtemp.write(vim_less)\n os.chdir(package_dir)\n vim_css = lesscpy.compile(vimtemp)\n vim_css += '\\n\\n'\n\n # install vim_custom_css to ...nbextensions/vim_binding/vim_binding.css\n vim_custom_css = os.path.join(vim_jupyter_nbext, 'vim_binding.css')\n with open(vim_custom_css, 'w') as vim_custom:\n vim_custom.write(vim_css)", "response": "Add style and compatibility with vim notebook extension"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nremoves custom. css and custom fonts", "response": "def reset_default(verbose=False):\n \"\"\"Remove custom.css and custom fonts\"\"\"\n paths = [jupyter_custom, jupyter_nbext]\n\n for fpath in paths:\n custom = '{0}{1}{2}.css'.format(fpath, os.sep, 'custom')\n try:\n os.remove(custom)\n except Exception:\n pass\n try:\n delete_font_files()\n except Exception:\n check_directories()\n delete_font_files()\n\n copyfile(defaultCSS, jupyter_customcss)\n copyfile(defaultJS, jupyter_customjs)\n\n if os.path.exists(theme_name_file):\n os.remove(theme_name_file)\n\n if verbose:\n print(\"Reset css and font defaults in:\\n{} &\\n{}\".format(*paths))"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef set_nb_theme(name):\n from IPython.core.display import HTML\n styles_dir = os.path.join(package_dir, 'styles/compiled/')\n css_path = glob('{0}/{1}.css'.format(styles_dir, name))[0]\n customcss = open(css_path, \"r\").read()\n\n return HTML(''.join(['']))", "response": "Set theme from within notebook"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\noverrides pycodestyle s function to provide indentation information.", "response": "def continued_indentation(logical_line, tokens, indent_level, hang_closing,\n indent_char, noqa):\n \"\"\"Override pycodestyle's function to provide indentation information.\"\"\"\n first_row = tokens[0][2][0]\n nrows = 1 + tokens[-1][2][0] - first_row\n if noqa or nrows == 1:\n return\n\n # indent_next tells us whether the next block is indented. Assuming\n # that it is indented by 4 spaces, then we should not allow 4-space\n # indents on the final continuation line. In turn, some other\n # indents are allowed to have an extra 4 spaces.\n indent_next = logical_line.endswith(':')\n\n row = depth = 0\n valid_hangs = (\n (DEFAULT_INDENT_SIZE,)\n if indent_char != '\\t' else (DEFAULT_INDENT_SIZE,\n 2 * DEFAULT_INDENT_SIZE)\n )\n\n # Remember how many brackets were opened on each line.\n parens = [0] * nrows\n\n # Relative indents of physical lines.\n rel_indent = [0] * nrows\n\n # For each depth, collect a list of opening rows.\n open_rows = [[0]]\n # For each depth, memorize the hanging indentation.\n hangs = [None]\n\n # Visual indents.\n indent_chances = {}\n last_indent = tokens[0][2]\n indent = [last_indent[1]]\n\n last_token_multiline = None\n line = None\n last_line = ''\n last_line_begins_with_multiline = False\n for token_type, text, start, end, line in tokens:\n\n newline = row < start[0] - first_row\n if newline:\n row = start[0] - first_row\n newline = (not last_token_multiline and\n token_type not in (tokenize.NL, tokenize.NEWLINE))\n last_line_begins_with_multiline = last_token_multiline\n\n if newline:\n # This is the beginning of a continuation line.\n last_indent = start\n\n # Record the initial indent.\n rel_indent[row] = pycodestyle.expand_indent(line) - indent_level\n\n # Identify closing bracket.\n close_bracket = (token_type == tokenize.OP and text in ']})')\n\n # Is the indent relative to an opening bracket line?\n for open_row in reversed(open_rows[depth]):\n hang = rel_indent[row] - rel_indent[open_row]\n hanging_indent = hang in valid_hangs\n if hanging_indent:\n break\n if hangs[depth]:\n hanging_indent = (hang == hangs[depth])\n\n visual_indent = (not close_bracket and hang > 0 and\n indent_chances.get(start[1]))\n\n if close_bracket and indent[depth]:\n # Closing bracket for visual indent.\n if start[1] != indent[depth]:\n yield (start, 'E124 {}'.format(indent[depth]))\n elif close_bracket and not hang:\n # closing bracket matches indentation of opening bracket's line\n if hang_closing:\n yield (start, 'E133 {}'.format(indent[depth]))\n elif indent[depth] and start[1] < indent[depth]:\n # Visual indent is broken.\n yield (start, 'E128 {}'.format(indent[depth]))\n elif (hanging_indent or\n (indent_next and\n rel_indent[row] == 2 * DEFAULT_INDENT_SIZE)):\n # Hanging indent is verified.\n if close_bracket and not hang_closing:\n yield (start, 'E123 {}'.format(indent_level +\n rel_indent[open_row]))\n hangs[depth] = hang\n elif visual_indent is True:\n # Visual indent is verified.\n indent[depth] = start[1]\n elif visual_indent in (text, unicode):\n # Ignore token lined up with matching one from a previous line.\n pass\n else:\n one_indented = (indent_level + rel_indent[open_row] +\n DEFAULT_INDENT_SIZE)\n # Indent is broken.\n if hang <= 0:\n error = ('E122', one_indented)\n elif indent[depth]:\n error = ('E127', indent[depth])\n elif not close_bracket and hangs[depth]:\n error = ('E131', one_indented)\n elif hang > DEFAULT_INDENT_SIZE:\n error = ('E126', one_indented)\n else:\n hangs[depth] = hang\n error = ('E121', one_indented)\n\n yield (start, '{} {}'.format(*error))\n\n # Look for visual indenting.\n if (\n parens[row] and\n token_type not in (tokenize.NL, tokenize.COMMENT) and\n not indent[depth]\n ):\n indent[depth] = start[1]\n indent_chances[start[1]] = True\n # Deal with implicit string concatenation.\n elif (token_type in (tokenize.STRING, tokenize.COMMENT) or\n text in ('u', 'ur', 'b', 'br')):\n indent_chances[start[1]] = unicode\n # Special case for the \"if\" statement because len(\"if (\") is equal to\n # 4.\n elif not indent_chances and not row and not depth and text == 'if':\n indent_chances[end[1] + 1] = True\n elif text == ':' and line[end[1]:].isspace():\n open_rows[depth].append(row)\n\n # Keep track of bracket depth.\n if token_type == tokenize.OP:\n if text in '([{':\n depth += 1\n indent.append(0)\n hangs.append(None)\n if len(open_rows) == depth:\n open_rows.append([])\n open_rows[depth].append(row)\n parens[row] += 1\n elif text in ')]}' and depth > 0:\n # Parent indents should not be more than this one.\n prev_indent = indent.pop() or last_indent[1]\n hangs.pop()\n for d in range(depth):\n if indent[d] > prev_indent:\n indent[d] = 0\n for ind in list(indent_chances):\n if ind >= prev_indent:\n del indent_chances[ind]\n del open_rows[depth + 1:]\n depth -= 1\n if depth:\n indent_chances[indent[depth]] = True\n for idx in range(row, -1, -1):\n if parens[idx]:\n parens[idx] -= 1\n break\n assert len(indent) == depth + 1\n if (\n start[1] not in indent_chances and\n # This is for purposes of speeding up E121 (GitHub #90).\n not last_line.rstrip().endswith(',')\n ):\n # Allow to line up tokens.\n indent_chances[start[1]] = text\n\n last_token_multiline = (start[0] != end[0])\n if last_token_multiline:\n rel_indent[end[0] - first_row] = rel_indent[row]\n\n last_line = line\n\n if (\n indent_next and\n not last_line_begins_with_multiline and\n pycodestyle.expand_indent(line) == indent_level + DEFAULT_INDENT_SIZE\n ):\n pos = (start[0], indent[0] + 4)\n desired_indent = indent_level + 2 * DEFAULT_INDENT_SIZE\n if visual_indent:\n yield (pos, 'E129 {}'.format(desired_indent))\n else:\n yield (pos, 'E125 {}'.format(desired_indent))"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_w605_position(tokens):\n # TODO: When this PR(*) change is released, use pos of pycodestyle\n # *: https://github.com/PyCQA/pycodestyle/pull/747\n valid = [\n '\\n', '\\\\', '\\'', '\"', 'a', 'b', 'f', 'n', 'r', 't', 'v',\n '0', '1', '2', '3', '4', '5', '6', '7', 'x',\n\n # Escape sequences only recognized in string literals\n 'N', 'u', 'U',\n ]\n\n for token_type, text, start_pos, _end_pos, _line in tokens:\n if token_type == tokenize.STRING:\n quote = text[-3:] if text[-3:] in ('\"\"\"', \"'''\") else text[-1]\n # Extract string modifiers (e.g. u or r)\n quote_pos = text.index(quote)\n prefix = text[:quote_pos].lower()\n start = quote_pos + len(quote)\n string = text[start:-len(quote)]\n\n if 'r' not in prefix:\n pos = string.find('\\\\')\n while pos >= 0:\n pos += 1\n if string[pos] not in valid:\n yield (\n # No need to search line, token stores position\n start_pos[1],\n \"W605 invalid escape sequence '\\\\%s'\" %\n string[pos],\n )\n pos = string.find('\\\\', pos + 1)", "response": "workaround get pointing out position by W605."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_module_imports_on_top_of_file(source, import_line_index):\n def is_string_literal(line):\n if line[0] in 'uUbB':\n line = line[1:]\n if line and line[0] in 'rR':\n line = line[1:]\n return line and (line[0] == '\"' or line[0] == \"'\")\n\n def is_future_import(line):\n nodes = ast.parse(line)\n for n in nodes.body:\n if isinstance(n, ast.ImportFrom) and n.module == '__future__':\n return True\n return False\n\n allowed_try_keywords = ('try', 'except', 'else', 'finally')\n in_docstring = False\n docstring_kind = '\"\"\"'\n for cnt, line in enumerate(source):\n if not in_docstring:\n m = DOCSTRING_START_REGEX.match(line.lstrip())\n if m is not None:\n in_docstring = True\n docstring_kind = m.group('kind')\n remain = line[m.end(): m.endpos].rstrip()\n if remain[-3:] == docstring_kind: # one line doc\n in_docstring = False\n continue\n if in_docstring:\n if line.rstrip()[-3:] == docstring_kind:\n in_docstring = False\n continue\n\n if not line.rstrip():\n continue\n elif line.startswith('#'):\n continue\n\n if line.startswith('import ') or line.startswith('from '):\n if cnt == import_line_index or is_future_import(line):\n continue\n return cnt\n elif pycodestyle.DUNDER_REGEX.match(line):\n continue\n elif any(line.startswith(kw) for kw in allowed_try_keywords):\n continue\n elif is_string_literal(line):\n return cnt\n else:\n return cnt\n return 0", "response": "returns a list of modules that are imported on top of a file"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_fixed_long_line(target, previous_line, original,\n indent_word=' ', max_line_length=79,\n aggressive=False, experimental=False, verbose=False):\n \"\"\"Break up long line and return result.\n\n Do this by generating multiple reformatted candidates and then\n ranking the candidates to heuristically select the best option.\n\n \"\"\"\n indent = _get_indentation(target)\n source = target[len(indent):]\n assert source.lstrip() == source\n assert not target.lstrip().startswith('#')\n\n # Check for partial multiline.\n tokens = list(generate_tokens(source))\n\n candidates = shorten_line(\n tokens, source, indent,\n indent_word,\n max_line_length,\n aggressive=aggressive,\n experimental=experimental,\n previous_line=previous_line)\n\n # Also sort alphabetically as a tie breaker (for determinism).\n candidates = sorted(\n sorted(set(candidates).union([target, original])),\n key=lambda x: line_shortening_rank(\n x,\n indent_word,\n max_line_length,\n experimental=experimental))\n\n if verbose >= 4:\n print(('-' * 79 + '\\n').join([''] + candidates + ['']),\n file=wrap_output(sys.stderr, 'utf-8'))\n\n if candidates:\n best_candidate = candidates[0]\n\n # Don't allow things to get longer.\n if longest_line_length(best_candidate) > longest_line_length(original):\n return None\n\n return best_candidate", "response": "Break up long line and return best candidate."} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns single line based on logical line input.", "response": "def join_logical_line(logical_line):\n \"\"\"Return single line based on logical line input.\"\"\"\n indentation = _get_indentation(logical_line)\n\n return indentation + untokenize_without_newlines(\n generate_tokens(logical_line.lstrip())) + '\\n'"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn source code based on tokens.", "response": "def untokenize_without_newlines(tokens):\n \"\"\"Return source code based on tokens.\"\"\"\n text = ''\n last_row = 0\n last_column = -1\n\n for t in tokens:\n token_string = t[1]\n (start_row, start_column) = t[2]\n (end_row, end_column) = t[3]\n\n if start_row > last_row:\n last_column = 0\n if (\n (start_column > last_column or token_string == '\\n') and\n not text.endswith(' ')\n ):\n text += ' '\n\n if token_string != '\\n':\n text += token_string\n\n last_row = end_row\n last_column = end_column\n\n return text.rstrip()"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning the logical line corresponding to the result.", "response": "def _get_logical(source_lines, result, logical_start, logical_end):\n \"\"\"Return the logical line corresponding to the result.\n\n Assumes input is already E702-clean.\n\n \"\"\"\n row = result['line'] - 1\n col = result['column'] - 1\n ls = None\n le = None\n for i in range(0, len(logical_start), 1):\n assert logical_end\n x = logical_end[i]\n if x[0] > row or (x[0] == row and x[1] > col):\n le = x\n ls = logical_start[i]\n break\n if ls is None:\n return None\n original = source_lines[ls[0]:le[0] + 1]\n return ls, le, original"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef code_almost_equal(a, b):\n split_a = split_and_strip_non_empty_lines(a)\n split_b = split_and_strip_non_empty_lines(b)\n\n if len(split_a) != len(split_b):\n return False\n\n for (index, _) in enumerate(split_a):\n if ''.join(split_a[index].split()) != ''.join(split_b[index].split()):\n return False\n\n return True", "response": "Return True if code is similar."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef fix_2to3(source,\n aggressive=True, select=None, ignore=None, filename='',\n where='global', verbose=False):\n \"\"\"Fix various deprecated code (via lib2to3).\"\"\"\n if not aggressive:\n return source\n\n select = select or []\n ignore = ignore or []\n\n return refactor(source,\n code_to_2to3(select=select,\n ignore=ignore,\n where=where,\n verbose=verbose),\n filename=filename)", "response": "Fix various deprecated code via lib2to3."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef find_newline(source):\n assert not isinstance(source, unicode)\n\n counter = collections.defaultdict(int)\n for line in source:\n if line.endswith(CRLF):\n counter[CRLF] += 1\n elif line.endswith(CR):\n counter[CR] += 1\n elif line.endswith(LF):\n counter[LF] += 1\n\n return (sorted(counter, key=counter.get, reverse=True) or [LF])[0]", "response": "Return type of newline used in source."} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn text of unified diff between old and new.", "response": "def get_diff_text(old, new, filename):\n \"\"\"Return text of unified diff between old and new.\"\"\"\n newline = '\\n'\n diff = difflib.unified_diff(\n old, new,\n 'original/' + filename,\n 'fixed/' + filename,\n lineterm=newline)\n\n text = ''\n for line in diff:\n text += line\n\n # Work around missing newline (http://bugs.python.org/issue2142).\n if text and not line.endswith(newline):\n text += newline + r'\\ No newline at end of file' + newline\n\n return text"} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ngenerate a list of tokens that are shortened at a given line.", "response": "def shorten_line(tokens, source, indentation, indent_word, max_line_length,\n aggressive=False, experimental=False, previous_line=''):\n \"\"\"Separate line at OPERATOR.\n\n Multiple candidates will be yielded.\n\n \"\"\"\n for candidate in _shorten_line(tokens=tokens,\n source=source,\n indentation=indentation,\n indent_word=indent_word,\n aggressive=aggressive,\n previous_line=previous_line):\n yield candidate\n\n if aggressive:\n for key_token_strings in SHORTEN_OPERATOR_GROUPS:\n shortened = _shorten_line_at_tokens(\n tokens=tokens,\n source=source,\n indentation=indentation,\n indent_word=indent_word,\n key_token_strings=key_token_strings,\n aggressive=aggressive)\n\n if shortened is not None and shortened != source:\n yield shortened\n\n if experimental:\n for shortened in _shorten_line_at_tokens_new(\n tokens=tokens,\n source=source,\n indentation=indentation,\n max_line_length=max_line_length):\n\n yield shortened"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _shorten_line(tokens, source, indentation, indent_word,\n aggressive=False, previous_line=''):\n \"\"\"Separate line at OPERATOR.\n\n The input is expected to be free of newlines except for inside multiline\n strings and at the end.\n\n Multiple candidates will be yielded.\n\n \"\"\"\n for (token_type,\n token_string,\n start_offset,\n end_offset) in token_offsets(tokens):\n\n if (\n token_type == tokenize.COMMENT and\n not is_probably_part_of_multiline(previous_line) and\n not is_probably_part_of_multiline(source) and\n not source[start_offset + 1:].strip().lower().startswith(\n ('noqa', 'pragma:', 'pylint:'))\n ):\n # Move inline comments to previous line.\n first = source[:start_offset]\n second = source[start_offset:]\n yield (indentation + second.strip() + '\\n' +\n indentation + first.strip() + '\\n')\n elif token_type == token.OP and token_string != '=':\n # Don't break on '=' after keyword as this violates PEP 8.\n\n assert token_type != token.INDENT\n\n first = source[:end_offset]\n\n second_indent = indentation\n if (first.rstrip().endswith('(') and\n source[end_offset:].lstrip().startswith(')')):\n pass\n elif first.rstrip().endswith('('):\n second_indent += indent_word\n elif '(' in first:\n second_indent += ' ' * (1 + first.find('('))\n else:\n second_indent += indent_word\n\n second = (second_indent + source[end_offset:].lstrip())\n if (\n not second.strip() or\n second.lstrip().startswith('#')\n ):\n continue\n\n # Do not begin a line with a comma\n if second.lstrip().startswith(','):\n continue\n # Do end a line with a dot\n if first.rstrip().endswith('.'):\n continue\n if token_string in '+-*/':\n fixed = first + ' \\\\' + '\\n' + second\n else:\n fixed = first + '\\n' + second\n\n # Only fix if syntax is okay.\n if check_syntax(normalize_multiline(fixed)\n if aggressive else fixed):\n yield indentation + fixed", "response": "Yields a list of tokens from the source line."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nparse a high - level container such as a list tuple etc.", "response": "def _parse_container(tokens, index, for_or_if=None):\n \"\"\"Parse a high-level container, such as a list, tuple, etc.\"\"\"\n\n # Store the opening bracket.\n items = [Atom(Token(*tokens[index]))]\n index += 1\n\n num_tokens = len(tokens)\n while index < num_tokens:\n tok = Token(*tokens[index])\n\n if tok.token_string in ',)]}':\n # First check if we're at the end of a list comprehension or\n # if-expression. Don't add the ending token as part of the list\n # comprehension or if-expression, because they aren't part of those\n # constructs.\n if for_or_if == 'for':\n return (ListComprehension(items), index - 1)\n\n elif for_or_if == 'if':\n return (IfExpression(items), index - 1)\n\n # We've reached the end of a container.\n items.append(Atom(tok))\n\n # If not, then we are at the end of a container.\n if tok.token_string == ')':\n # The end of a tuple.\n return (Tuple(items), index)\n\n elif tok.token_string == ']':\n # The end of a list.\n return (List(items), index)\n\n elif tok.token_string == '}':\n # The end of a dictionary or set.\n return (DictOrSet(items), index)\n\n elif tok.token_string in '([{':\n # A sub-container is being defined.\n (container, index) = _parse_container(tokens, index)\n items.append(container)\n\n elif tok.token_string == 'for':\n (container, index) = _parse_container(tokens, index, 'for')\n items.append(container)\n\n elif tok.token_string == 'if':\n (container, index) = _parse_container(tokens, index, 'if')\n items.append(container)\n\n else:\n items.append(Atom(tok))\n\n index += 1\n\n return (None, None)"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nparsing the tokens. This converts the tokens into a form where we can manipulate them more easily.", "response": "def _parse_tokens(tokens):\n \"\"\"Parse the tokens.\n\n This converts the tokens into a form where we can manipulate them\n more easily.\n\n \"\"\"\n\n index = 0\n parsed_tokens = []\n\n num_tokens = len(tokens)\n while index < num_tokens:\n tok = Token(*tokens[index])\n\n assert tok.token_type != token.INDENT\n if tok.token_type == tokenize.NEWLINE:\n # There's only one newline and it's at the end.\n break\n\n if tok.token_string in '([{':\n (container, index) = _parse_container(tokens, index)\n if not container:\n return None\n parsed_tokens.append(container)\n else:\n parsed_tokens.append(Atom(tok))\n\n index += 1\n\n return parsed_tokens"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _reflow_lines(parsed_tokens, indentation, max_line_length,\n start_on_prefix_line):\n \"\"\"Reflow the lines so that it looks nice.\"\"\"\n\n if unicode(parsed_tokens[0]) == 'def':\n # A function definition gets indented a bit more.\n continued_indent = indentation + ' ' * 2 * DEFAULT_INDENT_SIZE\n else:\n continued_indent = indentation + ' ' * DEFAULT_INDENT_SIZE\n\n break_after_open_bracket = not start_on_prefix_line\n\n lines = ReformattedLines(max_line_length)\n lines.add_indent(len(indentation.lstrip('\\r\\n')))\n\n if not start_on_prefix_line:\n # If splitting after the opening bracket will cause the first element\n # to be aligned weirdly, don't try it.\n first_token = get_item(parsed_tokens, 0)\n second_token = get_item(parsed_tokens, 1)\n\n if (\n first_token and second_token and\n unicode(second_token)[0] == '(' and\n len(indentation) + len(first_token) + 1 == len(continued_indent)\n ):\n return None\n\n for item in parsed_tokens:\n lines.add_space_if_needed(unicode(item), equal=True)\n\n save_continued_indent = continued_indent\n if start_on_prefix_line and isinstance(item, Container):\n start_on_prefix_line = False\n continued_indent = ' ' * (lines.current_size() + 1)\n\n item.reflow(lines, continued_indent, break_after_open_bracket)\n continued_indent = save_continued_indent\n\n return lines.emit()", "response": "Reflow the lines so that it looks nice."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nshorten the line at the given indentation and max_line_length.", "response": "def _shorten_line_at_tokens_new(tokens, source, indentation,\n max_line_length):\n \"\"\"Shorten the line taking its length into account.\n\n The input is expected to be free of newlines except for inside\n multiline strings and at the end.\n\n \"\"\"\n # Yield the original source so to see if it's a better choice than the\n # shortened candidate lines we generate here.\n yield indentation + source\n\n parsed_tokens = _parse_tokens(tokens)\n\n if parsed_tokens:\n # Perform two reflows. The first one starts on the same line as the\n # prefix. The second starts on the line after the prefix.\n fixed = _reflow_lines(parsed_tokens, indentation, max_line_length,\n start_on_prefix_line=True)\n if fixed and check_syntax(normalize_multiline(fixed.lstrip())):\n yield fixed\n\n fixed = _reflow_lines(parsed_tokens, indentation, max_line_length,\n start_on_prefix_line=False)\n if fixed and check_syntax(normalize_multiline(fixed.lstrip())):\n yield fixed"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nseparate line by breaking at tokens in key_token_strings.", "response": "def _shorten_line_at_tokens(tokens, source, indentation, indent_word,\n key_token_strings, aggressive):\n \"\"\"Separate line by breaking at tokens in key_token_strings.\n\n The input is expected to be free of newlines except for inside\n multiline strings and at the end.\n\n \"\"\"\n offsets = []\n for (index, _t) in enumerate(token_offsets(tokens)):\n (token_type,\n token_string,\n start_offset,\n end_offset) = _t\n\n assert token_type != token.INDENT\n\n if token_string in key_token_strings:\n # Do not break in containers with zero or one items.\n unwanted_next_token = {\n '(': ')',\n '[': ']',\n '{': '}'}.get(token_string)\n if unwanted_next_token:\n if (\n get_item(tokens,\n index + 1,\n default=[None, None])[1] == unwanted_next_token or\n get_item(tokens,\n index + 2,\n default=[None, None])[1] == unwanted_next_token\n ):\n continue\n\n if (\n index > 2 and token_string == '(' and\n tokens[index - 1][1] in ',(%['\n ):\n # Don't split after a tuple start, or before a tuple start if\n # the tuple is in a list.\n continue\n\n if end_offset < len(source) - 1:\n # Don't split right before newline.\n offsets.append(end_offset)\n else:\n # Break at adjacent strings. These were probably meant to be on\n # separate lines in the first place.\n previous_token = get_item(tokens, index - 1)\n if (\n token_type == tokenize.STRING and\n previous_token and previous_token[0] == tokenize.STRING\n ):\n offsets.append(start_offset)\n\n current_indent = None\n fixed = None\n for line in split_at_offsets(source, offsets):\n if fixed:\n fixed += '\\n' + current_indent + line\n\n for symbol in '([{':\n if line.endswith(symbol):\n current_indent += indent_word\n else:\n # First line.\n fixed = line\n assert not current_indent\n current_indent = indent_word\n\n assert fixed is not None\n\n if check_syntax(normalize_multiline(fixed)\n if aggressive > 1 else fixed):\n return indentation + fixed\n\n return None"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef token_offsets(tokens):\n end_offset = 0\n previous_end_row = 0\n previous_end_column = 0\n for t in tokens:\n token_type = t[0]\n token_string = t[1]\n (start_row, start_column) = t[2]\n (end_row, end_column) = t[3]\n\n # Account for the whitespace between tokens.\n end_offset += start_column\n if previous_end_row == start_row:\n end_offset -= previous_end_column\n\n # Record the start offset of the token.\n start_offset = end_offset\n\n # Account for the length of the token itself.\n end_offset += len(token_string)\n\n yield (token_type,\n token_string,\n start_offset,\n end_offset)\n\n previous_end_row = end_row\n previous_end_column = end_column", "response": "Yields tokens and offsets."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _execute_pep8(pep8_options, source):\n class QuietReport(pycodestyle.BaseReport):\n\n \"\"\"Version of checker that does not print.\"\"\"\n\n def __init__(self, options):\n super(QuietReport, self).__init__(options)\n self.__full_error_results = []\n\n def error(self, line_number, offset, text, check):\n \"\"\"Collect errors.\"\"\"\n code = super(QuietReport, self).error(line_number,\n offset,\n text,\n check)\n if code:\n self.__full_error_results.append(\n {'id': code,\n 'line': line_number,\n 'column': offset + 1,\n 'info': text})\n\n def full_error_results(self):\n \"\"\"Return error results in detail.\n\n Results are in the form of a list of dictionaries. Each\n dictionary contains 'id', 'line', 'column', and 'info'.\n\n \"\"\"\n return self.__full_error_results\n\n checker = pycodestyle.Checker('', lines=source, reporter=QuietReport,\n **pep8_options)\n checker.check_all()\n return checker.report.full_error_results()", "response": "Execute pycodestyle via python method calls."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _reindent_stats(tokens):\n find_stmt = 1 # Next token begins a fresh stmt?\n level = 0 # Current indent level.\n stats = []\n\n for t in tokens:\n token_type = t[0]\n sline = t[2][0]\n line = t[4]\n\n if token_type == tokenize.NEWLINE:\n # A program statement, or ENDMARKER, will eventually follow,\n # after some (possibly empty) run of tokens of the form\n # (NL | COMMENT)* (INDENT | DEDENT+)?\n find_stmt = 1\n\n elif token_type == tokenize.INDENT:\n find_stmt = 1\n level += 1\n\n elif token_type == tokenize.DEDENT:\n find_stmt = 1\n level -= 1\n\n elif token_type == tokenize.COMMENT:\n if find_stmt:\n stats.append((sline, -1))\n # But we're still looking for a new stmt, so leave\n # find_stmt alone.\n\n elif token_type == tokenize.NL:\n pass\n\n elif find_stmt:\n # This is the first \"real token\" following a NEWLINE, so it\n # must be the first token of the next program statement, or an\n # ENDMARKER.\n find_stmt = 0\n if line: # Not endmarker.\n stats.append((sline, level))\n\n return stats", "response": "Return list of tuples of lineno indentlevel"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _leading_space_count(line):\n i = 0\n while i < len(line) and line[i] == ' ':\n i += 1\n return i", "response": "Return number of leading spaces in line."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef refactor_with_2to3(source_text, fixer_names, filename=''):\n from lib2to3.refactor import RefactoringTool\n fixers = ['lib2to3.fixes.fix_' + name for name in fixer_names]\n tool = RefactoringTool(fixer_names=fixers, explicit=fixers)\n\n from lib2to3.pgen2 import tokenize as lib2to3_tokenize\n try:\n # The name parameter is necessary particularly for the \"import\" fixer.\n return unicode(tool.refactor_string(source_text, name=filename))\n except lib2to3_tokenize.TokenError:\n return source_text", "response": "Use lib2to3 to refactor the source code."} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nchecks if syntax is okay.", "response": "def check_syntax(code):\n \"\"\"Return True if syntax is okay.\"\"\"\n try:\n return compile(code, '', 'exec', dont_inherit=True)\n except (SyntaxError, TypeError, ValueError):\n return False"} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nfilter out spurious reports from pycodestyle.", "response": "def filter_results(source, results, aggressive):\n \"\"\"Filter out spurious reports from pycodestyle.\n\n If aggressive is True, we allow possibly unsafe fixes (E711, E712).\n\n \"\"\"\n non_docstring_string_line_numbers = multiline_string_lines(\n source, include_docstrings=False)\n all_string_line_numbers = multiline_string_lines(\n source, include_docstrings=True)\n\n commented_out_code_line_numbers = commented_out_code_lines(source)\n\n has_e901 = any(result['id'].lower() == 'e901' for result in results)\n\n for r in results:\n issue_id = r['id'].lower()\n\n if r['line'] in non_docstring_string_line_numbers:\n if issue_id.startswith(('e1', 'e501', 'w191')):\n continue\n\n if r['line'] in all_string_line_numbers:\n if issue_id in ['e501']:\n continue\n\n # We must offset by 1 for lines that contain the trailing contents of\n # multiline strings.\n if not aggressive and (r['line'] + 1) in all_string_line_numbers:\n # Do not modify multiline strings in non-aggressive mode. Remove\n # trailing whitespace could break doctests.\n if issue_id.startswith(('w29', 'w39')):\n continue\n\n if aggressive <= 0:\n if issue_id.startswith(('e711', 'e72', 'w6')):\n continue\n\n if aggressive <= 1:\n if issue_id.startswith(('e712', 'e713', 'e714')):\n continue\n\n if aggressive <= 2:\n if issue_id.startswith(('e704')):\n continue\n\n if r['line'] in commented_out_code_line_numbers:\n if issue_id.startswith(('e26', 'e501')):\n continue\n\n # Do not touch indentation if there is a token error caused by\n # incomplete multi-line statement. Otherwise, we risk screwing up the\n # indentation.\n if has_e901:\n if issue_id.startswith(('e1', 'e7')):\n continue\n\n yield r"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef multiline_string_lines(source, include_docstrings=False):\n line_numbers = set()\n previous_token_type = ''\n try:\n for t in generate_tokens(source):\n token_type = t[0]\n start_row = t[2][0]\n end_row = t[3][0]\n\n if token_type == tokenize.STRING and start_row != end_row:\n if (\n include_docstrings or\n previous_token_type != tokenize.INDENT\n ):\n # We increment by one since we want the contents of the\n # string.\n line_numbers |= set(range(1 + start_row, 1 + end_row))\n\n previous_token_type = token_type\n except (SyntaxError, tokenize.TokenError):\n pass\n\n return line_numbers", "response": "Return line numbers that are within multiline strings."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning line numbers of comments that are likely code.", "response": "def commented_out_code_lines(source):\n \"\"\"Return line numbers of comments that are likely code.\n\n Commented-out code is bad practice, but modifying it just adds even\n more clutter.\n\n \"\"\"\n line_numbers = []\n try:\n for t in generate_tokens(source):\n token_type = t[0]\n token_string = t[1]\n start_row = t[2][0]\n line = t[4]\n\n # Ignore inline comments.\n if not line.lstrip().startswith('#'):\n continue\n\n if token_type == tokenize.COMMENT:\n stripped_line = token_string.lstrip('#').strip()\n if (\n ' ' in stripped_line and\n '#' not in stripped_line and\n check_syntax(stripped_line)\n ):\n line_numbers.append(start_row)\n except (SyntaxError, tokenize.TokenError):\n pass\n\n return line_numbers"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn trimmed or split long comment line.", "response": "def shorten_comment(line, max_line_length, last_comment=False):\n \"\"\"Return trimmed or split long comment line.\n\n If there are no comments immediately following it, do a text wrap.\n Doing this wrapping on all comments in general would lead to jagged\n comment text.\n\n \"\"\"\n assert len(line) > max_line_length\n line = line.rstrip()\n\n # PEP 8 recommends 72 characters for comment text.\n indentation = _get_indentation(line) + '# '\n max_line_length = min(max_line_length,\n len(indentation) + 72)\n\n MIN_CHARACTER_REPEAT = 5\n if (\n len(line) - len(line.rstrip(line[-1])) >= MIN_CHARACTER_REPEAT and\n not line[-1].isalnum()\n ):\n # Trim comments that end with things like ---------\n return line[:max_line_length] + '\\n'\n elif last_comment and re.match(r'\\s*#+\\s*\\w+', line):\n split_lines = textwrap.wrap(line.lstrip(' \\t#'),\n initial_indent=indentation,\n subsequent_indent=indentation,\n width=max_line_length,\n break_long_words=False,\n break_on_hyphens=False)\n return '\\n'.join(split_lines) + '\\n'\n\n return line + '\\n'"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns fixed source code.", "response": "def fix_code(source, options=None, encoding=None, apply_config=False):\n \"\"\"Return fixed source code.\n\n \"encoding\" will be used to decode \"source\" if it is a byte string.\n\n \"\"\"\n options = _get_options(options, apply_config)\n\n if not isinstance(source, unicode):\n source = source.decode(encoding or get_encoding())\n\n sio = io.StringIO(source)\n return fix_lines(sio.readlines(), options=options)"} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns fixed source code.", "response": "def fix_lines(source_lines, options, filename=''):\n \"\"\"Return fixed source code.\"\"\"\n # Transform everything to line feed. Then change them back to original\n # before returning fixed source code.\n original_newline = find_newline(source_lines)\n tmp_source = ''.join(normalize_line_endings(source_lines, '\\n'))\n\n # Keep a history to break out of cycles.\n previous_hashes = set()\n\n if options.line_range:\n # Disable \"apply_local_fixes()\" for now due to issue #175.\n fixed_source = tmp_source\n else:\n pep8_options = {\n 'ignore': options.ignore,\n 'select': options.select,\n 'max_line_length': options.max_line_length,\n 'hang_closing': options.hang_closing,\n }\n sio = io.StringIO(tmp_source)\n contents = sio.readlines()\n results = _execute_pep8(pep8_options, contents)\n codes = {result['id'] for result in results\n if result['id'] in SELECTED_GLOBAL_FIXED_METHOD_CODES}\n # Apply global fixes only once (for efficiency).\n fixed_source = apply_global_fixes(tmp_source,\n options,\n filename=filename,\n codes=codes)\n\n passes = 0\n long_line_ignore_cache = set()\n while hash(fixed_source) not in previous_hashes:\n if options.pep8_passes >= 0 and passes > options.pep8_passes:\n break\n passes += 1\n\n previous_hashes.add(hash(fixed_source))\n\n tmp_source = copy.copy(fixed_source)\n\n fix = FixPEP8(\n filename,\n options,\n contents=tmp_source,\n long_line_ignore_cache=long_line_ignore_cache)\n\n fixed_source = fix.fix()\n\n sio = io.StringIO(fixed_source)\n return ''.join(normalize_line_endings(sio.readlines(), original_newline))"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef global_fixes():\n for function in list(globals().values()):\n if inspect.isfunction(function):\n arguments = _get_parameters(function)\n if arguments[:1] != ['source']:\n continue\n\n code = extract_code_from_function(function)\n if code:\n yield (code, function)", "response": "Yields multiple ( code function ) tuples."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef extract_code_from_function(function):\n if not function.__name__.startswith('fix_'):\n return None\n\n code = re.sub('^fix_', '', function.__name__)\n if not code:\n return None\n\n try:\n int(code[1:])\n except ValueError:\n return None\n\n return code", "response": "Return code handled by function."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef create_parser():\n parser = argparse.ArgumentParser(description=docstring_summary(__doc__),\n prog='autopep8')\n parser.add_argument('--version', action='version',\n version='%(prog)s {} ({})'.format(\n __version__, _get_package_version()))\n parser.add_argument('-v', '--verbose', action='count',\n default=0,\n help='print verbose messages; '\n 'multiple -v result in more verbose messages')\n parser.add_argument('-d', '--diff', action='store_true',\n help='print the diff for the fixed source')\n parser.add_argument('-i', '--in-place', action='store_true',\n help='make changes to files in place')\n parser.add_argument('--global-config', metavar='filename',\n default=DEFAULT_CONFIG,\n help='path to a global pep8 config file; if this file '\n 'does not exist then this is ignored '\n '(default: {})'.format(DEFAULT_CONFIG))\n parser.add_argument('--ignore-local-config', action='store_true',\n help=\"don't look for and apply local config files; \"\n 'if not passed, defaults are updated with any '\n \"config files in the project's root directory\")\n parser.add_argument('-r', '--recursive', action='store_true',\n help='run recursively over directories; '\n 'must be used with --in-place or --diff')\n parser.add_argument('-j', '--jobs', type=int, metavar='n', default=1,\n help='number of parallel jobs; '\n 'match CPU count if value is less than 1')\n parser.add_argument('-p', '--pep8-passes', metavar='n',\n default=-1, type=int,\n help='maximum number of additional pep8 passes '\n '(default: infinite)')\n parser.add_argument('-a', '--aggressive', action='count', default=0,\n help='enable non-whitespace changes; '\n 'multiple -a result in more aggressive changes')\n parser.add_argument('--experimental', action='store_true',\n help='enable experimental fixes')\n parser.add_argument('--exclude', metavar='globs',\n help='exclude file/directory names that match these '\n 'comma-separated globs')\n parser.add_argument('--list-fixes', action='store_true',\n help='list codes for fixes; '\n 'used by --ignore and --select')\n parser.add_argument('--ignore', metavar='errors', default='',\n help='do not fix these errors/warnings '\n '(default: {})'.format(DEFAULT_IGNORE))\n parser.add_argument('--select', metavar='errors', default='',\n help='fix only these errors/warnings (e.g. E4,W)')\n parser.add_argument('--max-line-length', metavar='n', default=79, type=int,\n help='set maximum allowed line length '\n '(default: %(default)s)')\n parser.add_argument('--line-range', '--range', metavar='line',\n default=None, type=int, nargs=2,\n help='only fix errors found within this inclusive '\n 'range of line numbers (e.g. 1 99); '\n 'line numbers are indexed at 1')\n parser.add_argument('--indent-size', default=DEFAULT_INDENT_SIZE,\n type=int, help=argparse.SUPPRESS)\n parser.add_argument('--hang-closing', action='store_true',\n help='hang-closing option passed to pycodestyle')\n parser.add_argument('--exit-code', action='store_true',\n help='change to behavior of exit code.'\n ' default behavior of return value, 0 is no '\n 'differences, 1 is error exit. return 2 when'\n ' add this option. 2 is exists differences.')\n parser.add_argument('files', nargs='*',\n help=\"files to format or '-' for standard in\")\n\n return parser", "response": "Create command - line parser."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef parse_args(arguments, apply_config=False):\n parser = create_parser()\n args = parser.parse_args(arguments)\n\n if not args.files and not args.list_fixes:\n parser.error('incorrect number of arguments')\n\n args.files = [decode_filename(name) for name in args.files]\n\n if apply_config:\n parser = read_config(args, parser)\n args = parser.parse_args(arguments)\n args.files = [decode_filename(name) for name in args.files]\n\n if '-' in args.files:\n if len(args.files) > 1:\n parser.error('cannot mix stdin and regular files')\n\n if args.diff:\n parser.error('--diff cannot be used with standard input')\n\n if args.in_place:\n parser.error('--in-place cannot be used with standard input')\n\n if args.recursive:\n parser.error('--recursive cannot be used with standard input')\n\n if len(args.files) > 1 and not (args.in_place or args.diff):\n parser.error('autopep8 only takes one filename as argument '\n 'unless the \"--in-place\" or \"--diff\" args are '\n 'used')\n\n if args.recursive and not (args.in_place or args.diff):\n parser.error('--recursive must be used with --in-place or --diff')\n\n if args.in_place and args.diff:\n parser.error('--in-place and --diff are mutually exclusive')\n\n if args.max_line_length <= 0:\n parser.error('--max-line-length must be greater than 0')\n\n if args.select:\n args.select = _split_comma_separated(args.select)\n\n if args.ignore:\n args.ignore = _split_comma_separated(args.ignore)\n if not all(\n any(\n conflicting_code.startswith(ignore_code)\n for ignore_code in args.ignore\n )\n for conflicting_code in CONFLICTING_CODES\n ):\n args.ignore.update(CONFLICTING_CODES)\n elif not args.select:\n if args.aggressive:\n # Enable everything by default if aggressive.\n args.select = {'E', 'W1', 'W2', 'W3', 'W6'}\n else:\n args.ignore = _split_comma_separated(DEFAULT_IGNORE)\n\n if args.exclude:\n args.exclude = _split_comma_separated(args.exclude)\n else:\n args.exclude = {}\n\n if args.jobs < 1:\n # Do not import multiprocessing globally in case it is not supported\n # on the platform.\n import multiprocessing\n args.jobs = multiprocessing.cpu_count()\n\n if args.jobs > 1 and not args.in_place:\n parser.error('parallel jobs requires --in-place')\n\n if args.line_range:\n if args.line_range[0] <= 0:\n parser.error('--range must be positive numbers')\n if args.line_range[0] > args.line_range[1]:\n parser.error('First value of --range should be less than or equal '\n 'to the second')\n\n return args", "response": "Parse command - line options."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef read_config(args, parser):\n try:\n from configparser import ConfigParser as SafeConfigParser\n from configparser import Error\n except ImportError:\n from ConfigParser import SafeConfigParser\n from ConfigParser import Error\n\n config = SafeConfigParser()\n\n try:\n config.read(args.global_config)\n\n if not args.ignore_local_config:\n parent = tail = args.files and os.path.abspath(\n os.path.commonprefix(args.files))\n while tail:\n if config.read([os.path.join(parent, fn)\n for fn in PROJECT_CONFIG]):\n break\n (parent, tail) = os.path.split(parent)\n\n defaults = {}\n option_list = {o.dest: o.type or type(o.default)\n for o in parser._actions}\n\n for section in ['pep8', 'pycodestyle', 'flake8']:\n if not config.has_section(section):\n continue\n for (k, _) in config.items(section):\n norm_opt = k.lstrip('-').replace('-', '_')\n if not option_list.get(norm_opt):\n continue\n opt_type = option_list[norm_opt]\n if opt_type is int:\n value = config.getint(section, k)\n elif opt_type is bool:\n value = config.getboolean(section, k)\n else:\n value = config.get(section, k)\n if args.verbose:\n print(\"enable config: section={}, key={}, value={}\".format(\n section, k, value))\n defaults[norm_opt] = value\n\n parser.set_defaults(**defaults)\n except Error:\n # Ignore for now.\n pass\n\n return parser", "response": "Read both user configuration and local configuration."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef supported_fixes():\n yield ('E101', docstring_summary(reindent.__doc__))\n\n instance = FixPEP8(filename=None, options=None, contents='')\n for attribute in dir(instance):\n code = re.match('fix_([ew][0-9][0-9][0-9])', attribute)\n if code:\n yield (\n code.group(1).upper(),\n re.sub(r'\\s+', ' ',\n docstring_summary(getattr(instance, attribute).__doc__))\n )\n\n for (code, function) in sorted(global_fixes()):\n yield (code.upper() + (4 - len(code)) * ' ',\n re.sub(r'\\s+', ' ', docstring_summary(function.__doc__)))\n\n for code in sorted(CODE_TO_2TO3):\n yield (code.upper() + (4 - len(code)) * ' ',\n re.sub(r'\\s+', ' ', docstring_summary(fix_2to3.__doc__)))", "response": "Yields all the error codes that autopep8 fixes are supported."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef line_shortening_rank(candidate, indent_word, max_line_length,\n experimental=False):\n \"\"\"Return rank of candidate.\n\n This is for sorting candidates.\n\n \"\"\"\n if not candidate.strip():\n return 0\n\n rank = 0\n lines = candidate.rstrip().split('\\n')\n\n offset = 0\n if (\n not lines[0].lstrip().startswith('#') and\n lines[0].rstrip()[-1] not in '([{'\n ):\n for (opening, closing) in ('()', '[]', '{}'):\n # Don't penalize empty containers that aren't split up. Things like\n # this \"foo(\\n )\" aren't particularly good.\n opening_loc = lines[0].find(opening)\n closing_loc = lines[0].find(closing)\n if opening_loc >= 0:\n if closing_loc < 0 or closing_loc != opening_loc + 1:\n offset = max(offset, 1 + opening_loc)\n\n current_longest = max(offset + len(x.strip()) for x in lines)\n\n rank += 4 * max(0, current_longest - max_line_length)\n\n rank += len(lines)\n\n # Too much variation in line length is ugly.\n rank += 2 * standard_deviation(len(line) for line in lines)\n\n bad_staring_symbol = {\n '(': ')',\n '[': ']',\n '{': '}'}.get(lines[0][-1])\n\n if len(lines) > 1:\n if (\n bad_staring_symbol and\n lines[1].lstrip().startswith(bad_staring_symbol)\n ):\n rank += 20\n\n for lineno, current_line in enumerate(lines):\n current_line = current_line.strip()\n\n if current_line.startswith('#'):\n continue\n\n for bad_start in ['.', '%', '+', '-', '/']:\n if current_line.startswith(bad_start):\n rank += 100\n\n # Do not tolerate operators on their own line.\n if current_line == bad_start:\n rank += 1000\n\n if (\n current_line.endswith(('.', '%', '+', '-', '/')) and\n \"': \" in current_line\n ):\n rank += 1000\n\n if current_line.endswith(('(', '[', '{', '.')):\n # Avoid lonely opening. They result in longer lines.\n if len(current_line) <= len(indent_word):\n rank += 100\n\n # Avoid the ugliness of \", (\\n\".\n if (\n current_line.endswith('(') and\n current_line[:-1].rstrip().endswith(',')\n ):\n rank += 100\n\n # Avoid the ugliness of \"something[\\n\" and something[index][\\n.\n if (\n current_line.endswith('[') and\n len(current_line) > 1 and\n (current_line[-2].isalnum() or current_line[-2] in ']')\n ):\n rank += 300\n\n # Also avoid the ugliness of \"foo.\\nbar\"\n if current_line.endswith('.'):\n rank += 100\n\n if has_arithmetic_operator(current_line):\n rank += 100\n\n # Avoid breaking at unary operators.\n if re.match(r'.*[(\\[{]\\s*[\\-\\+~]$', current_line.rstrip('\\\\ ')):\n rank += 1000\n\n if re.match(r'.*lambda\\s*\\*$', current_line.rstrip('\\\\ ')):\n rank += 1000\n\n if current_line.endswith(('%', '(', '[', '{')):\n rank -= 20\n\n # Try to break list comprehensions at the \"for\".\n if current_line.startswith('for '):\n rank -= 50\n\n if current_line.endswith('\\\\'):\n # If a line ends in \\-newline, it may be part of a\n # multiline string. In that case, we would like to know\n # how long that line is without the \\-newline. If it's\n # longer than the maximum, or has comments, then we assume\n # that the \\-newline is an okay candidate and only\n # penalize it a bit.\n total_len = len(current_line)\n lineno += 1\n while lineno < len(lines):\n total_len += len(lines[lineno])\n\n if lines[lineno].lstrip().startswith('#'):\n total_len = max_line_length\n break\n\n if not lines[lineno].endswith('\\\\'):\n break\n\n lineno += 1\n\n if total_len < max_line_length:\n rank += 10\n else:\n rank += 100 if experimental else 1\n\n # Prefer breaking at commas rather than colon.\n if ',' in current_line and current_line.endswith(':'):\n rank += 10\n\n # Avoid splitting dictionaries between key and value.\n if current_line.endswith(':'):\n rank += 100\n\n rank += 10 * count_unbalanced_brackets(current_line)\n\n return max(0, rank)", "response": "Return the rank of a candidate line."} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns standard deviation of a list of numbers.", "response": "def standard_deviation(numbers):\n \"\"\"Return standard deviation.\"\"\"\n numbers = list(numbers)\n if not numbers:\n return 0\n mean = sum(numbers) / len(numbers)\n return (sum((n - mean) ** 2 for n in numbers) /\n len(numbers)) ** .5"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning number of unmatched open and close brackets.", "response": "def count_unbalanced_brackets(line):\n \"\"\"Return number of unmatched open/close brackets.\"\"\"\n count = 0\n for opening, closing in ['()', '[]', '{}']:\n count += abs(line.count(opening) - line.count(closing))\n\n return count"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef split_at_offsets(line, offsets):\n result = []\n\n previous_offset = 0\n current_offset = 0\n for current_offset in sorted(offsets):\n if current_offset < len(line) and previous_offset != current_offset:\n result.append(line[previous_offset:current_offset].strip())\n previous_offset = current_offset\n\n result.append(line[current_offset:])\n\n return result", "response": "Split line at offsets.\n Return list of strings."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef match_file(filename, exclude):\n base_name = os.path.basename(filename)\n\n if base_name.startswith('.'):\n return False\n\n for pattern in exclude:\n if fnmatch.fnmatch(base_name, pattern):\n return False\n if fnmatch.fnmatch(filename, pattern):\n return False\n\n if not os.path.isdir(filename) and not is_python_file(filename):\n return False\n\n return True", "response": "Return True if file is okay for modifying or recursing."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nfix list of files.", "response": "def fix_multiple_files(filenames, options, output=None):\n \"\"\"Fix list of files.\n\n Optionally fix files recursively.\n\n \"\"\"\n results = []\n filenames = find_files(filenames, options.recursive, options.exclude)\n if options.jobs > 1:\n import multiprocessing\n pool = multiprocessing.Pool(options.jobs)\n ret = pool.map(_fix_file, [(name, options) for name in filenames])\n results.extend([x for x in ret if x is not None])\n else:\n for name in filenames:\n ret = _fix_file((name, options, output))\n if ret is None:\n continue\n if options.diff:\n if ret != '':\n results.append(ret)\n elif options.in_place:\n results.append(ret)\n else:\n original_source = readlines_from_file(name)\n if \"\".join(original_source).splitlines() != ret.splitlines():\n results.append(ret)\n return results"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef wrap_output(output, encoding):\n return codecs.getwriter(encoding)(output.buffer\n if hasattr(output, 'buffer')\n else output)", "response": "Wrap output with specified encoding."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns a version of the source code with PEP 8 violations fixed.", "response": "def fix(self):\n \"\"\"Return a version of the source code with PEP 8 violations fixed.\"\"\"\n pep8_options = {\n 'ignore': self.options.ignore,\n 'select': self.options.select,\n 'max_line_length': self.options.max_line_length,\n 'hang_closing': self.options.hang_closing,\n }\n results = _execute_pep8(pep8_options, self.source)\n\n if self.options.verbose:\n progress = {}\n for r in results:\n if r['id'] not in progress:\n progress[r['id']] = set()\n progress[r['id']].add(r['line'])\n print('---> {n} issue(s) to fix {progress}'.format(\n n=len(results), progress=progress), file=sys.stderr)\n\n if self.options.line_range:\n start, end = self.options.line_range\n results = [r for r in results\n if start <= r['line'] <= end]\n\n self._fix_source(filter_results(source=''.join(self.source),\n results=results,\n aggressive=self.options.aggressive))\n\n if self.options.line_range:\n # If number of lines has changed then change line_range.\n count = sum(sline.count('\\n')\n for sline in self.source[start - 1:end])\n self.options.line_range[1] = start + count - 1\n\n return ''.join(self.source)"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _fix_reindent(self, result):\n num_indent_spaces = int(result['info'].split()[1])\n line_index = result['line'] - 1\n target = self.source[line_index]\n\n self.source[line_index] = ' ' * num_indent_spaces + target.lstrip()", "response": "Fix a badly indented line.\n\n This is done by adding or removing from its initial indent only."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef fix_e112(self, result):\n line_index = result['line'] - 1\n target = self.source[line_index]\n\n if not target.lstrip().startswith('#'):\n # Don't screw with invalid syntax.\n return []\n\n self.source[line_index] = self.indent_word + target", "response": "Fix under - indented comments."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nfixing indentation undistinguish from the next logical line.", "response": "def fix_e125(self, result):\n \"\"\"Fix indentation undistinguish from the next logical line.\"\"\"\n num_indent_spaces = int(result['info'].split()[1])\n line_index = result['line'] - 1\n target = self.source[line_index]\n\n spaces_to_add = num_indent_spaces - len(_get_indentation(target))\n indent = len(_get_indentation(target))\n modified_lines = []\n\n while len(_get_indentation(self.source[line_index])) >= indent:\n self.source[line_index] = (' ' * spaces_to_add +\n self.source[line_index])\n modified_lines.append(1 + line_index) # Line indexed at 1.\n line_index -= 1\n\n return modified_lines"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nfix indentation undistinguish from the next logical line.", "response": "def fix_e131(self, result):\n \"\"\"Fix indentation undistinguish from the next logical line.\"\"\"\n num_indent_spaces = int(result['info'].split()[1])\n line_index = result['line'] - 1\n target = self.source[line_index]\n\n spaces_to_add = num_indent_spaces - len(_get_indentation(target))\n\n if spaces_to_add >= 0:\n self.source[line_index] = (' ' * spaces_to_add +\n self.source[line_index])\n else:\n offset = abs(spaces_to_add)\n self.source[line_index] = self.source[line_index][offset:]"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef fix_e224(self, result):\n target = self.source[result['line'] - 1]\n offset = result['column'] - 1\n fixed = target[:offset] + target[offset:].replace('\\t', ' ')\n self.source[result['line'] - 1] = fixed", "response": "Remove extraneous whitespace around operator."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nfixing missing whitespace around operator.", "response": "def fix_e225(self, result):\n \"\"\"Fix missing whitespace around operator.\"\"\"\n target = self.source[result['line'] - 1]\n offset = result['column'] - 1\n fixed = target[:offset] + ' ' + target[offset:]\n\n # Only proceed if non-whitespace characters match.\n # And make sure we don't break the indentation.\n if (\n fixed.replace(' ', '') == target.replace(' ', '') and\n _get_indentation(fixed) == _get_indentation(target)\n ):\n self.source[result['line'] - 1] = fixed\n error_code = result.get('id', 0)\n try:\n ts = generate_tokens(fixed)\n except (SyntaxError, tokenize.TokenError):\n return\n if not check_syntax(fixed.lstrip()):\n return\n errors = list(\n pycodestyle.missing_whitespace_around_operator(fixed, ts))\n for e in reversed(errors):\n if error_code != e[1].split()[0]:\n continue\n offset = e[0][1]\n fixed = fixed[:offset] + ' ' + fixed[offset:]\n self.source[result['line'] - 1] = fixed\n else:\n return []"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef fix_e251(self, result):\n line_index = result['line'] - 1\n target = self.source[line_index]\n\n # This is necessary since pycodestyle sometimes reports columns that\n # goes past the end of the physical line. This happens in cases like,\n # foo(bar\\n=None)\n c = min(result['column'] - 1,\n len(target) - 1)\n\n if target[c].strip():\n fixed = target\n else:\n fixed = target[:c].rstrip() + target[c:].lstrip()\n\n # There could be an escaped newline\n #\n # def foo(a=\\\n # 1)\n if fixed.endswith(('=\\\\\\n', '=\\\\\\r\\n', '=\\\\\\r')):\n self.source[line_index] = fixed.rstrip('\\n\\r \\t\\\\')\n self.source[line_index + 1] = self.source[line_index + 1].lstrip()\n return [line_index + 1, line_index + 2] # Line indexed at 1\n\n self.source[result['line'] - 1] = fixed", "response": "Fix the E251 error."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef fix_e262(self, result):\n target = self.source[result['line'] - 1]\n offset = result['column']\n\n code = target[:offset].rstrip(' \\t#')\n comment = target[offset:].lstrip(' \\t#')\n\n fixed = code + (' # ' + comment if comment.strip() else '\\n')\n\n self.source[result['line'] - 1] = fixed", "response": "Fix spacing after comment hash."} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nfixes extraneous whitespace around keywords.", "response": "def fix_e271(self, result):\n \"\"\"Fix extraneous whitespace around keywords.\"\"\"\n line_index = result['line'] - 1\n target = self.source[line_index]\n offset = result['column'] - 1\n\n fixed = fix_whitespace(target,\n offset=offset,\n replacement=' ')\n\n if fixed == target:\n return []\n else:\n self.source[line_index] = fixed"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef fix_e301(self, result):\n cr = '\\n'\n self.source[result['line'] - 1] = cr + self.source[result['line'] - 1]", "response": "Add missing blank line."} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nadd missing blank lines.", "response": "def fix_e302(self, result):\n \"\"\"Add missing 2 blank lines.\"\"\"\n add_linenum = 2 - int(result['info'].split()[-1])\n cr = '\\n' * add_linenum\n self.source[result['line'] - 1] = cr + self.source[result['line'] - 1]"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef fix_e303(self, result):\n delete_linenum = int(result['info'].split('(')[1].split(')')[0]) - 2\n delete_linenum = max(1, delete_linenum)\n\n # We need to count because pycodestyle reports an offset line number if\n # there are comments.\n cnt = 0\n line = result['line'] - 2\n modified_lines = []\n while cnt < delete_linenum and line >= 0:\n if not self.source[line].strip():\n self.source[line] = ''\n modified_lines.append(1 + line) # Line indexed at 1\n cnt += 1\n line -= 1\n\n return modified_lines", "response": "Remove extra blank lines."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef fix_e304(self, result):\n line = result['line'] - 2\n if not self.source[line].strip():\n self.source[line] = ''", "response": "Remove blank line following function decorator."} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nadds missing blank lines after end of function or class.", "response": "def fix_e305(self, result):\n \"\"\"Add missing 2 blank lines after end of function or class.\"\"\"\n add_delete_linenum = 2 - int(result['info'].split()[-1])\n cnt = 0\n offset = result['line'] - 2\n modified_lines = []\n if add_delete_linenum < 0:\n # delete cr\n add_delete_linenum = abs(add_delete_linenum)\n while cnt < add_delete_linenum and offset >= 0:\n if not self.source[offset].strip():\n self.source[offset] = ''\n modified_lines.append(1 + offset) # Line indexed at 1\n cnt += 1\n offset -= 1\n else:\n # add cr\n cr = '\\n'\n # check comment line\n while True:\n if offset < 0:\n break\n line = self.source[offset].lstrip()\n if not line:\n break\n if line[0] != '#':\n break\n offset -= 1\n offset += 1\n self.source[offset] = cr + self.source[offset]\n modified_lines.append(1 + offset) # Line indexed at 1.\n return modified_lines"} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nfix the import statement in the E401 error message.", "response": "def fix_e401(self, result):\n \"\"\"Put imports on separate lines.\"\"\"\n line_index = result['line'] - 1\n target = self.source[line_index]\n offset = result['column'] - 1\n\n if not target.lstrip().startswith('import'):\n return []\n\n indentation = re.split(pattern=r'\\bimport\\b',\n string=target, maxsplit=1)[0]\n fixed = (target[:offset].rstrip('\\t ,') + '\\n' +\n indentation + 'import ' + target[offset:].lstrip('\\t ,'))\n self.source[line_index] = fixed"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nremoving extraneous escape of newline.", "response": "def fix_e502(self, result):\n \"\"\"Remove extraneous escape of newline.\"\"\"\n (line_index, _, target) = get_index_offset_contents(result,\n self.source)\n self.source[line_index] = target.rstrip('\\n\\r \\t\\\\') + '\\n'"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nputs colon - separated compound statement on separate lines.", "response": "def fix_e701(self, result):\n \"\"\"Put colon-separated compound statement on separate lines.\"\"\"\n line_index = result['line'] - 1\n target = self.source[line_index]\n c = result['column']\n\n fixed_source = (target[:c] + '\\n' +\n _get_indentation(target) + self.indent_word +\n target[c:].lstrip('\\n\\r \\t\\\\'))\n self.source[result['line'] - 1] = fixed_source\n return [result['line'], result['line'] + 1]"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nfix the E702 compound statement on separate lines.", "response": "def fix_e702(self, result, logical):\n \"\"\"Put semicolon-separated compound statement on separate lines.\"\"\"\n if not logical:\n return [] # pragma: no cover\n logical_lines = logical[2]\n\n # Avoid applying this when indented.\n # https://docs.python.org/reference/compound_stmts.html\n for line in logical_lines:\n if (result['id'] == 'E702' and ':' in line\n and STARTSWITH_DEF_REGEX.match(line)):\n return []\n\n line_index = result['line'] - 1\n target = self.source[line_index]\n\n if target.rstrip().endswith('\\\\'):\n # Normalize '1; \\\\\\n2' into '1; 2'.\n self.source[line_index] = target.rstrip('\\n \\r\\t\\\\')\n self.source[line_index + 1] = self.source[line_index + 1].lstrip()\n return [line_index + 1, line_index + 2]\n\n if target.rstrip().endswith(';'):\n self.source[line_index] = target.rstrip('\\n \\r\\t;') + '\\n'\n return [line_index + 1]\n\n offset = result['column'] - 1\n first = target[:offset].rstrip(';').rstrip()\n second = (_get_indentation(logical_lines[0]) +\n target[offset:].lstrip(';').lstrip())\n\n # Find inline comment.\n inline_comment = None\n if target[offset:].lstrip(';').lstrip()[:2] == '# ':\n inline_comment = target[offset:].lstrip(';')\n\n if inline_comment:\n self.source[line_index] = first + inline_comment\n else:\n self.source[line_index] = first + '\\n' + second\n return [line_index + 1]"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef fix_e704(self, result):\n (line_index, _, target) = get_index_offset_contents(result,\n self.source)\n match = STARTSWITH_DEF_REGEX.match(target)\n if match:\n self.source[line_index] = '{}\\n{}{}'.format(\n match.group(0),\n _get_indentation(target) + self.indent_word,\n target[match.end(0):].lstrip())", "response": "Fix multiple statements on one line def"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef fix_e711(self, result):\n (line_index, offset, target) = get_index_offset_contents(result,\n self.source)\n\n right_offset = offset + 2\n if right_offset >= len(target):\n return []\n\n left = target[:offset].rstrip()\n center = target[offset:right_offset]\n right = target[right_offset:].lstrip()\n\n if not right.startswith('None'):\n return []\n\n if center.strip() == '==':\n new_center = 'is'\n elif center.strip() == '!=':\n new_center = 'is not'\n else:\n return []\n\n self.source[line_index] = ' '.join([left, new_center, right])", "response": "Fix comparison with None."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nfix comparison with boolean.", "response": "def fix_e712(self, result):\n \"\"\"Fix (trivial case of) comparison with boolean.\"\"\"\n (line_index, offset, target) = get_index_offset_contents(result,\n self.source)\n\n # Handle very easy \"not\" special cases.\n if re.match(r'^\\s*if [\\w.\"\\'\\[\\]]+ == False:$', target):\n self.source[line_index] = re.sub(r'if ([\\w.\"\\'\\[\\]]+) == False:',\n r'if not \\1:', target, count=1)\n elif re.match(r'^\\s*if [\\w.\"\\'\\[\\]]+ != True:$', target):\n self.source[line_index] = re.sub(r'if ([\\w.\"\\'\\[\\]]+) != True:',\n r'if not \\1:', target, count=1)\n else:\n right_offset = offset + 2\n if right_offset >= len(target):\n return []\n\n left = target[:offset].rstrip()\n center = target[offset:right_offset]\n right = target[right_offset:].lstrip()\n\n # Handle simple cases only.\n new_right = None\n if center.strip() == '==':\n if re.match(r'\\bTrue\\b', right):\n new_right = re.sub(r'\\bTrue\\b *', '', right, count=1)\n elif center.strip() == '!=':\n if re.match(r'\\bFalse\\b', right):\n new_right = re.sub(r'\\bFalse\\b *', '', right, count=1)\n\n if new_right is None:\n return []\n\n if new_right[0].isalnum():\n new_right = ' ' + new_right\n\n self.source[line_index] = left + new_right"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef fix_e713(self, result):\n (line_index, offset, target) = get_index_offset_contents(result,\n self.source)\n\n # to convert once 'not in' -> 'in'\n before_target = target[:offset]\n target = target[offset:]\n match_notin = COMPARE_NEGATIVE_REGEX_THROUGH.search(target)\n notin_pos_start, notin_pos_end = 0, 0\n if match_notin:\n notin_pos_start = match_notin.start(1)\n notin_pos_end = match_notin.end()\n target = '{}{} {}'.format(\n target[:notin_pos_start], 'in', target[notin_pos_end:])\n\n # fix 'not in'\n match = COMPARE_NEGATIVE_REGEX.search(target)\n if match:\n if match.group(3) == 'in':\n pos_start = match.start(1)\n new_target = '{5}{0}{1} {2} {3} {4}'.format(\n target[:pos_start], match.group(2), match.group(1),\n match.group(3), target[match.end():], before_target)\n if match_notin:\n # revert 'in' -> 'not in'\n pos_start = notin_pos_start + offset\n pos_end = notin_pos_end + offset - 4 # len('not ')\n new_target = '{}{} {}'.format(\n new_target[:pos_start], 'not in', new_target[pos_end:])\n self.source[line_index] = new_target", "response": "Fix the case of non - membership check."} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nfixes object identity should be is not case.", "response": "def fix_e714(self, result):\n \"\"\"Fix object identity should be 'is not' case.\"\"\"\n (line_index, offset, target) = get_index_offset_contents(result,\n self.source)\n\n # to convert once 'is not' -> 'is'\n before_target = target[:offset]\n target = target[offset:]\n match_isnot = COMPARE_NEGATIVE_REGEX_THROUGH.search(target)\n isnot_pos_start, isnot_pos_end = 0, 0\n if match_isnot:\n isnot_pos_start = match_isnot.start(1)\n isnot_pos_end = match_isnot.end()\n target = '{}{} {}'.format(\n target[:isnot_pos_start], 'in', target[isnot_pos_end:])\n\n match = COMPARE_NEGATIVE_REGEX.search(target)\n if match:\n if match.group(3).startswith('is'):\n pos_start = match.start(1)\n new_target = '{5}{0}{1} {2} {3} {4}'.format(\n target[:pos_start], match.group(2), match.group(3),\n match.group(1), target[match.end():], before_target)\n if match_isnot:\n # revert 'is' -> 'is not'\n pos_start = isnot_pos_start + offset\n pos_end = isnot_pos_end + offset - 4 # len('not ')\n new_target = '{}{} {}'.format(\n new_target[:pos_start], 'is not', new_target[pos_end:])\n self.source[line_index] = new_target"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nfixing do not assign a lambda expression check.", "response": "def fix_e731(self, result):\n \"\"\"Fix do not assign a lambda expression check.\"\"\"\n (line_index, _, target) = get_index_offset_contents(result,\n self.source)\n match = LAMBDA_REGEX.search(target)\n if match:\n end = match.end()\n self.source[line_index] = '{}def {}({}): return {}'.format(\n target[:match.start(0)], match.group(1), match.group(2),\n target[end:].lstrip())"} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nremoves trailing blank lines.", "response": "def fix_w391(self, _):\n \"\"\"Remove trailing blank lines.\"\"\"\n blank_count = 0\n for line in reversed(self.source):\n line = line.rstrip()\n if line:\n break\n else:\n blank_count += 1\n\n original_length = len(self.source)\n self.source = self.source[:original_length - blank_count]\n return range(1, 1 + original_length)"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef current_size(self):\n size = 0\n for item in reversed(self._lines):\n size += item.size\n if isinstance(item, self._LineBreak):\n break\n\n return size", "response": "The size of the current line minus the indentation."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _add_item(self, item, indent_amt):\n if self._prev_item and self._prev_item.is_string and item.is_string:\n # Place consecutive string literals on separate lines.\n self._lines.append(self._LineBreak())\n self._lines.append(self._Indent(indent_amt))\n\n item_text = unicode(item)\n if self._lines and self._bracket_depth:\n # Adding the item into a container.\n self._prevent_default_initializer_splitting(item, indent_amt)\n\n if item_text in '.,)]}':\n self._split_after_delimiter(item, indent_amt)\n\n elif self._lines and not self.line_empty():\n # Adding the item outside of a container.\n if self.fits_on_current_line(len(item_text)):\n self._enforce_space(item)\n\n else:\n # Line break for the new item.\n self._lines.append(self._LineBreak())\n self._lines.append(self._Indent(indent_amt))\n\n self._lines.append(item)\n self._prev_item, self._prev_prev_item = item, self._prev_item\n\n if item_text in '([{':\n self._bracket_depth += 1\n\n elif item_text in '}])':\n self._bracket_depth -= 1\n assert self._bracket_depth >= 0", "response": "Add an item to the line."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _prevent_default_initializer_splitting(self, item, indent_amt):\n if unicode(item) == '=':\n # This is the assignment in the initializer. Just remove spaces for\n # now.\n self._delete_whitespace()\n return\n\n if (not self._prev_item or not self._prev_prev_item or\n unicode(self._prev_item) != '='):\n return\n\n self._delete_whitespace()\n prev_prev_index = self._lines.index(self._prev_prev_item)\n\n if (\n isinstance(self._lines[prev_prev_index - 1], self._Indent) or\n self.fits_on_current_line(item.size + 1)\n ):\n # The default initializer is already the only item on this line.\n # Don't insert a newline here.\n return\n\n # Replace the space with a newline/indent combo.\n if isinstance(self._lines[prev_prev_index - 1], self._Space):\n del self._lines[prev_prev_index - 1]\n\n self.add_line_break_at(self._lines.index(self._prev_prev_item),\n indent_amt)", "response": "Prevents splitting between a default initializer."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _enforce_space(self, item):\n if isinstance(self._lines[-1],\n (self._Space, self._LineBreak, self._Indent)):\n return\n\n if not self._prev_item:\n return\n\n item_text = unicode(item)\n prev_text = unicode(self._prev_item)\n\n # Prefer a space around a '.' in an import statement, and between the\n # 'import' and '('.\n if (\n (item_text == '.' and prev_text == 'from') or\n (item_text == 'import' and prev_text == '.') or\n (item_text == '(' and prev_text == 'import')\n ):\n self._lines.append(self._Space())", "response": "Enforces a space in certain situations."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _delete_whitespace(self):\n while isinstance(self._lines[-1], (self._Space, self._LineBreak,\n self._Indent)):\n del self._lines[-1]", "response": "Delete all whitespace from the end of the line."} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn the extent of the full element.", "response": "def _get_extent(self, index):\n \"\"\"The extent of the full element.\n\n E.g., the length of a function call or keyword.\n\n \"\"\"\n extent = 0\n prev_item = get_item(self._items, index - 1)\n seen_dot = prev_item and unicode(prev_item) == '.'\n while index < len(self._items):\n item = get_item(self._items, index)\n index += 1\n\n if isinstance(item, (ListComprehension, IfExpression)):\n break\n\n if isinstance(item, Container):\n if prev_item and prev_item.is_name:\n if seen_dot:\n extent += 1\n else:\n extent += item.size\n\n prev_item = item\n continue\n elif (unicode(item) not in ['.', '=', ':', 'not'] and\n not item.is_name and not item.is_string):\n break\n\n if unicode(item) == '.':\n seen_dot = True\n\n extent += item.size\n prev_item = item\n\n return extent"} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns modified line numbers.", "response": "def run(self, indent_size=DEFAULT_INDENT_SIZE):\n \"\"\"Fix indentation and return modified line numbers.\n\n Line numbers are indexed at 1.\n\n \"\"\"\n if indent_size < 1:\n return self.input_text\n\n try:\n stats = _reindent_stats(tokenize.generate_tokens(self.getline))\n except (SyntaxError, tokenize.TokenError):\n return self.input_text\n # Remove trailing empty lines.\n lines = self.lines\n # Sentinel.\n stats.append((len(lines), 0))\n # Map count of leading spaces to # we want.\n have2want = {}\n # Program after transformation.\n after = []\n # Copy over initial empty lines -- there's nothing to do until\n # we see a line with *something* on it.\n i = stats[0][0]\n after.extend(lines[1:i])\n for i in range(len(stats) - 1):\n thisstmt, thislevel = stats[i]\n nextstmt = stats[i + 1][0]\n have = _leading_space_count(lines[thisstmt])\n want = thislevel * indent_size\n if want < 0:\n # A comment line.\n if have:\n # An indented comment line. If we saw the same\n # indentation before, reuse what it most recently\n # mapped to.\n want = have2want.get(have, -1)\n if want < 0:\n # Then it probably belongs to the next real stmt.\n for j in range(i + 1, len(stats) - 1):\n jline, jlevel = stats[j]\n if jlevel >= 0:\n if have == _leading_space_count(lines[jline]):\n want = jlevel * indent_size\n break\n # Maybe it's a hanging comment like this one,\n if want < 0:\n # in which case we should shift it like its base\n # line got shifted.\n for j in range(i - 1, -1, -1):\n jline, jlevel = stats[j]\n if jlevel >= 0:\n want = (have + _leading_space_count(\n after[jline - 1]) -\n _leading_space_count(lines[jline]))\n break\n if want < 0:\n # Still no luck -- leave it alone.\n want = have\n else:\n want = 0\n assert want >= 0\n have2want[have] = want\n diff = want - have\n if diff == 0 or have == 0:\n after.extend(lines[thisstmt:nextstmt])\n else:\n for line_number, line in enumerate(lines[thisstmt:nextstmt],\n start=thisstmt):\n if line_number in self.string_content_line_numbers:\n after.append(line)\n elif diff > 0:\n if line == '\\n':\n after.append(line)\n else:\n after.append(' ' * diff + line)\n else:\n remove = min(_leading_space_count(line), -diff)\n after.append(line[remove:])\n\n return ''.join(after)"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef getline(self):\n if self.index >= len(self.lines):\n line = ''\n else:\n line = self.lines[self.index]\n self.index += 1\n return line", "response": "Line - getter for tokenize."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef generate_tokens(self, text):\n if text != self.last_text:\n string_io = io.StringIO(text)\n self.last_tokens = list(\n tokenize.generate_tokens(string_io.readline)\n )\n self.last_text = text\n return self.last_tokens", "response": "A stand - in for tokenize. generate_tokens"} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreset the filter to state at time of construction", "response": "def reset(self):\n \"\"\" reset filter back to state at time of construction\"\"\"\n\n self.n = 0 # nth step in the recursion\n self.x = np.zeros(self._order + 1)\n self.K = np.zeros(self._order + 1)\n self.y = 0"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nupdate filter with new measurement z", "response": "def update(self, z):\n \"\"\" Update filter with new measurement `z`\n\n Returns\n -------\n\n x : np.array\n estimate for this time step (same as self.x)\n \"\"\"\n\n self.n += 1\n # rename for readability\n n = self.n\n dt = self.dt\n x = self.x\n K = self.K\n y = self.y\n\n if self._order == 0:\n K[0] = 1. / n\n y = z - x\n x[0] += K[0] * y\n\n elif self._order == 1:\n K[0] = 2. * (2*n - 1) / (n*(n + 1))\n K[1] = 6. / (n*(n + 1)*dt)\n\n y = z - x[0] - (dt * x[1])\n\n x[0] += (K[0] * y) + (dt * x[1])\n x[1] += (K[1] * y)\n\n else:\n den = n * (n+1) * (n+2)\n K[0] = 3. * (3*n**2 - 3*n + 2) / den\n K[1] = 18. * (2*n-1) / (den*dt)\n K[2] = 60. / (den*dt**2)\n\n y = z - x[0] - (dt * x[1]) - (0.5 * dt**2 * x[2])\n\n x[0] += (K[0] * y) + (x[1] * dt) + (.5 * dt**2 * x[2])\n x[1] += (K[1] * y) + (x[2] * dt)\n x[2] += (K[2] * y)\n return self.x"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncomputing and returns the error and standard deviation of the .", "response": "def errors(self):\n \"\"\"\n Computes and returns the error and standard deviation of the\n filter at this time step.\n\n Returns\n -------\n\n error : np.array size 1xorder+1\n std : np.array size 1xorder+1\n \"\"\"\n\n n = self.n\n dt = self.dt\n order = self._order\n sigma = self.sigma\n\n error = np.zeros(order + 1)\n std = np.zeros(order + 1)\n\n if n == 0:\n return (error, std)\n\n if order == 0:\n error[0] = sigma/sqrt(n)\n std[0] = sigma/sqrt(n)\n\n elif order == 1:\n if n > 1:\n error[0] = sigma * sqrt(2*(2*n-1) / (n*(n+1)))\n error[1] = sigma * sqrt(12. / (n*(n*n-1)*dt*dt))\n std[0] = sigma * sqrt((2*(2*n-1)) / (n*(n+1)))\n std[1] = (sigma/dt) * sqrt(12. / (n*(n*n-1)))\n\n elif order == 2:\n dt2 = dt * dt\n\n if n >= 3:\n error[0] = sigma * sqrt(3*(3*n*n-3*n+2) / (n*(n+1)*(n+2)))\n error[1] = sigma * sqrt(12*(16*n*n-30*n+11) /\n (n*(n*n-1)*(n*n-4)*dt2))\n error[2] = sigma * sqrt(720/(n*(n*n-1)*(n*n-4)*dt2*dt2))\n\n std[0] = sigma * sqrt((3*(3*n*n - 3*n + 2)) / (n*(n+1)*(n+2)))\n std[1] = (sigma/dt) * sqrt((12*(16*n*n - 30*n + 11)) /\n (n*(n*n - 1)*(n*n - 4)))\n std[2] = (sigma/dt2) * sqrt(720 / (n*(n*n-1)*(n*n-4)))\n\n return error, std"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nadds a new measurement (z) to the Kalman filter. If z is None, nothing is changed. This can handle either the multidimensional or unidimensional case. If all parameters are floats instead of arrays the filter will still work, and return floats for x, P as the result. update(1, 2, 1, 1, 1) # univariate update(x, P, 1 Parameters ---------- x : numpy.array(dim_x, 1), or float State estimate vector P : numpy.array(dim_x, dim_x), or float Covariance matrix z : (dim_z, 1): array_like measurement for this update. z can be a scalar if dim_z is 1, otherwise it must be convertible to a column vector. R : numpy.array(dim_z, dim_z), or float Measurement noise matrix H : numpy.array(dim_x, dim_x), or float, optional Measurement function. If not provided, a value of 1 is assumed. return_all : bool, default False If true, y, K, S, and log_likelihood are returned, otherwise only x and P are returned. Returns ------- x : numpy.array Posterior state estimate vector P : numpy.array Posterior covariance matrix y : numpy.array or scalar Residua. Difference between measurement and state in measurement space K : numpy.array Kalman gain S : numpy.array System uncertainty in measurement space log_likelihood : float log likelihood of the measurement", "response": "def update(x, P, z, R, H=None, return_all=False):\n \"\"\"\n Add a new measurement (z) to the Kalman filter. If z is None, nothing\n is changed.\n\n This can handle either the multidimensional or unidimensional case. If\n all parameters are floats instead of arrays the filter will still work,\n and return floats for x, P as the result.\n\n update(1, 2, 1, 1, 1) # univariate\n update(x, P, 1\n\n\n\n Parameters\n ----------\n\n x : numpy.array(dim_x, 1), or float\n State estimate vector\n\n P : numpy.array(dim_x, dim_x), or float\n Covariance matrix\n\n z : (dim_z, 1): array_like\n measurement for this update. z can be a scalar if dim_z is 1,\n otherwise it must be convertible to a column vector.\n\n R : numpy.array(dim_z, dim_z), or float\n Measurement noise matrix\n\n H : numpy.array(dim_x, dim_x), or float, optional\n Measurement function. If not provided, a value of 1 is assumed.\n\n return_all : bool, default False\n If true, y, K, S, and log_likelihood are returned, otherwise\n only x and P are returned.\n\n Returns\n -------\n\n x : numpy.array\n Posterior state estimate vector\n\n P : numpy.array\n Posterior covariance matrix\n\n y : numpy.array or scalar\n Residua. Difference between measurement and state in measurement space\n\n K : numpy.array\n Kalman gain\n\n S : numpy.array\n System uncertainty in measurement space\n\n log_likelihood : float\n log likelihood of the measurement\n \"\"\"\n\n #pylint: disable=bare-except\n\n if z is None:\n if return_all:\n return x, P, None, None, None, None\n return x, P\n\n if H is None:\n H = np.array([1])\n\n if np.isscalar(H):\n H = np.array([H])\n\n Hx = np.atleast_1d(dot(H, x))\n z = reshape_z(z, Hx.shape[0], x.ndim)\n\n # error (residual) between measurement and prediction\n y = z - Hx\n\n # project system uncertainty into measurement space\n S = dot(dot(H, P), H.T) + R\n\n\n # map system uncertainty into kalman gain\n try:\n K = dot(dot(P, H.T), linalg.inv(S))\n except:\n # can't invert a 1D array, annoyingly\n K = dot(dot(P, H.T), 1./S)\n\n\n # predict new x with residual scaled by the kalman gain\n x = x + dot(K, y)\n\n # P = (I-KH)P(I-KH)' + KRK'\n KH = dot(K, H)\n\n try:\n I_KH = np.eye(KH.shape[0]) - KH\n except:\n I_KH = np.array([1 - KH])\n P = dot(dot(I_KH, P), I_KH.T) + dot(dot(K, R), K.T)\n\n\n if return_all:\n # compute log likelihood\n log_likelihood = logpdf(z, dot(H, x), S)\n return x, P, y, K, S, log_likelihood\n return x, P"} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\npredicts next state using the Kalman filter state propagation and prior state propagation equations.", "response": "def predict(x, P, F=1, Q=0, u=0, B=1, alpha=1.):\n \"\"\"\n Predict next state (prior) using the Kalman filter state propagation\n equations.\n\n Parameters\n ----------\n\n x : numpy.array\n State estimate vector\n\n P : numpy.array\n Covariance matrix\n\n F : numpy.array()\n State Transition matrix\n\n Q : numpy.array, Optional\n Process noise matrix\n\n\n u : numpy.array, Optional, default 0.\n Control vector. If non-zero, it is multiplied by B\n to create the control input into the system.\n\n B : numpy.array, optional, default 0.\n Control transition matrix.\n\n alpha : float, Optional, default=1.0\n Fading memory setting. 1.0 gives the normal Kalman filter, and\n values slightly larger than 1.0 (such as 1.02) give a fading\n memory effect - previous measurements have less influence on the\n filter's estimates. This formulation of the Fading memory filter\n (there are many) is due to Dan Simon\n\n Returns\n -------\n\n x : numpy.array\n Prior state estimate vector\n\n P : numpy.array\n Prior covariance matrix\n \"\"\"\n\n if np.isscalar(F):\n F = np.array(F)\n x = dot(F, x) + dot(B, u)\n P = (alpha * alpha) * dot(dot(F, P), F.T) + Q\n\n return x, P"} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\npredicts next state of a set of states in a Kalman filter state propagation system using the Kalman filter state propagation system equations. This steady state form only computes x assuming that the covariance matrix is constant.", "response": "def predict_steadystate(x, F=1, u=0, B=1):\n \"\"\"\n Predict next state (prior) using the Kalman filter state propagation\n equations. This steady state form only computes x, assuming that the\n covariance is constant.\n\n Parameters\n ----------\n\n x : numpy.array\n State estimate vector\n\n P : numpy.array\n Covariance matrix\n\n F : numpy.array()\n State Transition matrix\n\n u : numpy.array, Optional, default 0.\n Control vector. If non-zero, it is multiplied by B\n to create the control input into the system.\n\n B : numpy.array, optional, default 0.\n Control transition matrix.\n\n Returns\n -------\n\n x : numpy.array\n Prior state estimate vector\n \"\"\"\n\n if np.isscalar(F):\n F = np.array(F)\n x = dot(F, x) + dot(B, u)\n\n return x"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef batch_filter(x, P, zs, Fs, Qs, Hs, Rs, Bs=None, us=None,\n update_first=False, saver=None):\n \"\"\"\n Batch processes a sequences of measurements.\n\n Parameters\n ----------\n\n zs : list-like\n list of measurements at each time step. Missing measurements must be\n represented by None.\n\n Fs : list-like\n list of values to use for the state transition matrix matrix.\n\n Qs : list-like\n list of values to use for the process error\n covariance.\n\n Hs : list-like\n list of values to use for the measurement matrix.\n\n Rs : list-like\n list of values to use for the measurement error\n covariance.\n\n Bs : list-like, optional\n list of values to use for the control transition matrix;\n a value of None in any position will cause the filter\n to use `self.B` for that time step.\n\n us : list-like, optional\n list of values to use for the control input vector;\n a value of None in any position will cause the filter to use\n 0 for that time step.\n\n update_first : bool, optional\n controls whether the order of operations is update followed by\n predict, or predict followed by update. Default is predict->update.\n\n saver : filterpy.common.Saver, optional\n filterpy.common.Saver object. If provided, saver.save() will be\n called after every epoch\n\n Returns\n -------\n\n means : np.array((n,dim_x,1))\n array of the state for each time step after the update. Each entry\n is an np.array. In other words `means[k,:]` is the state at step\n `k`.\n\n covariance : np.array((n,dim_x,dim_x))\n array of the covariances for each time step after the update.\n In other words `covariance[k,:,:]` is the covariance at step `k`.\n\n means_predictions : np.array((n,dim_x,1))\n array of the state for each time step after the predictions. Each\n entry is an np.array. In other words `means[k,:]` is the state at\n step `k`.\n\n covariance_predictions : np.array((n,dim_x,dim_x))\n array of the covariances for each time step after the prediction.\n In other words `covariance[k,:,:]` is the covariance at step `k`.\n\n Examples\n --------\n\n .. code-block:: Python\n\n zs = [t + random.randn()*4 for t in range (40)]\n Fs = [kf.F for t in range (40)]\n Hs = [kf.H for t in range (40)]\n\n (mu, cov, _, _) = kf.batch_filter(zs, Rs=R_list, Fs=Fs, Hs=Hs, Qs=None,\n Bs=None, us=None, update_first=False)\n (xs, Ps, Ks) = kf.rts_smoother(mu, cov, Fs=Fs, Qs=None)\n\n \"\"\"\n\n n = np.size(zs, 0)\n dim_x = x.shape[0]\n\n # mean estimates from Kalman Filter\n if x.ndim == 1:\n means = zeros((n, dim_x))\n means_p = zeros((n, dim_x))\n else:\n means = zeros((n, dim_x, 1))\n means_p = zeros((n, dim_x, 1))\n\n # state covariances from Kalman Filter\n covariances = zeros((n, dim_x, dim_x))\n covariances_p = zeros((n, dim_x, dim_x))\n\n if us is None:\n us = [0.] * n\n Bs = [0.] * n\n\n if update_first:\n for i, (z, F, Q, H, R, B, u) in enumerate(zip(zs, Fs, Qs, Hs, Rs, Bs, us)):\n\n x, P = update(x, P, z, R=R, H=H)\n means[i, :] = x\n covariances[i, :, :] = P\n\n x, P = predict(x, P, u=u, B=B, F=F, Q=Q)\n means_p[i, :] = x\n covariances_p[i, :, :] = P\n if saver is not None:\n saver.save()\n else:\n for i, (z, F, Q, H, R, B, u) in enumerate(zip(zs, Fs, Qs, Hs, Rs, Bs, us)):\n\n x, P = predict(x, P, u=u, B=B, F=F, Q=Q)\n means_p[i, :] = x\n covariances_p[i, :, :] = P\n\n x, P = update(x, P, z, R=R, H=H)\n means[i, :] = x\n covariances[i, :, :] = P\n if saver is not None:\n saver.save()\n\n return (means, covariances, means_p, covariances_p)", "response": "This function processes a sequence of measurements and returns a list of states that can be used to filter the states of the control input vectors at each time step."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef rts_smoother(Xs, Ps, Fs, Qs):\n\n if len(Xs) != len(Ps):\n raise ValueError('length of Xs and Ps must be the same')\n\n n = Xs.shape[0]\n dim_x = Xs.shape[1]\n\n # smoother gain\n K = zeros((n, dim_x, dim_x))\n x, P, pP = Xs.copy(), Ps.copy(), Ps.copy()\n\n for k in range(n-2, -1, -1):\n pP[k] = dot(dot(Fs[k], P[k]), Fs[k].T) + Qs[k]\n\n #pylint: disable=bad-whitespace\n K[k] = dot(dot(P[k], Fs[k].T), linalg.inv(pP[k]))\n x[k] += dot(K[k], x[k+1] - dot(Fs[k], x[k]))\n P[k] += dot(dot(K[k], P[k+1] - pP[k]), K[k].T)\n\n return (x, P, K, pP)", "response": "Runs the Rauch - Tung - Striebal Kalman smoother on a set of state variables Xs Ps and Qs."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef predict(self, u=None, B=None, F=None, Q=None):\n\n if B is None:\n B = self.B\n if F is None:\n F = self.F\n if Q is None:\n Q = self.Q\n elif isscalar(Q):\n Q = eye(self.dim_x) * Q\n\n # x = Fx + Bu\n if B is not None and u is not None:\n self.x = dot(F, self.x) + dot(B, u)\n else:\n self.x = dot(F, self.x)\n\n # P = FPF' + Q\n self.P = self._alpha_sq * dot(dot(F, self.P), F.T) + Q\n\n # save prior\n self.x_prior = self.x.copy()\n self.P_prior = self.P.copy()", "response": "Predict next state of the state propagation system using the Kalman filter state propagation system equations."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef predict_steadystate(self, u=0, B=None):\n\n if B is None:\n B = self.B\n\n # x = Fx + Bu\n if B is not None:\n self.x = dot(self.F, self.x) + dot(B, u)\n else:\n self.x = dot(self.F, self.x)\n\n # save prior\n self.x_prior = self.x.copy()\n self.P_prior = self.P.copy()", "response": "Predict state of the state of the target system using the Kalman filter state propagation."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef update_steadystate(self, z):\n\n # set to None to force recompute\n self._log_likelihood = None\n self._likelihood = None\n self._mahalanobis = None\n\n if z is None:\n self.z = np.array([[None]*self.dim_z]).T\n self.x_post = self.x.copy()\n self.P_post = self.P.copy()\n self.y = zeros((self.dim_z, 1))\n return\n\n z = reshape_z(z, self.dim_z, self.x.ndim)\n\n # y = z - Hx\n # error (residual) between measurement and prediction\n self.y = z - dot(self.H, self.x)\n\n # x = x + Ky\n # predict new x with residual scaled by the kalman gain\n self.x = self.x + dot(self.K, self.y)\n\n self.z = deepcopy(z)\n self.x_post = self.x.copy()\n self.P_post = self.P.copy()\n\n # set to None to force recompute\n self._log_likelihood = None\n self._likelihood = None\n self._mahalanobis = None", "response": "Update the state of the current object with the new measurement z."} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nupdate the correlated state of the Kalman filter assuming that the process noise and measurement noise are correlated.", "response": "def update_correlated(self, z, R=None, H=None):\n \"\"\" Add a new measurement (z) to the Kalman filter assuming that\n process noise and measurement noise are correlated as defined in\n the `self.M` matrix.\n\n If z is None, nothing is changed.\n\n Parameters\n ----------\n z : (dim_z, 1): array_like\n measurement for this update. z can be a scalar if dim_z is 1,\n otherwise it must be convertible to a column vector.\n\n R : np.array, scalar, or None\n Optionally provide R to override the measurement noise for this\n one call, otherwise self.R will be used.\n\n H : np.array, or None\n Optionally provide H to override the measurement function for this\n one call, otherwise self.H will be used.\n \"\"\"\n\n # set to None to force recompute\n self._log_likelihood = None\n self._likelihood = None\n self._mahalanobis = None\n\n if z is None:\n self.z = np.array([[None]*self.dim_z]).T\n self.x_post = self.x.copy()\n self.P_post = self.P.copy()\n self.y = zeros((self.dim_z, 1))\n return\n\n z = reshape_z(z, self.dim_z, self.x.ndim)\n\n if R is None:\n R = self.R\n elif isscalar(R):\n R = eye(self.dim_z) * R\n\n # rename for readability and a tiny extra bit of speed\n if H is None:\n H = self.H\n\n # handle special case: if z is in form [[z]] but x is not a column\n # vector dimensions will not match\n if self.x.ndim == 1 and shape(z) == (1, 1):\n z = z[0]\n\n if shape(z) == (): # is it scalar, e.g. z=3 or z=np.array(3)\n z = np.asarray([z])\n\n # y = z - Hx\n # error (residual) between measurement and prediction\n self.y = z - dot(H, self.x)\n\n # common subexpression for speed\n PHT = dot(self.P, H.T)\n\n # project system uncertainty into measurement space\n self.S = dot(H, PHT) + dot(H, self.M) + dot(self.M.T, H.T) + R\n self.SI = self.inv(self.S)\n\n # K = PH'inv(S)\n # map system uncertainty into kalman gain\n self.K = dot(PHT + self.M, self.SI)\n\n # x = x + Ky\n # predict new x with residual scaled by the kalman gain\n self.x = self.x + dot(self.K, self.y)\n self.P = self.P - dot(self.K, dot(H, self.P) + self.M.T)\n\n self.z = deepcopy(z)\n self.x_post = self.x.copy()\n self.P_post = self.P.copy()"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef batch_filter(self, zs, Fs=None, Qs=None, Hs=None,\n Rs=None, Bs=None, us=None, update_first=False,\n saver=None):\n \"\"\" Batch processes a sequences of measurements.\n\n Parameters\n ----------\n\n zs : list-like\n list of measurements at each time step `self.dt`. Missing\n measurements must be represented by `None`.\n\n Fs : None, list-like, default=None\n optional value or list of values to use for the state transition\n matrix F.\n\n If Fs is None then self.F is used for all epochs.\n\n Otherwise it must contain a list-like list of F's, one for\n each epoch. This allows you to have varying F per epoch.\n\n Qs : None, np.array or list-like, default=None\n optional value or list of values to use for the process error\n covariance Q.\n\n If Qs is None then self.Q is used for all epochs.\n\n Otherwise it must contain a list-like list of Q's, one for\n each epoch. This allows you to have varying Q per epoch.\n\n Hs : None, np.array or list-like, default=None\n optional list of values to use for the measurement matrix H.\n\n If Hs is None then self.H is used for all epochs.\n\n If Hs contains a single matrix, then it is used as H for all\n epochs.\n\n Otherwise it must contain a list-like list of H's, one for\n each epoch. This allows you to have varying H per epoch.\n\n Rs : None, np.array or list-like, default=None\n optional list of values to use for the measurement error\n covariance R.\n\n If Rs is None then self.R is used for all epochs.\n\n Otherwise it must contain a list-like list of R's, one for\n each epoch. This allows you to have varying R per epoch.\n\n Bs : None, np.array or list-like, default=None\n optional list of values to use for the control transition matrix B.\n\n If Bs is None then self.B is used for all epochs.\n\n Otherwise it must contain a list-like list of B's, one for\n each epoch. This allows you to have varying B per epoch.\n\n us : None, np.array or list-like, default=None\n optional list of values to use for the control input vector;\n\n If us is None then None is used for all epochs (equivalent to 0,\n or no control input).\n\n Otherwise it must contain a list-like list of u's, one for\n each epoch.\n\n update_first : bool, optional, default=False\n controls whether the order of operations is update followed by\n predict, or predict followed by update. Default is predict->update.\n\n saver : filterpy.common.Saver, optional\n filterpy.common.Saver object. If provided, saver.save() will be\n called after every epoch\n\n Returns\n -------\n\n means : np.array((n,dim_x,1))\n array of the state for each time step after the update. Each entry\n is an np.array. In other words `means[k,:]` is the state at step\n `k`.\n\n covariance : np.array((n,dim_x,dim_x))\n array of the covariances for each time step after the update.\n In other words `covariance[k,:,:]` is the covariance at step `k`.\n\n means_predictions : np.array((n,dim_x,1))\n array of the state for each time step after the predictions. Each\n entry is an np.array. In other words `means[k,:]` is the state at\n step `k`.\n\n covariance_predictions : np.array((n,dim_x,dim_x))\n array of the covariances for each time step after the prediction.\n In other words `covariance[k,:,:]` is the covariance at step `k`.\n\n Examples\n --------\n\n .. code-block:: Python\n\n # this example demonstrates tracking a measurement where the time\n # between measurement varies, as stored in dts. This requires\n # that F be recomputed for each epoch. The output is then smoothed\n # with an RTS smoother.\n\n zs = [t + random.randn()*4 for t in range (40)]\n Fs = [np.array([[1., dt], [0, 1]] for dt in dts]\n\n (mu, cov, _, _) = kf.batch_filter(zs, Fs=Fs)\n (xs, Ps, Ks) = kf.rts_smoother(mu, cov, Fs=Fs)\n \"\"\"\n\n #pylint: disable=too-many-statements\n n = np.size(zs, 0)\n if Fs is None:\n Fs = [self.F] * n\n if Qs is None:\n Qs = [self.Q] * n\n if Hs is None:\n Hs = [self.H] * n\n if Rs is None:\n Rs = [self.R] * n\n if Bs is None:\n Bs = [self.B] * n\n if us is None:\n us = [0] * n\n\n # mean estimates from Kalman Filter\n if self.x.ndim == 1:\n means = zeros((n, self.dim_x))\n means_p = zeros((n, self.dim_x))\n else:\n means = zeros((n, self.dim_x, 1))\n means_p = zeros((n, self.dim_x, 1))\n\n # state covariances from Kalman Filter\n covariances = zeros((n, self.dim_x, self.dim_x))\n covariances_p = zeros((n, self.dim_x, self.dim_x))\n\n if update_first:\n for i, (z, F, Q, H, R, B, u) in enumerate(zip(zs, Fs, Qs, Hs, Rs, Bs, us)):\n\n self.update(z, R=R, H=H)\n means[i, :] = self.x\n covariances[i, :, :] = self.P\n\n self.predict(u=u, B=B, F=F, Q=Q)\n means_p[i, :] = self.x\n covariances_p[i, :, :] = self.P\n\n if saver is not None:\n saver.save()\n else:\n for i, (z, F, Q, H, R, B, u) in enumerate(zip(zs, Fs, Qs, Hs, Rs, Bs, us)):\n\n self.predict(u=u, B=B, F=F, Q=Q)\n means_p[i, :] = self.x\n covariances_p[i, :, :] = self.P\n\n self.update(z, R=R, H=H)\n means[i, :] = self.x\n covariances[i, :, :] = self.P\n\n if saver is not None:\n saver.save()\n\n return (means, covariances, means_p, covariances_p)", "response": "This method is used to filter out missing measurements at each time step."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef rts_smoother(self, Xs, Ps, Fs=None, Qs=None, inv=np.linalg.inv):\n\n if len(Xs) != len(Ps):\n raise ValueError('length of Xs and Ps must be the same')\n\n n = Xs.shape[0]\n dim_x = Xs.shape[1]\n\n if Fs is None:\n Fs = [self.F] * n\n if Qs is None:\n Qs = [self.Q] * n\n\n # smoother gain\n K = zeros((n, dim_x, dim_x))\n\n x, P, Pp = Xs.copy(), Ps.copy(), Ps.copy()\n for k in range(n-2, -1, -1):\n Pp[k] = dot(dot(Fs[k+1], P[k]), Fs[k+1].T) + Qs[k+1]\n\n #pylint: disable=bad-whitespace\n K[k] = dot(dot(P[k], Fs[k+1].T), inv(Pp[k]))\n x[k] += dot(K[k], x[k+1] - dot(Fs[k+1], x[k]))\n P[k] += dot(dot(K[k], P[k+1] - Pp[k]), K[k].T)\n\n return (x, P, K, Pp)", "response": "Runs the Rauch - Tung - Striebal Kalman smoother on a set of state variables Xs Ps Qs and Pp."} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\npredicting the next state of the filter and returns it without altering the state of the filter.", "response": "def get_prediction(self, u=0):\n \"\"\"\n Predicts the next state of the filter and returns it without\n altering the state of the filter.\n\n Parameters\n ----------\n\n u : np.array\n optional control input\n\n Returns\n -------\n\n (x, P) : tuple\n State vector and covariance array of the prediction.\n \"\"\"\n\n x = dot(self.F, self.x) + dot(self.B, u)\n P = self._alpha_sq * dot(dot(self.F, self.P), self.F.T) + self.Q\n return (x, P)"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_update(self, z=None):\n\n if z is None:\n return self.x, self.P\n z = reshape_z(z, self.dim_z, self.x.ndim)\n\n R = self.R\n H = self.H\n P = self.P\n x = self.x\n\n # error (residual) between measurement and prediction\n y = z - dot(H, x)\n\n # common subexpression for speed\n PHT = dot(P, H.T)\n\n # project system uncertainty into measurement space\n S = dot(H, PHT) + R\n\n # map system uncertainty into kalman gain\n K = dot(PHT, self.inv(S))\n\n # predict new x with residual scaled by the kalman gain\n x = x + dot(K, y)\n\n # P = (I-KH)P(I-KH)' + KRK'\n I_KH = self._I - dot(K, H)\n P = dot(dot(I_KH, P), I_KH.T) + dot(dot(K, R), K.T)\n\n return x, P", "response": "Computes the new estimate based on measurement z and returns the new estimate x and P."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nlogging - likelihood of the last measurement.", "response": "def log_likelihood(self):\n \"\"\"\n log-likelihood of the last measurement.\n \"\"\"\n if self._log_likelihood is None:\n self._log_likelihood = logpdf(x=self.y, cov=self.S)\n return self._log_likelihood"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ncomputing the likelihood of the a .", "response": "def likelihood(self):\n \"\"\"\n Computed from the log-likelihood. The log-likelihood can be very\n small, meaning a large negative value such as -28000. Taking the\n exp() of that results in 0.0, which can break typical algorithms\n which multiply by this value, so by default we always return a\n number >= sys.float_info.min.\n \"\"\"\n if self._likelihood is None:\n self._likelihood = exp(self.log_likelihood)\n if self._likelihood == 0:\n self._likelihood = sys.float_info.min\n return self._likelihood"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef mahalanobis(self):\n if self._mahalanobis is None:\n self._mahalanobis = sqrt(float(dot(dot(self.y.T, self.SI), self.y)))\n return self._mahalanobis", "response": "Returns the Mahalanobis distance of the measurement."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn the log likelihood of the measurement z.", "response": "def log_likelihood_of(self, z):\n \"\"\"\n log likelihood of the measurement `z`. This should only be called\n after a call to update(). Calling after predict() will yield an\n incorrect result.\"\"\"\n\n if z is None:\n return log(sys.float_info.min)\n return logpdf(z, dot(self.H, self.x), self.S)"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef sigma_points(self, x, P):\n\n if self.n != np.size(x):\n raise ValueError(\"expected size(x) {}, but size is {}\".format(\n self.n, np.size(x)))\n\n n = self.n\n\n if np.isscalar(x):\n x = np.asarray([x])\n\n if np.isscalar(P):\n P = np.eye(n)*P\n else:\n P = np.atleast_2d(P)\n\n lambda_ = self.alpha**2 * (n + self.kappa) - n\n U = self.sqrt((lambda_ + n)*P)\n\n sigmas = np.zeros((2*n+1, n))\n sigmas[0] = x\n for k in range(n):\n # pylint: disable=bad-whitespace\n sigmas[k+1] = self.subtract(x, -U[k])\n sigmas[n+k+1] = self.subtract(x, U[k])\n\n return sigmas", "response": "Computes the sigma points for an unscented Kalman filter."} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ncomputes the weights for the scaled unscented Kalman filter.", "response": "def _compute_weights(self):\n \"\"\" Computes the weights for the scaled unscented Kalman filter.\n\n \"\"\"\n\n n = self.n\n lambda_ = self.alpha**2 * (n +self.kappa) - n\n\n c = .5 / (n + lambda_)\n self.Wc = np.full(2*n + 1, c)\n self.Wm = np.full(2*n + 1, c)\n self.Wc[0] = lambda_ / (n + lambda_) + (1 - self.alpha**2 + self.beta)\n self.Wm[0] = lambda_ / (n + lambda_)"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncompute the weights for the unscented Kalman filter.", "response": "def _compute_weights(self):\n \"\"\" Computes the weights for the unscented Kalman filter. In this\n formulation the weights for the mean and covariance are the same.\n \"\"\"\n\n n = self.n\n k = self.kappa\n\n self.Wm = np.full(2*n+1, .5 / (n + k))\n self.Wm[0] = k / (n+k)\n self.Wc = self.Wm"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef sigma_points(self, x, P):\n\n if self.n != np.size(x):\n raise ValueError(\"expected size(x) {}, but size is {}\".format(\n self.n, np.size(x)))\n\n n = self.n\n\n if np.isscalar(x):\n x = np.asarray([x])\n x = x.reshape(-1, 1)\n\n if np.isscalar(P):\n P = np.eye(n) * P\n else:\n P = np.atleast_2d(P)\n\n U = self.sqrt(P)\n\n lambda_ = n / (n + 1)\n Istar = np.array([[-1/np.sqrt(2*lambda_), 1/np.sqrt(2*lambda_)]])\n for d in range(2, n+1):\n row = np.ones((1, Istar.shape[1] + 1)) * 1. / np.sqrt(lambda_*d*(d + 1))\n row[0, -1] = -d / np.sqrt(lambda_ * d * (d + 1))\n Istar = np.r_[np.c_[Istar, np.zeros((Istar.shape[0]))], row]\n\n I = np.sqrt(n)*Istar\n scaled_unitary = U.dot(I)\n\n sigmas = self.subtract(x, -scaled_unitary)\n return sigmas.T", "response": "Computes the sigma points for an unscented Kalman filter."} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncomputing the weights for the scaled unscented Kalman filter.", "response": "def _compute_weights(self):\n \"\"\" Computes the weights for the scaled unscented Kalman filter. \"\"\"\n\n n = self.n\n c = 1. / (n + 1)\n self.Wm = np.full(n + 1, c)\n self.Wc = self.Wm"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef mahalanobis(x, mean, cov):\n\n x = _validate_vector(x)\n mean = _validate_vector(mean)\n\n if x.shape != mean.shape:\n raise ValueError(\"length of input vectors must be the same\")\n\n y = x - mean\n S = np.atleast_2d(cov)\n\n dist = float(np.dot(np.dot(y.T, inv(S)), y))\n return math.sqrt(dist)", "response": "Computes the Mahalanobis distance between the state vector x and the state vector mean with covariance cov."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef log_likelihood(z, x, P, H, R):\n S = np.dot(H, np.dot(P, H.T)) + R\n return logpdf(z, np.dot(H, x), S)", "response": "Returns the log - likelihood of the measurement z given the Gaussian\n posterior x P and measurement function H and measurement covariance error R."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef likelihood(z, x, P, H, R):\n return np.exp(log_likelihood(z, x, P, H, R))", "response": "Returns the likelihood of the measurement z given the Gaussian\n posterior x P H and measurement function R"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef logpdf(x, mean=None, cov=1, allow_singular=True):\n\n if mean is not None:\n flat_mean = np.asarray(mean).flatten()\n else:\n flat_mean = None\n\n flat_x = np.asarray(x).flatten()\n\n if _support_singular:\n return multivariate_normal.logpdf(flat_x, flat_mean, cov, allow_singular)\n return multivariate_normal.logpdf(flat_x, flat_mean, cov)", "response": "Computes the log of the probability density function of the normal of the data x."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn a normal distribution for x given a Gaussian with the specified mean and variance.", "response": "def gaussian(x, mean, var, normed=True):\n \"\"\"\n returns normal distribution (pdf) for x given a Gaussian with the\n specified mean and variance. All must be scalars.\n\n gaussian (1,2,3) is equivalent to scipy.stats.norm(2,math.sqrt(3)).pdf(1)\n It is quite a bit faster albeit much less flexible than the latter.\n\n Parameters\n ----------\n\n x : scalar or array-like\n The value for which we compute the probability\n\n mean : scalar\n Mean of the Gaussian\n\n var : scalar\n Variance of the Gaussian\n\n norm : bool, default True\n Normalize the output if the input is an array of values.\n\n Returns\n -------\n\n probability : float\n probability of x for the Gaussian (mean, var). E.g. 0.101 denotes\n 10.1%.\n\n Examples\n --------\n\n >>> gaussian(8, 1, 2)\n 1.3498566943461957e-06\n\n >>> gaussian([8, 7, 9], 1, 2)\n array([1.34985669e-06, 3.48132630e-05, 3.17455867e-08])\n \"\"\"\n\n g = ((2*math.pi*var)**-.5) * np.exp((-0.5*(np.asarray(x)-mean)**2.) / var)\n if normed and len(np.shape(g)) > 0:\n g = g / sum(g)\n\n return g"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef mul(mean1, var1, mean2, var2):\n\n mean = (var1*mean2 + var2*mean1) / (var1 + var2)\n var = 1 / (1/var1 + 1/var2)\n return (mean, var)", "response": "Multiply two Gaussian terms and return the resulting Gaussian terms as a tuple."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nmultiplies two Gaussian terms and return the result as a tuple.", "response": "def mul_pdf(mean1, var1, mean2, var2):\n \"\"\"\n Multiply Gaussian (mean1, var1) with (mean2, var2) and return the\n results as a tuple (mean, var, scale_factor).\n\n Strictly speaking the product of two Gaussian PDFs is a Gaussian\n function, not Gaussian PDF. It is, however, proportional to a Gaussian\n PDF. `scale_factor` provides this proportionality constant\n\n Parameters\n ----------\n mean1 : scalar\n mean of first Gaussian\n\n var1 : scalar\n variance of first Gaussian\n\n mean2 : scalar\n mean of second Gaussian\n\n var2 : scalar\n variance of second Gaussian\n\n Returns\n -------\n mean : scalar\n mean of product\n\n var : scalar\n variance of product\n\n scale_factor : scalar\n proportionality constant\n\n\n Examples\n --------\n >>> mul(1, 2, 3, 4)\n (1.6666666666666667, 1.3333333333333333)\n\n References\n ----------\n Bromily. \"Products and Convolutions of Gaussian Probability Functions\",\n Tina Memo No. 2003-003.\n http://www.tina-vision.net/docs/memos/2003-003.pdf\n \"\"\"\n\n mean = (var1*mean2 + var2*mean1) / (var1 + var2)\n var = 1. / (1./var1 + 1./var2)\n\n S = math.exp(-(mean1 - mean2)**2 / (2*(var1 + var2))) / \\\n math.sqrt(2 * math.pi * (var1 + var2))\n\n return mean, var, S"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef multivariate_gaussian(x, mu, cov):\n\n warnings.warn(\n (\"This was implemented before SciPy version 0.14, which implemented \"\n \"scipy.stats.multivariate_normal. This function will be removed in \"\n \"a future release of FilterPy\"), DeprecationWarning)\n\n # force all to numpy.array type, and flatten in case they are vectors\n x = np.array(x, copy=False, ndmin=1).flatten()\n mu = np.array(mu, copy=False, ndmin=1).flatten()\n\n nx = len(mu)\n cov = _to_cov(cov, nx)\n\n\n norm_coeff = nx*math.log(2*math.pi) + np.linalg.slogdet(cov)[1]\n\n err = x - mu\n if sp.issparse(cov):\n numerator = spln.spsolve(cov, err).T.dot(err)\n else:\n numerator = np.linalg.solve(cov, err).T.dot(err)\n\n return math.exp(-0.5*(norm_coeff + numerator))", "response": "This function is used to compute the probability for a multivariate Gaussian."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef multivariate_multiply(m1, c1, m2, c2):\n\n C1 = np.asarray(c1)\n C2 = np.asarray(c2)\n M1 = np.asarray(m1)\n M2 = np.asarray(m2)\n\n sum_inv = np.linalg.inv(C1+C2)\n C3 = np.dot(C1, sum_inv).dot(C2)\n\n M3 = (np.dot(C2, sum_inv).dot(M1) +\n np.dot(C1, sum_inv).dot(M2))\n\n return M3, C3", "response": "Multiplies two multivariate Gaussians together and returns the result as the tuple M3 and C3."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef plot_discrete_cdf(xs, ys, ax=None, xlabel=None, ylabel=None,\n label=None):\n \"\"\"\n Plots a normal distribution CDF with the given mean and variance.\n x-axis contains the mean, the y-axis shows the cumulative probability.\n\n Parameters\n ----------\n\n xs : list-like of scalars\n x values corresponding to the values in `y`s. Can be `None`, in which\n case range(len(ys)) will be used.\n\n ys : list-like of scalars\n list of probabilities to be plotted which should sum to 1.\n\n ax : matplotlib axes object, optional\n If provided, the axes to draw on, otherwise plt.gca() is used.\n\n xlim, ylim: (float,float), optional\n specify the limits for the x or y axis as tuple (low,high).\n If not specified, limits will be automatically chosen to be 'nice'\n\n xlabel : str,optional\n label for the x-axis\n\n ylabel : str, optional\n label for the y-axis\n\n label : str, optional\n label for the legend\n\n Returns\n -------\n axis of plot\n \"\"\"\n import matplotlib.pyplot as plt\n\n if ax is None:\n ax = plt.gca()\n\n if xs is None:\n xs = range(len(ys))\n ys = np.cumsum(ys)\n ax.plot(xs, ys, label=label)\n ax.set_xlabel(xlabel)\n ax.set_ylabel(ylabel)\n return ax", "response": "Plots a discrete CDF with the given mean and variance."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef plot_gaussian_cdf(mean=0., variance=1.,\n ax=None,\n xlim=None, ylim=(0., 1.),\n xlabel=None, ylabel=None,\n label=None):\n \"\"\"\n Plots a normal distribution CDF with the given mean and variance.\n x-axis contains the mean, the y-axis shows the cumulative probability.\n\n Parameters\n ----------\n\n mean : scalar, default 0.\n mean for the normal distribution.\n\n variance : scalar, default 0.\n variance for the normal distribution.\n\n ax : matplotlib axes object, optional\n If provided, the axes to draw on, otherwise plt.gca() is used.\n\n xlim, ylim: (float,float), optional\n specify the limits for the x or y axis as tuple (low,high).\n If not specified, limits will be automatically chosen to be 'nice'\n\n xlabel : str,optional\n label for the x-axis\n\n ylabel : str, optional\n label for the y-axis\n\n label : str, optional\n label for the legend\n\n Returns\n -------\n axis of plot\n \"\"\"\n import matplotlib.pyplot as plt\n\n if ax is None:\n ax = plt.gca()\n\n sigma = math.sqrt(variance)\n n = norm(mean, sigma)\n if xlim is None:\n xlim = [n.ppf(0.001), n.ppf(0.999)]\n\n xs = np.arange(xlim[0], xlim[1], (xlim[1] - xlim[0]) / 1000.)\n cdf = n.cdf(xs)\n ax.plot(xs, cdf, label=label)\n ax.set_xlim(xlim)\n ax.set_ylim(ylim)\n ax.set_xlabel(xlabel)\n ax.set_ylabel(ylabel)\n return ax", "response": "Plots a normal distribution CDF with the given mean and variance."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef plot_gaussian_pdf(mean=0.,\n variance=1.,\n std=None,\n ax=None,\n mean_line=False,\n xlim=None, ylim=None,\n xlabel=None, ylabel=None,\n label=None):\n \"\"\"\n Plots a normal distribution PDF with the given mean and variance.\n x-axis contains the mean, the y-axis shows the probability density.\n\n Parameters\n ----------\n\n mean : scalar, default 0.\n mean for the normal distribution.\n\n variance : scalar, default 1., optional\n variance for the normal distribution.\n\n std: scalar, default=None, optional\n standard deviation of the normal distribution. Use instead of\n `variance` if desired\n\n ax : matplotlib axes object, optional\n If provided, the axes to draw on, otherwise plt.gca() is used.\n\n mean_line : boolean\n draws a line at x=mean\n\n xlim, ylim: (float,float), optional\n specify the limits for the x or y axis as tuple (low,high).\n If not specified, limits will be automatically chosen to be 'nice'\n\n xlabel : str,optional\n label for the x-axis\n\n ylabel : str, optional\n label for the y-axis\n\n label : str, optional\n label for the legend\n\n Returns\n -------\n axis of plot\n \"\"\"\n\n import matplotlib.pyplot as plt\n\n if ax is None:\n ax = plt.gca()\n\n if variance is not None and std is not None:\n raise ValueError('Specify only one of variance and std')\n\n if variance is None and std is None:\n raise ValueError('Specify variance or std')\n\n if variance is not None:\n std = math.sqrt(variance)\n\n n = norm(mean, std)\n\n if xlim is None:\n xlim = [n.ppf(0.001), n.ppf(0.999)]\n\n xs = np.arange(xlim[0], xlim[1], (xlim[1] - xlim[0]) / 1000.)\n ax.plot(xs, n.pdf(xs), label=label)\n ax.set_xlim(xlim)\n\n if ylim is not None:\n ax.set_ylim(ylim)\n\n if mean_line:\n plt.axvline(mean)\n\n if xlabel is not None:\n ax.set_xlabel(xlabel)\n if ylabel is not None:\n ax.set_ylabel(ylabel)\n return ax", "response": "Plots a normal distribution PDF with the given mean and variance."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef plot_gaussian(mean=0., variance=1.,\n ax=None,\n mean_line=False,\n xlim=None,\n ylim=None,\n xlabel=None,\n ylabel=None,\n label=None):\n \"\"\"\n DEPRECATED. Use plot_gaussian_pdf() instead. This is poorly named, as\n there are multiple ways to plot a Gaussian.\n \"\"\"\n\n warnings.warn('This function is deprecated. It is poorly named. '\\\n 'A Gaussian can be plotted as a PDF or CDF. This '\\\n 'plots a PDF. Use plot_gaussian_pdf() instead,',\n DeprecationWarning)\n return plot_gaussian_pdf(mean, variance, ax, mean_line, xlim, ylim, xlabel,\n ylabel, label)", "response": "DEPRECATED. Use plot_gaussian_pdf() instead. This is poorly named, as\n there are multiple ways to plot a Gaussian."} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn a tuple defining the ellipse representing the 2 dimensional covariance matrix P.", "response": "def covariance_ellipse(P, deviations=1):\n \"\"\"\n Returns a tuple defining the ellipse representing the 2 dimensional\n covariance matrix P.\n\n Parameters\n ----------\n\n P : nd.array shape (2,2)\n covariance matrix\n\n deviations : int (optional, default = 1)\n # of standard deviations. Default is 1.\n\n Returns (angle_radians, width_radius, height_radius)\n \"\"\"\n\n U, s, _ = linalg.svd(P)\n orientation = math.atan2(U[1, 0], U[0, 0])\n width = deviations * math.sqrt(s[0])\n height = deviations * math.sqrt(s[1])\n\n if height > width:\n raise ValueError('width must be greater than height')\n\n return (orientation, width, height)"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning the eigenvalues and eigenvectors of a covariance matrix and returns them sorted by eigenvalue.", "response": "def _eigsorted(cov, asc=True):\n \"\"\"\n Computes eigenvalues and eigenvectors of a covariance matrix and returns\n them sorted by eigenvalue.\n\n Parameters\n ----------\n cov : ndarray\n covariance matrix\n\n asc : bool, default=True\n determines whether we are sorted smallest to largest (asc=True),\n or largest to smallest (asc=False)\n\n Returns\n -------\n eigval : 1D ndarray\n eigenvalues of covariance ordered largest to smallest\n\n eigvec : 2D ndarray\n eigenvectors of covariance matrix ordered to match `eigval` ordering.\n I.e eigvec[:, 0] is the rotation vector for eigval[0]\n \"\"\"\n\n eigval, eigvec = np.linalg.eigh(cov)\n order = eigval.argsort()\n if not asc:\n # sort largest to smallest\n order = order[::-1]\n\n return eigval[order], eigvec[:, order]"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nplotting a 3D covariance matrix.", "response": "def plot_3d_covariance(mean, cov, std=1.,\n ax=None, title=None,\n color=None, alpha=1.,\n label_xyz=True,\n N=60,\n shade=True,\n limit_xyz=True,\n **kwargs):\n \"\"\"\n Plots a covariance matrix `cov` as a 3D ellipsoid centered around\n the `mean`.\n\n Parameters\n ----------\n\n mean : 3-vector\n mean in x, y, z. Can be any type convertable to a row vector.\n\n cov : ndarray 3x3\n covariance matrix\n\n std : double, default=1\n standard deviation of ellipsoid\n\n ax : matplotlib.axes._subplots.Axes3DSubplot, optional\n Axis to draw on. If not provided, a new 3d axis will be generated\n for the current figure\n\n title : str, optional\n If provided, specifies the title for the plot\n\n color : any value convertible to a color\n if specified, color of the ellipsoid.\n\n alpha : float, default 1.\n Alpha value of the ellipsoid. <1 makes is semi-transparent.\n\n label_xyz: bool, default True\n\n Gives labels 'X', 'Y', and 'Z' to the axis.\n\n N : int, default=60\n Number of segments to compute ellipsoid in u,v space. Large numbers\n can take a very long time to plot. Default looks nice.\n\n shade : bool, default=True\n Use shading to draw the ellipse\n\n limit_xyz : bool, default=True\n Limit the axis range to fit the ellipse\n\n **kwargs : optional\n keyword arguments to supply to the call to plot_surface()\n \"\"\"\n\n from mpl_toolkits.mplot3d import Axes3D\n import matplotlib.pyplot as plt\n\n # force mean to be a 1d vector no matter its shape when passed in\n mean = np.atleast_2d(mean)\n if mean.shape[1] == 1:\n mean = mean.T\n\n if not(mean.shape[0] == 1 and mean.shape[1] == 3):\n raise ValueError('mean must be convertible to a 1x3 row vector')\n mean = mean[0]\n\n # force covariance to be 3x3 np.array\n cov = np.asarray(cov)\n if cov.shape[0] != 3 or cov.shape[1] != 3:\n raise ValueError(\"covariance must be 3x3\")\n\n # The idea is simple - find the 3 axis of the covariance matrix\n # by finding the eigenvalues and vectors. The eigenvalues are the\n # radii (squared, since covariance has squared terms), and the\n # eigenvectors give the rotation. So we make an ellipse with the\n # given radii and then rotate it to the proper orientation.\n\n eigval, eigvec = _eigsorted(cov, asc=True)\n radii = std * np.sqrt(np.real(eigval))\n\n if eigval[0] < 0:\n raise ValueError(\"covariance matrix must be positive definite\")\n\n\n # calculate cartesian coordinates for the ellipsoid surface\n u = np.linspace(0.0, 2.0 * np.pi, N)\n v = np.linspace(0.0, np.pi, N)\n x = np.outer(np.cos(u), np.sin(v)) * radii[0]\n y = np.outer(np.sin(u), np.sin(v)) * radii[1]\n z = np.outer(np.ones_like(u), np.cos(v)) * radii[2]\n\n # rotate data with eigenvector and center on mu\n a = np.kron(eigvec[:, 0], x)\n b = np.kron(eigvec[:, 1], y)\n c = np.kron(eigvec[:, 2], z)\n\n data = a + b + c\n N = data.shape[0]\n x = data[:, 0:N] + mean[0]\n y = data[:, N:N*2] + mean[1]\n z = data[:, N*2:] + mean[2]\n\n fig = plt.gcf()\n if ax is None:\n ax = fig.add_subplot(111, projection='3d')\n\n ax.plot_surface(x, y, z,\n rstride=3, cstride=3, linewidth=0.1, alpha=alpha,\n shade=shade, color=color, **kwargs)\n\n # now make it pretty!\n\n if label_xyz:\n ax.set_xlabel('X')\n ax.set_ylabel('Y')\n ax.set_zlabel('Z')\n\n if limit_xyz:\n r = radii.max()\n ax.set_xlim(-r + mean[0], r + mean[0])\n ax.set_ylim(-r + mean[1], r + mean[1])\n ax.set_zlim(-r + mean[2], r + mean[2])\n\n if title is not None:\n plt.title(title)\n\n #pylint: disable=pointless-statement\n Axes3D #kill pylint warning about unused import\n\n return ax"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nplot a covariance ellipse for a single resource.", "response": "def plot_covariance_ellipse(\n mean, cov=None, variance=1.0, std=None,\n ellipse=None, title=None, axis_equal=True, show_semiaxis=False,\n facecolor=None, edgecolor=None,\n fc='none', ec='#004080',\n alpha=1.0, xlim=None, ylim=None,\n ls='solid'):\n \"\"\"\n Deprecated function to plot a covariance ellipse. Use plot_covariance\n instead.\n\n See Also\n --------\n\n plot_covariance\n \"\"\"\n\n warnings.warn(\"deprecated, use plot_covariance instead\", DeprecationWarning)\n plot_covariance(mean=mean, cov=cov, variance=variance, std=std,\n ellipse=ellipse, title=title, axis_equal=axis_equal,\n show_semiaxis=show_semiaxis, facecolor=facecolor,\n edgecolor=edgecolor, fc=fc, ec=ec, alpha=alpha,\n xlim=xlim, ylim=ylim, ls=ls)"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _std_tuple_of(var=None, std=None, interval=None):\n\n if std is not None:\n if np.isscalar(std):\n std = (std,)\n return std\n\n\n if interval is not None:\n if np.isscalar(interval):\n interval = (interval,)\n\n return norm.interval(interval)[1]\n\n if var is None:\n raise ValueError(\"no inputs were provided\")\n\n if np.isscalar(var):\n var = (var,)\n return np.sqrt(var)", "response": "Returns a tuple of standard deviation and interval for the current language."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef plot_covariance(\n mean, cov=None, variance=1.0, std=None, interval=None,\n ellipse=None, title=None, axis_equal=True,\n show_semiaxis=False, show_center=True,\n facecolor=None, edgecolor=None,\n fc='none', ec='#004080',\n alpha=1.0, xlim=None, ylim=None,\n ls='solid'):\n \"\"\"\n Plots the covariance ellipse for the 2D normal defined by (mean, cov)\n\n `variance` is the normal sigma^2 that we want to plot. If list-like,\n ellipses for all ellipses will be ploted. E.g. [1,2] will plot the\n sigma^2 = 1 and sigma^2 = 2 ellipses. Alternatively, use std for the\n standard deviation, in which case `variance` will be ignored.\n\n ellipse is a (angle,width,height) tuple containing the angle in radians,\n and width and height radii.\n\n You may provide either cov or ellipse, but not both.\n\n Parameters\n ----------\n\n mean : row vector like (2x1)\n The mean of the normal\n\n cov : ndarray-like\n 2x2 covariance matrix\n\n variance : float, default 1, or iterable float, optional\n Variance of the plotted ellipse. May specify std or interval instead.\n If iterable, such as (1, 2**2, 3**2), then ellipses will be drawn\n for all in the list.\n\n\n std : float, or iterable float, optional\n Standard deviation of the plotted ellipse. If specified, variance\n is ignored, and interval must be `None`.\n\n If iterable, such as (1, 2, 3), then ellipses will be drawn\n for all in the list.\n\n interval : float range [0,1), or iterable float, optional\n Confidence interval for the plotted ellipse. For example, .68 (for\n 68%) gives roughly 1 standand deviation. If specified, variance\n is ignored and `std` must be `None`\n\n If iterable, such as (.68, .95), then ellipses will be drawn\n for all in the list.\n\n\n ellipse: (float, float, float)\n Instead of a covariance, plots an ellipse described by (angle, width,\n height), where angle is in radians, and the width and height are the\n minor and major sub-axis radii. `cov` must be `None`.\n\n title: str, optional\n title for the plot\n\n axis_equal: bool, default=True\n Use the same scale for the x-axis and y-axis to ensure the aspect\n ratio is correct.\n\n show_semiaxis: bool, default=False\n Draw the semiaxis of the ellipse\n\n show_center: bool, default=True\n Mark the center of the ellipse with a cross\n\n facecolor, fc: color, default=None\n If specified, fills the ellipse with the specified color. `fc` is an\n allowed abbreviation\n\n edgecolor, ec: color, default=None\n If specified, overrides the default color sequence for the edge color\n of the ellipse. `ec` is an allowed abbreviation\n\n alpha: float range [0,1], default=1.\n alpha value for the ellipse\n\n xlim: float or (float,float), default=None\n specifies the limits for the x-axis\n\n ylim: float or (float,float), default=None\n specifies the limits for the y-axis\n\n ls: str, default='solid':\n line style for the edge of the ellipse\n \"\"\"\n\n from matplotlib.patches import Ellipse\n import matplotlib.pyplot as plt\n\n if cov is not None and ellipse is not None:\n raise ValueError('You cannot specify both cov and ellipse')\n\n if cov is None and ellipse is None:\n raise ValueError('Specify one of cov or ellipse')\n\n if facecolor is None:\n facecolor = fc\n\n if edgecolor is None:\n edgecolor = ec\n\n if cov is not None:\n ellipse = covariance_ellipse(cov)\n\n if axis_equal:\n plt.axis('equal')\n\n if title is not None:\n plt.title(title)\n\n ax = plt.gca()\n\n angle = np.degrees(ellipse[0])\n width = ellipse[1] * 2.\n height = ellipse[2] * 2.\n\n std = _std_tuple_of(variance, std, interval)\n for sd in std:\n e = Ellipse(xy=mean, width=sd*width, height=sd*height, angle=angle,\n facecolor=facecolor,\n edgecolor=edgecolor,\n alpha=alpha,\n lw=2, ls=ls)\n ax.add_patch(e)\n x, y = mean\n if show_center:\n plt.scatter(x, y, marker='+', color=edgecolor)\n\n if xlim is not None:\n ax.set_xlim(xlim)\n\n if ylim is not None:\n ax.set_ylim(ylim)\n\n if show_semiaxis:\n a = ellipse[0]\n h, w = height/4, width/4\n plt.plot([x, x+ h*cos(a+np.pi/2)], [y, y + h*sin(a+np.pi/2)])\n plt.plot([x, x+ w*cos(a)], [y, y + w*sin(a)])", "response": "Plots the covariance matrix of the current key - entry pair."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef norm_cdf(x_range, mu, var=1, std=None):\n\n if std is None:\n std = math.sqrt(var)\n return abs(norm.cdf(x_range[0], loc=mu, scale=std) -\n norm.cdf(x_range[1], loc=mu, scale=std))", "response": "Computes the probability that a Gaussian distribution lieson within a range of values."} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nconverts x to a covariance matrix.", "response": "def _to_cov(x, n):\n \"\"\"\n If x is a scalar, returns a covariance matrix generated from it\n as the identity matrix multiplied by x. The dimension will be nxn.\n If x is already a 2D numpy array then it is returned unchanged.\n\n Raises ValueError if not positive definite\n \"\"\"\n\n if np.isscalar(x):\n if x < 0:\n raise ValueError('covariance must be > 0')\n return np.eye(n) * x\n\n x = np.atleast_2d(x)\n try:\n # quickly find out if we are positive definite\n np.linalg.cholesky(x)\n except:\n raise ValueError('covariance must be positive definit')\n\n return x"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn a random number distributed by student s t distribution with the specified mean and standard deviation.", "response": "def rand_student_t(df, mu=0, std=1):\n \"\"\"\n return random number distributed by student's t distribution with\n `df` degrees of freedom with the specified mean and standard deviation.\n \"\"\"\n\n x = random.gauss(0, std)\n y = 2.0*random.gammavariate(0.5 * df, 2.0)\n return x / (math.sqrt(y / df)) + mu"} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncompute the normalized estimated error squared test on a sequence of true values for the state x.", "response": "def NESS(xs, est_xs, ps):\n \"\"\"\n Computes the normalized estimated error squared test on a sequence\n of estimates. The estimates are optimal if the mean error is zero and\n the covariance matches the Kalman filter's covariance. If this holds,\n then the mean of the NESS should be equal to or less than the dimension\n of x.\n\n Examples\n --------\n\n .. code-block: Python\n\n xs = ground_truth()\n est_xs, ps, _, _ = kf.batch_filter(zs)\n NESS(xs, est_xs, ps)\n\n Parameters\n ----------\n\n xs : list-like\n sequence of true values for the state x\n\n est_xs : list-like\n sequence of estimates from an estimator (such as Kalman filter)\n\n ps : list-like\n sequence of covariance matrices from the estimator\n\n Returns\n -------\n\n ness : list of floats\n list of NESS computed for each estimate\n\n \"\"\"\n\n est_err = xs - est_xs\n ness = []\n for x, p in zip(est_err, ps):\n ness.append(np.dot(x.T, linalg.inv(p)).dot(x))\n return ness"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef ckf_transform(Xs, Q):\n\n m, n = Xs.shape\n\n x = sum(Xs, 0)[:, None] / m\n P = np.zeros((n, n))\n xf = x.flatten()\n for k in range(m):\n P += np.outer(Xs[k], Xs[k]) - np.outer(xf, xf)\n\n P *= 1 / m\n P += Q\n\n return x, P", "response": "Compute mean and covariance of array of cubature points Xs and Q."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef update(self, z, R=None, hx_args=()):\n\n if z is None:\n self.z = np.array([[None]*self.dim_z]).T\n self.x_post = self.x.copy()\n self.P_post = self.P.copy()\n return\n\n if not isinstance(hx_args, tuple):\n hx_args = (hx_args,)\n\n if R is None:\n R = self.R\n elif isscalar(R):\n R = eye(self.dim_z) * R\n\n for k in range(self._num_sigmas):\n self.sigmas_h[k] = self.hx(self.sigmas_f[k], *hx_args)\n\n # mean and covariance of prediction passed through unscented transform\n zp, self.S = ckf_transform(self.sigmas_h, R)\n self.SI = inv(self.S)\n\n # compute cross variance of the state and the measurements\n m = self._num_sigmas # literaure uses m for scaling factor\n xf = self.x.flatten()\n zpf = zp.flatten()\n Pxz = outer_product_sum(self.sigmas_f - xf, self.sigmas_h - zpf) / m\n\n self.K = dot(Pxz, self.SI) # Kalman gain\n self.y = self.residual_z(z, zp) # residual\n\n self.x = self.x + dot(self.K, self.y)\n self.P = self.P - dot(self.K, self.S).dot(self.K.T)\n\n # save measurement and posterior state\n self.z = deepcopy(z)\n self.x_post = self.x.copy()\n self.P_post = self.P.copy()\n\n # set to None to force recompute\n self._log_likelihood = None\n self._likelihood = None\n self._mahalanobis = None", "response": "Update the CKF with the given measurements. On return self. x and self. P contain the new mean and covariance of the filter. On return self. x and self. P contain the new mean and covariance of the filter. On return self. x and self. P contain the new covariance of the filter. On return self. x and self. P contain the new residual z."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef update(self, z, R_inv=None):\n\n if z is None:\n self.z = None\n self.x_post = self.x.copy()\n self.P_inv_post = self.P_inv.copy()\n return\n\n if R_inv is None:\n R_inv = self.R_inv\n elif np.isscalar(R_inv):\n R_inv = eye(self.dim_z) * R_inv\n\n # rename for readability and a tiny extra bit of speed\n H = self.H\n H_T = H.T\n P_inv = self.P_inv\n x = self.x\n\n if self._no_information:\n self.x = dot(P_inv, x) + dot(H_T, R_inv).dot(z)\n self.P_inv = P_inv + dot(H_T, R_inv).dot(H)\n self.log_likelihood = math.log(sys.float_info.min)\n self.likelihood = sys.float_info.min\n\n else:\n # y = z - Hx\n # error (residual) between measurement and prediction\n self.y = z - dot(H, x)\n\n # S = HPH' + R\n # project system uncertainty into measurement space\n self.S = P_inv + dot(H_T, R_inv).dot(H)\n self.K = dot(self.inv(self.S), H_T).dot(R_inv)\n\n # x = x + Ky\n # predict new x with residual scaled by the kalman gain\n self.x = x + dot(self.K, self.y)\n self.P_inv = P_inv + dot(H_T, R_inv).dot(H)\n\n self.z = np.copy(reshape_z(z, self.dim_z, np.ndim(self.x)))\n\n if self.compute_log_likelihood:\n self.log_likelihood = logpdf(x=self.y, cov=self.S)\n self.likelihood = math.exp(self.log_likelihood)\n if self.likelihood == 0:\n self.likelihood = sys.float_info.min\n\n # save measurement and posterior state\n self.z = deepcopy(z)\n self.x_post = self.x.copy()\n self.P_inv_post = self.P_inv.copy()", "response": "Update the internal state of the object with new values."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef predict(self, u=0):\n\n # x = Fx + Bu\n\n A = dot(self._F_inv.T, self.P_inv).dot(self._F_inv)\n #pylint: disable=bare-except\n try:\n AI = self.inv(A)\n invertable = True\n if self._no_information:\n try:\n self.x = dot(self.inv(self.P_inv), self.x)\n except:\n self.x = dot(0, self.x)\n self._no_information = False\n except:\n invertable = False\n self._no_information = True\n\n if invertable:\n self.x = dot(self._F, self.x) + dot(self.B, u)\n self.P_inv = self.inv(AI + self.Q)\n\n # save priors\n self.P_inv_prior = np.copy(self.P_inv)\n self.x_prior = np.copy(self.x)\n else:\n I_PF = self._I - dot(self.P_inv, self._F_inv)\n FTI = self.inv(self._F.T)\n FTIX = dot(FTI, self.x)\n AQI = self.inv(A + self.Q)\n self.x = dot(FTI, dot(I_PF, AQI).dot(FTIX))\n\n # save priors\n self.x_prior = np.copy(self.x)\n self.P_inv_prior = np.copy(AQI)", "response": "Predict next position.\n\n Parameters\n ----------\n\n u : ndarray\n Optional control vector. If non-zero, it is multiplied by B\n to create the control input into the system."} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nbatching processes a list of measurements at each time step and returns a list of information for each entry in zs and Rs.", "response": "def batch_filter(self, zs, Rs=None, update_first=False, saver=None):\n \"\"\" Batch processes a sequences of measurements.\n\n Parameters\n ----------\n\n zs : list-like\n list of measurements at each time step `self.dt` Missing\n measurements must be represented by 'None'.\n\n Rs : list-like, optional\n optional list of values to use for the measurement error\n covariance; a value of None in any position will cause the filter\n to use `self.R` for that time step.\n\n update_first : bool, optional,\n controls whether the order of operations is update followed by\n predict, or predict followed by update. Default is predict->update.\n\n saver : filterpy.common.Saver, optional\n filterpy.common.Saver object. If provided, saver.save() will be\n called after every epoch\n\n Returns\n -------\n\n means: np.array((n,dim_x,1))\n array of the state for each time step. Each entry is an np.array.\n In other words `means[k,:]` is the state at step `k`.\n\n covariance: np.array((n,dim_x,dim_x))\n array of the covariances for each time step. In other words\n `covariance[k,:,:]` is the covariance at step `k`.\n \"\"\"\n\n raise NotImplementedError(\"this is not implemented yet\")\n\n #pylint: disable=unreachable, no-member\n\n # this is a copy of the code from kalman_filter, it has not been\n # turned into the information filter yet. DO NOT USE.\n\n n = np.size(zs, 0)\n if Rs is None:\n Rs = [None] * n\n\n # mean estimates from Kalman Filter\n means = zeros((n, self.dim_x, 1))\n\n # state covariances from Kalman Filter\n covariances = zeros((n, self.dim_x, self.dim_x))\n\n if update_first:\n for i, (z, r) in enumerate(zip(zs, Rs)):\n self.update(z, r)\n means[i, :] = self.x\n covariances[i, :, :] = self._P\n self.predict()\n\n if saver is not None:\n saver.save()\n else:\n for i, (z, r) in enumerate(zip(zs, Rs)):\n self.predict()\n self.update(z, r)\n\n means[i, :] = self.x\n covariances[i, :, :] = self._P\n\n if saver is not None:\n saver.save()\n\n return (means, covariances)"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef F(self, value):\n self._F = value\n self._F_inv = self.inv(self._F)", "response": "set the state transition matrix"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef runge_kutta4(y, x, dx, f):\n\n k1 = dx * f(y, x)\n k2 = dx * f(y + 0.5*k1, x + 0.5*dx)\n k3 = dx * f(y + 0.5*k2, x + 0.5*dx)\n k4 = dx * f(y + k3, x + dx)\n\n return y + (k1 + 2*k2 + 2*k3 + k4) / 6.", "response": "computes 4th order Runge - Kutta for dy and dx"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ngenerate a pretty printed NumPy array with an assignment. Optionally transposes column vectors so they are drawn on one line. Strictly speaking arr can be any time convertible by `str(arr)`, but the output may not be what you want if the type of the variable is not a scalar or an ndarray. Examples -------- >>> pprint('cov', np.array([[4., .1], [.1, 5]])) cov = [[4. 0.1] [0.1 5. ]] >>> print(pretty_str('x', np.array([[1], [2], [3]]))) x = [[1 2 3]].T", "response": "def pretty_str(label, arr):\n \"\"\"\n Generates a pretty printed NumPy array with an assignment. Optionally\n transposes column vectors so they are drawn on one line. Strictly speaking\n arr can be any time convertible by `str(arr)`, but the output may not\n be what you want if the type of the variable is not a scalar or an\n ndarray.\n\n Examples\n --------\n >>> pprint('cov', np.array([[4., .1], [.1, 5]]))\n cov = [[4. 0.1]\n [0.1 5. ]]\n\n >>> print(pretty_str('x', np.array([[1], [2], [3]])))\n x = [[1 2 3]].T\n \"\"\"\n\n def is_col(a):\n \"\"\" return true if a is a column vector\"\"\"\n try:\n return a.shape[0] > 1 and a.shape[1] == 1\n except (AttributeError, IndexError):\n return False\n\n if label is None:\n label = ''\n\n if label:\n label += ' = '\n\n if is_col(arr):\n return label + str(arr.T).replace('\\n', '') + '.T'\n\n rows = str(arr).split('\\n')\n if not rows:\n return ''\n\n s = label + rows[0]\n pad = ' ' * len(label)\n for line in rows[1:]:\n s = s + '\\n' + pad + line\n\n return s"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef reshape_z(z, dim_z, ndim):\n\n z = np.atleast_2d(z)\n if z.shape[1] == dim_z:\n z = z.T\n\n if z.shape != (dim_z, 1):\n raise ValueError('z must be convertible to shape ({}, 1)'.format(dim_z))\n\n if ndim == 1:\n z = z[:, 0]\n\n if ndim == 0:\n z = z[0, 0]\n\n return z", "response": "ensure z is a ( dim_z 1 ) shaped vector"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncompute the inverse of a diagonal NxN array S.", "response": "def inv_diagonal(S):\n \"\"\"\n Computes the inverse of a diagonal NxN np.array S. In general this will\n be much faster than calling np.linalg.inv().\n\n However, does NOT check if the off diagonal elements are non-zero. So long\n as S is truly diagonal, the output is identical to np.linalg.inv().\n\n Parameters\n ----------\n S : np.array\n diagonal NxN array to take inverse of\n\n Returns\n -------\n S_inv : np.array\n inverse of S\n\n\n Examples\n --------\n\n This is meant to be used as a replacement inverse function for\n the KalmanFilter class when you know the system covariance S is\n diagonal. It just makes the filter run faster, there is\n\n >>> kf = KalmanFilter(dim_x=3, dim_z=1)\n >>> kf.inv = inv_diagonal # S is 1x1, so safely diagonal\n \"\"\"\n\n S = np.asarray(S)\n\n if S.ndim != 2 or S.shape[0] != S.shape[1]:\n raise ValueError('S must be a square Matrix')\n\n si = np.zeros(S.shape)\n for i in range(len(S)):\n si[i, i] = 1. / S[i, i]\n return si"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncomputing the sum of the outer products of the rows in A and B.", "response": "def outer_product_sum(A, B=None):\n \"\"\"\n Computes the sum of the outer products of the rows in A and B\n\n P = \\Sum {A[i] B[i].T} for i in 0..N\n\n Notionally:\n\n P = 0\n for y in A:\n P += np.outer(y, y)\n\n This is a standard computation for sigma points used in the UKF, ensemble\n Kalman filter, etc., where A would be the residual of the sigma points\n and the filter's state or measurement.\n\n The computation is vectorized, so it is much faster than the for loop\n for large A.\n\n Parameters\n ----------\n A : np.array, shape (M, N)\n rows of N-vectors to have the outer product summed\n\n B : np.array, shape (M, N)\n rows of N-vectors to have the outer product summed\n If it is `None`, it is set to A.\n\n Returns\n -------\n P : np.array, shape(N, N)\n sum of the outer product of the rows of A and B\n\n Examples\n --------\n\n Here sigmas is of shape (M, N), and x is of shape (N). The two sets of\n code compute the same thing.\n\n >>> P = outer_product_sum(sigmas - x)\n >>>\n >>> P = 0\n >>> for s in sigmas:\n >>> y = s - x\n >>> P += np.outer(y, y)\n \"\"\"\n\n if B is None:\n B = A\n\n outer = np.einsum('ij,ik->ijk', A, B)\n return np.sum(outer, axis=0)"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef save(self):\n\n kf = self._kf\n\n # force all attributes to be computed. this is only necessary\n # if the class uses properties that compute data only when\n # accessed\n for prop in self.properties:\n self._DL[prop[0]].append(getattr(kf, prop[0]))\n\n v = copy.deepcopy(kf.__dict__)\n\n if self._skip_private:\n for key in list(v.keys()):\n if key.startswith('_'):\n print('deleting', key)\n del v[key]\n\n if self._skip_callable:\n for key in list(v.keys()):\n if callable(v[key]):\n del v[key]\n\n for ig in self._ignore:\n if ig in v:\n del v[ig]\n\n for key in list(v.keys()):\n self._DL[key].append(v[key])\n\n self.__dict__.update(self._DL)\n self._len += 1", "response": "save the current state of the Kalman filter"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef to_array(self):\n for key in self.keys:\n try:\n self.__dict__[key] = np.array(self._DL[key])\n except:\n # get back to lists so we are in a valid state\n self.__dict__.update(self._DL)\n\n raise ValueError(\n \"could not convert {} into np.array\".format(key))", "response": "Convert all saved attributes from a list to np. array."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nflatten any np. array of column vectors into 1D arrays.", "response": "def flatten(self):\n \"\"\"\n Flattens any np.array of column vectors into 1D arrays. Basically,\n this makes data readable for humans if you are just inspecting via\n the REPL. For example, if you have saved a KalmanFilter object with 89\n epochs, self.x will be shape (89, 9, 1) (for example). After flatten\n is run, self.x.shape == (89, 9), which displays nicely from the REPL.\n\n There is no way to unflatten, so it's a one way trip.\n \"\"\"\n\n for key in self.keys:\n try:\n arr = self.__dict__[key]\n shape = arr.shape\n if shape[2] == 1:\n self.__dict__[key] = arr.reshape(shape[0], shape[1])\n except:\n # not an ndarray or not a column vector\n pass"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nupdate the state of the current filter with new values.", "response": "def update(self, z):\n \"\"\"\n Add a new measurement (z) to the Kalman filter. If z is None, nothing\n is changed.\n\n Parameters\n ----------\n\n z : np.array\n measurement for this update.\n \"\"\"\n\n # run update on each filter, and save the likelihood\n for i, f in enumerate(self.filters):\n f.update(z)\n self.likelihood[i] = f.likelihood\n\n # update mode probabilities from total probability * likelihood\n self.mu = self.cbar * self.likelihood\n self.mu /= np.sum(self.mu) # normalize\n\n self._compute_mixing_probabilities()\n\n # compute mixed IMM state and covariance and save posterior estimate\n self._compute_state_estimate()\n self.x_post = self.x.copy()\n self.P_post = self.P.copy()"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef predict(self, u=None):\n\n # compute mixed initial conditions\n xs, Ps = [], []\n for i, (f, w) in enumerate(zip(self.filters, self.omega.T)):\n x = zeros(self.x.shape)\n for kf, wj in zip(self.filters, w):\n x += kf.x * wj\n xs.append(x)\n\n P = zeros(self.P.shape)\n for kf, wj in zip(self.filters, w):\n y = kf.x - x\n P += wj * (outer(y, y) + kf.P)\n Ps.append(P)\n\n # compute each filter's prior using the mixed initial conditions\n for i, f in enumerate(self.filters):\n # propagate using the mixed state estimate and covariance\n f.x = xs[i].copy()\n f.P = Ps[i].copy()\n f.predict(u)\n\n # compute mixed IMM state and covariance and save posterior estimate\n self._compute_state_estimate()\n self.x_prior = self.x.copy()\n self.P_prior = self.P.copy()", "response": "Predict next state of the system using the IMM state propagation and covariance."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _compute_state_estimate(self):\n self.x.fill(0)\n for f, mu in zip(self.filters, self.mu):\n self.x += f.x * mu\n\n self.P.fill(0)\n for f, mu in zip(self.filters, self.mu):\n y = f.x - self.x\n self.P += mu * (outer(y, y) + f.P)", "response": "Computes the mixed state estimate from each filter using the mode probability self. mu to weight the estimates."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncomputes the mixing probability for each filter.", "response": "def _compute_mixing_probabilities(self):\n \"\"\"\n Compute the mixing probability for each filter.\n \"\"\"\n\n self.cbar = dot(self.mu, self.M)\n for i in range(self.N):\n for j in range(self.N):\n self.omega[i, j] = (self.M[i, j]*self.mu[i]) / self.cbar[j]"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef update(self, z, R=None, UT=None, hx=None, **hx_args):\n\n if z is None:\n self.z = np.array([[None]*self._dim_z]).T\n self.x_post = self.x.copy()\n self.P_post = self.P.copy()\n return\n\n if hx is None:\n hx = self.hx\n\n if UT is None:\n UT = unscented_transform\n\n if R is None:\n R = self.R\n elif isscalar(R):\n R = eye(self._dim_z) * R\n\n # pass prior sigmas through h(x) to get measurement sigmas\n # the shape of sigmas_h will vary if the shape of z varies, so\n # recreate each time\n sigmas_h = []\n for s in self.sigmas_f:\n sigmas_h.append(hx(s, **hx_args))\n\n self.sigmas_h = np.atleast_2d(sigmas_h)\n\n # mean and covariance of prediction passed through unscented transform\n zp, self.S = UT(self.sigmas_h, self.Wm, self.Wc, R, self.z_mean, self.residual_z)\n self.SI = self.inv(self.S)\n\n # compute cross variance of the state and the measurements\n Pxz = self.cross_variance(self.x, zp, self.sigmas_f, self.sigmas_h)\n\n\n self.K = dot(Pxz, self.SI) # Kalman gain\n self.y = self.residual_z(z, zp) # residual\n\n # update Gaussian state estimate (x, P)\n self.x = self.x + dot(self.K, self.y)\n self.P = self.P - dot(self.K, dot(self.S, self.K.T))\n\n # save measurement and posterior state\n self.z = deepcopy(z)\n self.x_post = self.x.copy()\n self.P_post = self.P.copy()\n\n # set to None to force recompute\n self._log_likelihood = None\n self._likelihood = None\n self._mahalanobis = None", "response": "Update the UKF with the given measurements. On return self. x and self. P contain the new mean and covariance of the filter."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef cross_variance(self, x, z, sigmas_f, sigmas_h):\n\n Pxz = zeros((sigmas_f.shape[1], sigmas_h.shape[1]))\n N = sigmas_f.shape[0]\n for i in range(N):\n dx = self.residual_x(sigmas_f[i], x)\n dz = self.residual_z(sigmas_h[i], z)\n Pxz += self.Wc[i] * outer(dx, dz)\n return Pxz", "response": "Compute the cross variance of the state x and measurement z."} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncomputes the values of sigmas_f for each process in the cluster", "response": "def compute_process_sigmas(self, dt, fx=None, **fx_args):\n \"\"\"\n computes the values of sigmas_f. Normally a user would not call\n this, but it is useful if you need to call update more than once\n between calls to predict (to update for multiple simultaneous\n measurements), so the sigmas correctly reflect the updated state\n x, P.\n \"\"\"\n\n if fx is None:\n fx = self.fx\n\n # calculate sigma points for given mean and covariance\n sigmas = self.points_fn.sigma_points(self.x, self.P)\n\n for i, s in enumerate(sigmas):\n self.sigmas_f[i] = fx(s, dt, **fx_args)"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef batch_filter(self, zs, Rs=None, dts=None, UT=None, saver=None):\n #pylint: disable=too-many-arguments\n\n try:\n z = zs[0]\n except TypeError:\n raise TypeError('zs must be list-like')\n\n if self._dim_z == 1:\n if not(isscalar(z) or (z.ndim == 1 and len(z) == 1)):\n raise TypeError('zs must be a list of scalars or 1D, 1 element arrays')\n else:\n if len(z) != self._dim_z:\n raise TypeError(\n 'each element in zs must be a 1D array of length {}'.format(self._dim_z))\n\n z_n = np.size(zs, 0)\n if Rs is None:\n Rs = [self.R] * z_n\n\n if dts is None:\n dts = [self._dt] * z_n\n\n # mean estimates from Kalman Filter\n if self.x.ndim == 1:\n means = zeros((z_n, self._dim_x))\n else:\n means = zeros((z_n, self._dim_x, 1))\n\n # state covariances from Kalman Filter\n covariances = zeros((z_n, self._dim_x, self._dim_x))\n\n for i, (z, r, dt) in enumerate(zip(zs, Rs, dts)):\n self.predict(dt=dt, UT=UT)\n self.update(z, r, UT=UT)\n means[i, :] = self.x\n covariances[i, :, :] = self.P\n\n if saver is not None:\n saver.save()\n\n return (means, covariances)", "response": "This method performs the batch filter over the list of measurements at each time step."} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nruns the Rauch - Tung - Striebal Kalman smoother on a set of state variables Xs Ps Qs dts and UT.", "response": "def rts_smoother(self, Xs, Ps, Qs=None, dts=None, UT=None):\n \"\"\"\n Runs the Rauch-Tung-Striebal Kalman smoother on a set of\n means and covariances computed by the UKF. The usual input\n would come from the output of `batch_filter()`.\n\n Parameters\n ----------\n\n Xs : numpy.array\n array of the means (state variable x) of the output of a Kalman\n filter.\n\n Ps : numpy.array\n array of the covariances of the output of a kalman filter.\n\n Qs: list-like collection of numpy.array, optional\n Process noise of the Kalman filter at each time step. Optional,\n if not provided the filter's self.Q will be used\n\n dt : optional, float or array-like of float\n If provided, specifies the time step of each step of the filter.\n If float, then the same time step is used for all steps. If\n an array, then each element k contains the time at step k.\n Units are seconds.\n\n UT : function(sigmas, Wm, Wc, noise_cov), optional\n Optional function to compute the unscented transform for the sigma\n points passed through hx. Typically the default function will\n work - you can use x_mean_fn and z_mean_fn to alter the behavior\n of the unscented transform.\n\n Returns\n -------\n\n x : numpy.ndarray\n smoothed means\n\n P : numpy.ndarray\n smoothed state covariances\n\n K : numpy.ndarray\n smoother gain at each step\n\n Examples\n --------\n\n .. code-block:: Python\n\n zs = [t + random.randn()*4 for t in range (40)]\n\n (mu, cov, _, _) = kalman.batch_filter(zs)\n (x, P, K) = rts_smoother(mu, cov, fk.F, fk.Q)\n \"\"\"\n #pylint: disable=too-many-locals, too-many-arguments\n\n if len(Xs) != len(Ps):\n raise ValueError('Xs and Ps must have the same length')\n\n n, dim_x = Xs.shape\n\n if dts is None:\n dts = [self._dt] * n\n elif isscalar(dts):\n dts = [dts] * n\n\n if Qs is None:\n Qs = [self.Q] * n\n\n if UT is None:\n UT = unscented_transform\n\n # smoother gain\n Ks = zeros((n, dim_x, dim_x))\n\n num_sigmas = self._num_sigmas\n\n xs, ps = Xs.copy(), Ps.copy()\n sigmas_f = zeros((num_sigmas, dim_x))\n\n for k in reversed(range(n-1)):\n # create sigma points from state estimate, pass through state func\n sigmas = self.points_fn.sigma_points(xs[k], ps[k])\n for i in range(num_sigmas):\n sigmas_f[i] = self.fx(sigmas[i], dts[k])\n\n xb, Pb = UT(\n sigmas_f, self.Wm, self.Wc, self.Q,\n self.x_mean, self.residual_x)\n\n # compute cross variance\n Pxb = 0\n for i in range(num_sigmas):\n y = self.residual_x(sigmas_f[i], xb)\n z = self.residual_x(sigmas[i], Xs[k])\n Pxb += self.Wc[i] * outer(z, y)\n\n # compute gain\n K = dot(Pxb, self.inv(Pb))\n\n # update the smoothed estimates\n xs[k] += dot(K, self.residual_x(xs[k+1], xb))\n ps[k] += dot(K, ps[k+1] - Pb).dot(K.T)\n Ks[k] = K\n\n return (xs, ps, Ks)"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get_range(self, process_err_pct=0.05):\n\n vel = self.vel + 5 * randn()\n alt = self.alt + 10 * randn()\n\n self.pos += vel*self.dt\n\n err = (self.pos * process_err_pct) * randn()\n slant_range = (self.pos**2 + alt**2)**.5 + err\n\n return slant_range", "response": "Returns the slant range to the object."} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\npredicts next position using the Kalman filter state propagation equations for each filter in the bank.", "response": "def predict(self, u=0):\n \"\"\"\n Predict next position using the Kalman filter state propagation\n equations for each filter in the bank.\n\n Parameters\n ----------\n\n u : np.array\n Optional control vector. If non-zero, it is multiplied by B\n to create the control input into the system.\n \"\"\"\n\n for f in self.filters:\n f.predict(u)\n\n # save prior\n self.x_prior = self.x.copy()\n self.P_prior = self.P.copy()"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nupdates the state of the Kalman filter with new measurement z.", "response": "def update(self, z, R=None, H=None):\n \"\"\"\n Add a new measurement (z) to the Kalman filter. If z is None, nothing\n is changed.\n\n Parameters\n ----------\n\n z : np.array\n measurement for this update.\n\n R : np.array, scalar, or None\n Optionally provide R to override the measurement noise for this\n one call, otherwise self.R will be used.\n\n H : np.array, or None\n Optionally provide H to override the measurement function for this\n one call, otherwise self.H will be used.\n \"\"\"\n\n if H is None:\n H = self.H\n\n # new probability is recursively defined as prior * likelihood\n for i, f in enumerate(self.filters):\n f.update(z, R, H)\n self.p[i] *= f.likelihood\n\n self.p /= sum(self.p) # normalize\n\n # compute estimated state and covariance of the bank of filters.\n self.P = np.zeros(self.filters[0].P.shape)\n\n # state can be in form [x,y,z,...] or [[x, y, z,...]].T\n is_row_vector = (self.filters[0].x.ndim == 1)\n if is_row_vector:\n self.x = np.zeros(self.dim_x)\n for f, p in zip(self.filters, self.p):\n self.x += np.dot(f.x, p)\n else:\n self.x = np.zeros((self.dim_x, 1))\n for f, p in zip(self.filters, self.p):\n self.x = np.zeros((self.dim_x, 1))\n self.x += np.dot(f.x, p)\n\n for x, f, p in zip(self.x, self.filters, self.p):\n y = f.x - x\n self.P += p*(np.outer(y, y) + f.P)\n\n\n # save measurement and posterior state\n self.z = deepcopy(z)\n self.x_post = self.x.copy()\n self.P_post = self.P.copy()"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nperforms the residual resampling algorithm used by particle filters. Based on observation that we don't need to use random numbers to select most of the weights. Take int(N*w^i) samples of each particle i, and then resample any remaining using a standard resampling algorithm [1] Parameters ---------- weights : list-like of float list of weights as floats Returns ------- indexes : ndarray of ints array of indexes into the weights defining the resample. i.e. the index of the zeroth resample is indexes[0], etc. References ---------- .. [1] J. S. Liu and R. Chen. Sequential Monte Carlo methods for dynamic systems. Journal of the American Statistical Association, 93(443):1032\u20131044, 1998.", "response": "def residual_resample(weights):\n \"\"\" Performs the residual resampling algorithm used by particle filters.\n\n Based on observation that we don't need to use random numbers to select\n most of the weights. Take int(N*w^i) samples of each particle i, and then\n resample any remaining using a standard resampling algorithm [1]\n\n\n Parameters\n ----------\n\n weights : list-like of float\n list of weights as floats\n\n Returns\n -------\n\n indexes : ndarray of ints\n array of indexes into the weights defining the resample. i.e. the\n index of the zeroth resample is indexes[0], etc.\n\n References\n ----------\n\n .. [1] J. S. Liu and R. Chen. Sequential Monte Carlo methods for dynamic\n systems. Journal of the American Statistical Association,\n 93(443):1032\u20131044, 1998.\n \"\"\"\n\n N = len(weights)\n indexes = np.zeros(N, 'i')\n\n # take int(N*w) copies of each weight, which ensures particles with the\n # same weight are drawn uniformly\n num_copies = (np.floor(N*np.asarray(weights))).astype(int)\n k = 0\n for i in range(N):\n for _ in range(num_copies[i]): # make n copies\n indexes[k] = i\n k += 1\n\n # use multinormal resample on the residual to fill up the rest. This\n # maximizes the variance of the samples\n residual = weights - num_copies # get fractional part\n residual /= sum(residual) # normalize\n cumulative_sum = np.cumsum(residual)\n cumulative_sum[-1] = 1. # avoid round-off errors: ensures sum is exactly one\n indexes[k:N] = np.searchsorted(cumulative_sum, random(N-k))\n\n return indexes"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef stratified_resample(weights):\n\n N = len(weights)\n # make N subdivisions, and chose a random position within each one\n positions = (random(N) + range(N)) / N\n\n indexes = np.zeros(N, 'i')\n cumulative_sum = np.cumsum(weights)\n i, j = 0, 0\n while i < N:\n if positions[i] < cumulative_sum[j]:\n indexes[i] = j\n i += 1\n else:\n j += 1\n return indexes", "response": "This function performs a stratified resampling algorithm used by particle filters."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef systematic_resample(weights):\n N = len(weights)\n\n # make N subdivisions, and choose positions with a consistent random offset\n positions = (random() + np.arange(N)) / N\n\n indexes = np.zeros(N, 'i')\n cumulative_sum = np.cumsum(weights)\n i, j = 0, 0\n while i < N:\n if positions[i] < cumulative_sum[j]:\n indexes[i] = j\n i += 1\n else:\n j += 1\n return indexes", "response": "This function performs the systemic resampling algorithm used by particle filters."} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nupdating the internal state of the object with new measurement z.", "response": "def update(self, z):\n \"\"\"\n Add a new measurement `z` to the H-Infinity filter. If `z` is None,\n nothing is changed.\n\n Parameters\n ----------\n z : ndarray\n measurement for this update.\n \"\"\"\n\n if z is None:\n return\n\n # rename for readability and a tiny extra bit of speed\n I = self._I\n gamma = self.gamma\n Q = self.Q\n H = self.H\n P = self.P\n x = self.x\n V_inv = self._V_inv\n F = self.F\n W = self.W\n\n # common subexpression H.T * V^-1\n HTVI = dot(H.T, V_inv)\n\n L = linalg.inv(I - gamma * dot(Q, P) + dot(HTVI, H).dot(P))\n\n # common subexpression P*L\n PL = dot(P, L)\n\n K = dot(F, PL).dot(HTVI)\n\n self.y = z - dot(H, x)\n\n # x = x + Ky\n # predict new x with residual scaled by the H-Infinity gain\n self.x = self.x + dot(K, self.y)\n self.P = dot(F, PL).dot(F.T) + W\n\n # force P to be symmetric\n self.P = (self.P + self.P.T) / 2\n\n # pylint: disable=bare-except\n try:\n self.z = np.copy(z)\n except:\n self.z = copy.deepcopy(z)"} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\npredicts next position. Parameters ---------- u : ndarray Optional control vector. If non-zero, it is multiplied by `B` to create the control input into the system.", "response": "def predict(self, u=0):\n \"\"\"\n Predict next position.\n\n Parameters\n ----------\n u : ndarray\n Optional control vector. If non-zero, it is multiplied by `B`\n to create the control input into the system.\n \"\"\"\n\n # x = Fx + Bu\n self.x = dot(self.F, self.x) + dot(self.B, u)"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef batch_filter(self, Zs,update_first=False, saver=False):\n\n n = np.size(Zs, 0)\n\n # mean estimates from H-Infinity Filter\n means = zeros((n, self.dim_x, 1))\n\n # state covariances from H-Infinity Filter\n covariances = zeros((n, self.dim_x, self.dim_x))\n\n if update_first:\n for i, z in enumerate(Zs):\n self.update(z)\n means[i, :] = self.x\n covariances[i, :, :] = self.P\n self.predict()\n\n if saver is not None:\n saver.save()\n else:\n for i, z in enumerate(Zs):\n self.predict()\n self.update(z)\n\n means[i, :] = self.x\n covariances[i, :, :] = self.P\n\n if saver is not None:\n saver.save()\n\n return (means, covariances)", "response": "Batch processes a list of measurements at each time step and returns a 2D array of the mean estimates and covariance matrices."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_prediction(self, u=0):\n return dot(self.F, self.x) + dot(self.B, u)", "response": "Predicts the next state of the filter and returns it."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef V(self, value):\n\n if np.isscalar(value):\n self._V = np.array([[value]], dtype=float)\n else:\n self._V = value\n self._V_inv = linalg.inv(self._V)", "response": "set the measurement noise matrix"} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nupdates the internal state of the object with new measurement z.", "response": "def update(self, z, R2=None):\n \"\"\"\n Add a new measurement (z) to the kalman filter. If z is None, nothing\n is changed.\n\n Parameters\n ----------\n\n z : np.array\n measurement for this update.\n\n R2 : np.array, scalar, or None\n Sqrt of meaaurement noize. Optionally provide to override the\n measurement noise for this one call, otherwise self.R2 will\n be used.\n \"\"\"\n\n if z is None:\n self.z = np.array([[None]*self.dim_z]).T\n self.x_post = self.x.copy()\n self._P1_2_post = np.copy(self._P1_2)\n return\n\n if R2 is None:\n R2 = self._R1_2\n elif np.isscalar(R2):\n R2 = eye(self.dim_z) * R2\n\n # rename for convienance\n dim_z = self.dim_z\n M = self.M\n\n M[0:dim_z, 0:dim_z] = R2.T\n M[dim_z:, 0:dim_z] = dot(self.H, self._P1_2).T\n M[dim_z:, dim_z:] = self._P1_2.T\n\n _, self.S = qr(M)\n self.K = self.S[0:dim_z, dim_z:].T\n N = self.S[0:dim_z, 0:dim_z].T\n\n # y = z - Hx\n # error (residual) between measurement and prediction\n self.y = z - dot(self.H, self.x)\n\n # x = x + Ky\n # predict new x with residual scaled by the kalman gain\n self.x += dot(self.K, pinv(N)).dot(self.y)\n self._P1_2 = self.S[dim_z:, dim_z:].T\n\n self.z = deepcopy(z)\n self.x_post = self.x.copy()\n self._P1_2_post = np.copy(self._P1_2)"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef predict(self, u=0):\n\n # x = Fx + Bu\n self.x = dot(self.F, self.x) + dot(self.B, u)\n\n # P = FPF' + Q\n _, P2 = qr(np.hstack([dot(self.F, self._P1_2), self._Q1_2]).T)\n self._P1_2 = P2[:self.dim_x, :self.dim_x].T\n\n # copy prior\n self.x_prior = np.copy(self.x)\n self._P1_2_prior = np.copy(self._P1_2)", "response": "Predict next state of the class."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef P(self, value):\n self._P = value\n self._P1_2 = cholesky(self._P, lower=True)", "response": "set the covariance matrix"} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nset the measurement uncertainty", "response": "def R(self, value):\n \"\"\" measurement uncertainty\"\"\"\n self._R = value\n self._R1_2 = cholesky(self._R, lower=True)"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\nasync def ws_handler(self, protocol, path):\n self.handshake_completed_event.set()\n await self.closed_event.wait()", "response": "This is the main handler function for the websockets implementation. It is called by the websockets module when the websocket is started."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nwrapping around the ASGI callable handling exceptions and unexpected termination states.", "response": "async def run_asgi(self):\n \"\"\"\n Wrapper around the ASGI callable, handling exceptions and unexpected\n termination states.\n \"\"\"\n try:\n result = await self.app(self.scope, self.asgi_receive, self.asgi_send)\n except BaseException as exc:\n self.closed_event.set()\n msg = \"Exception in ASGI application\\n\"\n self.logger.error(msg, exc_info=exc)\n if not self.handshake_started_event.is_set():\n self.send_500_response()\n else:\n await self.handshake_completed_event.wait()\n self.transport.close()\n else:\n self.closed_event.set()\n if not self.handshake_started_event.is_set():\n msg = \"ASGI callable returned without sending handshake.\"\n self.logger.error(msg)\n self.send_500_response()\n self.transport.close()\n elif result is not None:\n msg = \"ASGI callable should return None, but returned '%s'.\"\n self.logger.error(msg, result)\n await self.handshake_completed_event.wait()\n self.transport.close()"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ncalls by the server to commence a graceful shutdown.", "response": "def shutdown(self):\n \"\"\"\n Called by the server to commence a graceful shutdown.\n \"\"\"\n if self.cycle is None or self.cycle.response_complete:\n self.transport.close()\n else:\n self.cycle.keep_alive = False"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef shutdown(self):\n if self.cycle is None or self.cycle.response_complete:\n event = h11.ConnectionClosed()\n self.conn.send(event)\n self.transport.close()\n else:\n self.cycle.keep_alive = False", "response": "Gracefully shutdown the connection."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncall on a keep - alive connection.", "response": "def timeout_keep_alive_handler(self):\n \"\"\"\n Called on a keep-alive connection if no new data is received after a short delay.\n \"\"\"\n if not self.transport.is_closing():\n event = h11.ConnectionClosed()\n self.conn.send(event)\n self.transport.close()"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef build_environ(scope, message, body):\n environ = {\n \"REQUEST_METHOD\": scope[\"method\"],\n \"SCRIPT_NAME\": \"\",\n \"PATH_INFO\": scope[\"path\"],\n \"QUERY_STRING\": scope[\"query_string\"].decode(\"ascii\"),\n \"SERVER_PROTOCOL\": \"HTTP/%s\" % scope[\"http_version\"],\n \"wsgi.version\": (1, 0),\n \"wsgi.url_scheme\": scope.get(\"scheme\", \"http\"),\n \"wsgi.input\": io.BytesIO(body),\n \"wsgi.errors\": sys.stdout,\n \"wsgi.multithread\": True,\n \"wsgi.multiprocess\": True,\n \"wsgi.run_once\": False,\n }\n\n # Get server name and port - required in WSGI, not in ASGI\n server = scope.get(\"server\", (\"localhost\", 80))\n environ[\"SERVER_NAME\"] = server[0]\n environ[\"SERVER_PORT\"] = server[1]\n\n # Get client IP address\n if \"client\" in scope:\n environ[\"REMOTE_ADDR\"] = scope[\"client\"][0]\n\n # Go through headers and make them into environ entries\n for name, value in scope.get(\"headers\", []):\n name = name.decode(\"latin1\")\n if name == \"content-length\":\n corrected_name = \"CONTENT_LENGTH\"\n elif name == \"content-type\":\n corrected_name = \"CONTENT_TYPE\"\n else:\n corrected_name = \"HTTP_%s\" % name.upper().replace(\"-\", \"_\")\n # HTTPbis say only ASCII chars are allowed in headers, but we latin1 just in case\n value = value.decode(\"latin1\")\n if corrected_name in environ:\n value = environ[corrected_name] + \",\" + value\n environ[corrected_name] = value\n return environ", "response": "Builds a WSGI environment object from a scope and request message."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn an ASGI message with any body - type content omitted and replaced with a placeholder.", "response": "def message_with_placeholders(message):\n \"\"\"\n Return an ASGI message, with any body-type content omitted and replaced\n with a placeholder.\n \"\"\"\n new_message = message.copy()\n for attr in PLACEHOLDER_FORMAT.keys():\n if message.get(attr) is not None:\n content = message[attr]\n placeholder = PLACEHOLDER_FORMAT[attr].format(length=len(content))\n new_message[attr] = placeholder\n return new_message"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef setup_logger(log_level, log_file=None):\n level = getattr(logging, log_level.upper(), None)\n if not level:\n color_print(\"Invalid log level: %s\" % log_level, \"RED\")\n sys.exit(1)\n\n # hide traceback when log level is INFO/WARNING/ERROR/CRITICAL\n if level >= logging.INFO:\n sys.tracebacklimit = 0\n\n formatter = ColoredFormatter(\n u\"%(log_color)s%(bg_white)s%(levelname)-8s%(reset)s %(message)s\",\n datefmt=None,\n reset=True,\n log_colors=log_colors_config\n )\n\n if log_file:\n handler = logging.FileHandler(log_file, encoding=\"utf-8\")\n else:\n handler = logging.StreamHandler()\n\n handler.setFormatter(formatter)\n logger.addHandler(handler)\n logger.setLevel(level)", "response": "setup root logger with ColoredFormatter."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nlog with color by different level", "response": "def log_with_color(level):\n \"\"\" log with color by different level\n \"\"\"\n def wrapper(text):\n color = log_colors_config[level.upper()]\n getattr(logger, level.lower())(coloring(text, color))\n\n return wrapper"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _handle_skip_feature(self, test_dict):\n # TODO: move skip to initialize\n skip_reason = None\n\n if \"skip\" in test_dict:\n skip_reason = test_dict[\"skip\"]\n\n elif \"skipIf\" in test_dict:\n skip_if_condition = test_dict[\"skipIf\"]\n if self.session_context.eval_content(skip_if_condition):\n skip_reason = \"{} evaluate to True\".format(skip_if_condition)\n\n elif \"skipUnless\" in test_dict:\n skip_unless_condition = test_dict[\"skipUnless\"]\n if not self.session_context.eval_content(skip_unless_condition):\n skip_reason = \"{} evaluate to False\".format(skip_unless_condition)\n\n if skip_reason:\n raise SkipTest(skip_reason)", "response": "handle skip feature for test\n "} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncall hook actions. Args: actions (list): each action in actions list maybe in two format. format1 (dict): assignment, the value returned by hook function will be assigned to variable. {\"var\": \"${func()}\"} format2 (str): only call hook functions. ${func()} hook_type (enum): setup/teardown", "response": "def do_hook_actions(self, actions, hook_type):\n \"\"\" call hook actions.\n\n Args:\n actions (list): each action in actions list maybe in two format.\n\n format1 (dict): assignment, the value returned by hook function will be assigned to variable.\n {\"var\": \"${func()}\"}\n format2 (str): only call hook functions.\n ${func()}\n\n hook_type (enum): setup/teardown\n\n \"\"\"\n logger.log_debug(\"call {} hook actions.\".format(hook_type))\n for action in actions:\n\n if isinstance(action, dict) and len(action) == 1:\n # format 1\n # {\"var\": \"${func()}\"}\n var_name, hook_content = list(action.items())[0]\n hook_content_eval = self.session_context.eval_content(hook_content)\n logger.log_debug(\n \"assignment with hook: {} = {} => {}\".format(\n var_name, hook_content, hook_content_eval\n )\n )\n self.session_context.update_test_variables(\n var_name, hook_content_eval\n )\n else:\n # format 2\n logger.log_debug(\"call hook function: {}\".format(action))\n # TODO: check hook function if valid\n self.session_context.eval_content(action)"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nextract output variables from the list of output variables", "response": "def extract_output(self, output_variables_list):\n \"\"\" extract output variables\n \"\"\"\n variables_mapping = self.session_context.session_variables_mapping\n\n output = {}\n for variable in output_variables_list:\n if variable not in variables_mapping:\n logger.log_warning(\n \"variable '{}' can not be found in variables mapping, failed to output!\"\\\n .format(variable)\n )\n continue\n\n output[variable] = variables_mapping[variable]\n\n utils.print_info(output)\n return output"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nruns tests in test_suite", "response": "def _run_suite(self, test_suite):\n \"\"\" run tests in test_suite\n\n Args:\n test_suite: unittest.TestSuite()\n\n Returns:\n list: tests_results\n\n \"\"\"\n tests_results = []\n\n for testcase in test_suite:\n testcase_name = testcase.config.get(\"name\")\n logger.log_info(\"Start to run testcase: {}\".format(testcase_name))\n\n result = self.unittest_runner.run(testcase)\n tests_results.append((testcase, result))\n\n return tests_results"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\naggregate results of a list of tests_results into a single summary object.", "response": "def _aggregate(self, tests_results):\n \"\"\" aggregate results\n\n Args:\n tests_results (list): list of (testcase, result)\n\n \"\"\"\n summary = {\n \"success\": True,\n \"stat\": {\n \"testcases\": {\n \"total\": len(tests_results),\n \"success\": 0,\n \"fail\": 0\n },\n \"teststeps\": {}\n },\n \"time\": {},\n \"platform\": report.get_platform(),\n \"details\": []\n }\n\n for tests_result in tests_results:\n testcase, result = tests_result\n testcase_summary = report.get_summary(result)\n\n if testcase_summary[\"success\"]:\n summary[\"stat\"][\"testcases\"][\"success\"] += 1\n else:\n summary[\"stat\"][\"testcases\"][\"fail\"] += 1\n\n summary[\"success\"] &= testcase_summary[\"success\"]\n testcase_summary[\"name\"] = testcase.config.get(\"name\")\n testcase_summary[\"in_out\"] = utils.get_testcase_io(testcase)\n\n report.aggregate_stat(summary[\"stat\"][\"teststeps\"], testcase_summary[\"stat\"])\n report.aggregate_stat(summary[\"time\"], testcase_summary[\"time\"])\n\n summary[\"details\"].append(testcase_summary)\n\n return summary"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nrunning testcase at specified path", "response": "def run_path(self, path, dot_env_path=None, mapping=None):\n \"\"\" run testcase/testsuite file or folder.\n\n Args:\n path (str): testcase/testsuite file/foler path.\n dot_env_path (str): specified .env file path.\n mapping (dict): if mapping is specified, it will override variables in config block.\n\n Returns:\n instance: HttpRunner() instance\n\n \"\"\"\n # load tests\n self.exception_stage = \"load tests\"\n tests_mapping = loader.load_tests(path, dot_env_path)\n tests_mapping[\"project_mapping\"][\"test_path\"] = path\n\n if mapping:\n tests_mapping[\"project_mapping\"][\"variables\"] = mapping\n\n return self.run_tests(tests_mapping)"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nsetting variables mapping to os. environ", "response": "def set_os_environ(variables_mapping):\n \"\"\" set variables mapping to os.environ\n \"\"\"\n for variable in variables_mapping:\n os.environ[variable] = variables_mapping[variable]\n logger.log_debug(\"Set OS environment variable: {}\".format(variable))"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef unset_os_environ(variables_mapping):\n for variable in variables_mapping:\n os.environ.pop(variable)\n logger.log_debug(\"Unset OS environment variable: {}\".format(variable))", "response": "unset the variables mapping to os. environ"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nbuild url from base url and path", "response": "def build_url(base_url, path):\n \"\"\" prepend url with base_url unless it's already an absolute URL \"\"\"\n if absolute_http_url_regexp.match(path):\n return path\n elif base_url:\n return \"{}/{}\".format(base_url.rstrip(\"/\"), path.lstrip(\"/\"))\n else:\n raise ParamsError(\"base url missed!\")"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ndo an xpath - like query with json_content.", "response": "def query_json(json_content, query, delimiter='.'):\n \"\"\" Do an xpath-like query with json_content.\n\n Args:\n json_content (dict/list/string): content to be queried.\n query (str): query string.\n delimiter (str): delimiter symbol.\n\n Returns:\n str: queried result.\n\n Examples:\n >>> json_content = {\n \"ids\": [1, 2, 3, 4],\n \"person\": {\n \"name\": {\n \"first_name\": \"Leo\",\n \"last_name\": \"Lee\",\n },\n \"age\": 29,\n \"cities\": [\"Guangzhou\", \"Shenzhen\"]\n }\n }\n >>>\n >>> query_json(json_content, \"person.name.first_name\")\n >>> Leo\n >>>\n >>> query_json(json_content, \"person.name.first_name.0\")\n >>> L\n >>>\n >>> query_json(json_content, \"person.cities.0\")\n >>> Guangzhou\n\n \"\"\"\n raise_flag = False\n response_body = u\"response body: {}\\n\".format(json_content)\n try:\n for key in query.split(delimiter):\n if isinstance(json_content, (list, basestring)):\n json_content = json_content[int(key)]\n elif isinstance(json_content, dict):\n json_content = json_content[key]\n else:\n logger.log_error(\n \"invalid type value: {}({})\".format(json_content, type(json_content)))\n raise_flag = True\n except (KeyError, ValueError, IndexError):\n raise_flag = True\n\n if raise_flag:\n err_msg = u\"Failed to extract! => {}\\n\".format(query)\n err_msg += response_body\n logger.log_error(err_msg)\n raise exceptions.ExtractFailure(err_msg)\n\n return json_content"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef deep_update_dict(origin_dict, override_dict):\n if not override_dict:\n return origin_dict\n\n for key, val in override_dict.items():\n if isinstance(val, dict):\n tmp = deep_update_dict(origin_dict.get(key, {}), val)\n origin_dict[key] = tmp\n elif val is None:\n # fix #64: when headers in test is None, it should inherit from config\n continue\n else:\n origin_dict[key] = override_dict[key]\n\n return origin_dict", "response": "update origin dict with override dict recursively"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef convert_dict_to_params(src_dict):\n return \"&\".join([\n \"{}={}\".format(key, value)\n for key, value in src_dict.items()\n ])", "response": "convert dict to params string"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nconvert keys in dict to lower case", "response": "def lower_dict_keys(origin_dict):\n \"\"\" convert keys in dict to lower case\n\n Args:\n origin_dict (dict): mapping data structure\n\n Returns:\n dict: mapping with all keys lowered.\n\n Examples:\n >>> origin_dict = {\n \"Name\": \"\",\n \"Request\": \"\",\n \"URL\": \"\",\n \"METHOD\": \"\",\n \"Headers\": \"\",\n \"Data\": \"\"\n }\n >>> lower_dict_keys(origin_dict)\n {\n \"name\": \"\",\n \"request\": \"\",\n \"url\": \"\",\n \"method\": \"\",\n \"headers\": \"\",\n \"data\": \"\"\n }\n\n \"\"\"\n if not origin_dict or not isinstance(origin_dict, dict):\n return origin_dict\n\n return {\n key.lower(): value\n for key, value in origin_dict.items()\n }"} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nensures variables are in mapping format.", "response": "def ensure_mapping_format(variables):\n \"\"\" ensure variables are in mapping format.\n\n Args:\n variables (list/dict): original variables\n\n Returns:\n dict: ensured variables in dict format\n\n Examples:\n >>> variables = [\n {\"a\": 1},\n {\"b\": 2}\n ]\n >>> print(ensure_mapping_format(variables))\n {\n \"a\": 1,\n \"b\": 2\n }\n\n \"\"\"\n if isinstance(variables, list):\n variables_dict = {}\n for map_dict in variables:\n variables_dict.update(map_dict)\n\n return variables_dict\n\n elif isinstance(variables, dict):\n return variables\n\n else:\n raise exceptions.ParamsError(\"variables format error!\")"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef extend_variables(raw_variables, override_variables):\n if not raw_variables:\n override_variables_mapping = ensure_mapping_format(override_variables)\n return override_variables_mapping\n\n elif not override_variables:\n raw_variables_mapping = ensure_mapping_format(raw_variables)\n return raw_variables_mapping\n\n else:\n raw_variables_mapping = ensure_mapping_format(raw_variables)\n override_variables_mapping = ensure_mapping_format(override_variables)\n raw_variables_mapping.update(override_variables_mapping)\n return raw_variables_mapping", "response": "extend raw_variables with override_variables."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef print_info(info_mapping):\n if not info_mapping:\n return\n\n content_format = \"{:<16} : {:<}\\n\"\n content = \"\\n==================== Output ====================\\n\"\n content += content_format.format(\"Variable\", \"Value\")\n content += content_format.format(\"-\" * 16, \"-\" * 29)\n\n for key, value in info_mapping.items():\n if isinstance(value, (tuple, collections.deque)):\n continue\n elif isinstance(value, (dict, list)):\n value = json.dumps(value)\n elif value is None:\n value = \"None\"\n\n if is_py2:\n if isinstance(key, unicode):\n key = key.encode(\"utf-8\")\n if isinstance(value, unicode):\n value = value.encode(\"utf-8\")\n\n content += content_format.format(key, value)\n\n content += \"-\" * 48 + \"\\n\"\n logger.log_info(content)", "response": "print info in mapping."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef create_scaffold(project_name):\n if os.path.isdir(project_name):\n logger.log_warning(u\"Folder {} exists, please specify a new folder name.\".format(project_name))\n return\n\n logger.color_print(\"Start to create new project: {}\".format(project_name), \"GREEN\")\n logger.color_print(\"CWD: {}\\n\".format(os.getcwd()), \"BLUE\")\n\n def create_path(path, ptype):\n if ptype == \"folder\":\n os.makedirs(path)\n elif ptype == \"file\":\n open(path, 'w').close()\n\n msg = \"created {}: {}\".format(ptype, path)\n logger.color_print(msg, \"BLUE\")\n\n path_list = [\n (project_name, \"folder\"),\n (os.path.join(project_name, \"api\"), \"folder\"),\n (os.path.join(project_name, \"testcases\"), \"folder\"),\n (os.path.join(project_name, \"testsuites\"), \"folder\"),\n (os.path.join(project_name, \"reports\"), \"folder\"),\n (os.path.join(project_name, \"debugtalk.py\"), \"file\"),\n (os.path.join(project_name, \".env\"), \"file\")\n ]\n [create_path(p[0], p[1]) for p in path_list]\n\n # create .gitignore file\n ignore_file = os.path.join(project_name, \".gitignore\")\n ignore_content = \".env\\nreports/*\"\n with open(ignore_file, \"w\") as f:\n f.write(ignore_content)", "response": "create scaffold with specified project name."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ngenerates cartesian product for lists of lists", "response": "def gen_cartesian_product(*args):\n \"\"\" generate cartesian product for lists\n\n Args:\n args (list of list): lists to be generated with cartesian product\n\n Returns:\n list: cartesian product in list\n\n Examples:\n\n >>> arg1 = [{\"a\": 1}, {\"a\": 2}]\n >>> arg2 = [{\"x\": 111, \"y\": 112}, {\"x\": 121, \"y\": 122}]\n >>> args = [arg1, arg2]\n >>> gen_cartesian_product(*args)\n >>> # same as below\n >>> gen_cartesian_product(arg1, arg2)\n [\n {'a': 1, 'x': 111, 'y': 112},\n {'a': 1, 'x': 121, 'y': 122},\n {'a': 2, 'x': 111, 'y': 112},\n {'a': 2, 'x': 121, 'y': 122}\n ]\n\n \"\"\"\n if not args:\n return []\n elif len(args) == 1:\n return args[0]\n\n product_list = []\n for product_item_tuple in itertools.product(*args):\n product_item_dict = {}\n for item in product_item_tuple:\n product_item_dict.update(item)\n\n product_list.append(product_item_dict)\n\n return product_list"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nprettify JSON testcase format", "response": "def prettify_json_file(file_list):\n \"\"\" prettify JSON testcase format\n \"\"\"\n for json_file in set(file_list):\n if not json_file.endswith(\".json\"):\n logger.log_warning(\"Only JSON file format can be prettified, skip: {}\".format(json_file))\n continue\n\n logger.color_print(\"Start to prettify JSON file: {}\".format(json_file), \"GREEN\")\n\n dir_path = os.path.dirname(json_file)\n file_name, file_suffix = os.path.splitext(os.path.basename(json_file))\n outfile = os.path.join(dir_path, \"{}.pretty.json\".format(file_name))\n\n with io.open(json_file, 'r', encoding='utf-8') as stream:\n try:\n obj = json.load(stream)\n except ValueError as e:\n raise SystemExit(e)\n\n with io.open(outfile, 'w', encoding='utf-8') as out:\n json.dump(obj, out, indent=4, separators=(',', ': '))\n out.write('\\n')\n\n print(\"success: {}\".format(outfile))"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef omit_long_data(body, omit_len=512):\n if not isinstance(body, basestring):\n return body\n\n body_len = len(body)\n if body_len <= omit_len:\n return body\n\n omitted_body = body[0:omit_len]\n\n appendix_str = \" ... OMITTED {} CHARACTORS ...\".format(body_len - omit_len)\n if isinstance(body, bytes):\n appendix_str = appendix_str.encode(\"utf-8\")\n\n return omitted_body + appendix_str", "response": "omit too long str or bytes"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef dump_json_file(json_data, pwd_dir_path, dump_file_name):\n class PythonObjectEncoder(json.JSONEncoder):\n def default(self, obj):\n try:\n return super().default(self, obj)\n except TypeError:\n return str(obj)\n\n logs_dir_path = os.path.join(pwd_dir_path, \"logs\")\n if not os.path.isdir(logs_dir_path):\n os.makedirs(logs_dir_path)\n\n dump_file_path = os.path.join(logs_dir_path, dump_file_name)\n\n try:\n with io.open(dump_file_path, 'w', encoding='utf-8') as outfile:\n if is_py2:\n outfile.write(\n unicode(json.dumps(\n json_data,\n indent=4,\n separators=(',', ':'),\n ensure_ascii=False,\n cls=PythonObjectEncoder\n ))\n )\n else:\n json.dump(\n json_data,\n outfile,\n indent=4,\n separators=(',', ':'),\n ensure_ascii=False,\n cls=PythonObjectEncoder\n )\n\n msg = \"dump file: {}\".format(dump_file_path)\n logger.color_print(msg, \"BLUE\")\n\n except TypeError as ex:\n msg = \"Failed to dump json file: {}\\nReason: {}\".format(dump_file_path, ex)\n logger.color_print(msg, \"RED\")", "response": "dump json data to file\n "} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nprepare dump file info.", "response": "def _prepare_dump_info(project_mapping, tag_name):\n \"\"\" prepare dump file info.\n \"\"\"\n test_path = project_mapping.get(\"test_path\") or \"tests_mapping\"\n pwd_dir_path = project_mapping.get(\"PWD\") or os.getcwd()\n file_name, file_suffix = os.path.splitext(os.path.basename(test_path.rstrip(\"/\")))\n dump_file_name = \"{}.{}.json\".format(file_name, tag_name)\n\n return pwd_dir_path, dump_file_name"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef dump_logs(json_data, project_mapping, tag_name):\n pwd_dir_path, dump_file_name = _prepare_dump_info(project_mapping, tag_name)\n dump_json_file(json_data, pwd_dir_path, dump_file_name)", "response": "Dump the json data to the log file."} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nchecking testcase file format", "response": "def _check_format(file_path, content):\n \"\"\" check testcase format if valid\n \"\"\"\n # TODO: replace with JSON schema validation\n if not content:\n # testcase file content is empty\n err_msg = u\"Testcase file content is empty: {}\".format(file_path)\n logger.log_error(err_msg)\n raise exceptions.FileFormatError(err_msg)\n\n elif not isinstance(content, (list, dict)):\n # testcase file content does not match testcase format\n err_msg = u\"Testcase file content format invalid: {}\".format(file_path)\n logger.log_error(err_msg)\n raise exceptions.FileFormatError(err_msg)"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef load_yaml_file(yaml_file):\n with io.open(yaml_file, 'r', encoding='utf-8') as stream:\n yaml_content = yaml.load(stream)\n _check_format(yaml_file, yaml_content)\n return yaml_content", "response": "load yaml file and check file content format\n "} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef load_json_file(json_file):\n with io.open(json_file, encoding='utf-8') as data_file:\n try:\n json_content = json.load(data_file)\n except exceptions.JSONDecodeError:\n err_msg = u\"JSONDecodeError: JSON file format error: {}\".format(json_file)\n logger.log_error(err_msg)\n raise exceptions.FileFormatError(err_msg)\n\n _check_format(json_file, json_content)\n return json_content", "response": "load json file and check file content format\n "} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nloading a CSV file and checks file content format and returns list of parameters each parameter is in dict format", "response": "def load_csv_file(csv_file):\n \"\"\" load csv file and check file content format\n\n Args:\n csv_file (str): csv file path, csv file content is like below:\n\n Returns:\n list: list of parameters, each parameter is in dict format\n\n Examples:\n >>> cat csv_file\n username,password\n test1,111111\n test2,222222\n test3,333333\n\n >>> load_csv_file(csv_file)\n [\n {'username': 'test1', 'password': '111111'},\n {'username': 'test2', 'password': '222222'},\n {'username': 'test3', 'password': '333333'}\n ]\n\n \"\"\"\n if not os.path.isabs(csv_file):\n project_working_directory = tests_def_mapping[\"PWD\"] or os.getcwd()\n # make compatible with Windows/Linux\n csv_file = os.path.join(project_working_directory, *csv_file.split(\"/\"))\n\n if not os.path.isfile(csv_file):\n # file path not exist\n raise exceptions.CSVNotFound(csv_file)\n\n csv_content_list = []\n\n with io.open(csv_file, encoding='utf-8') as csvfile:\n reader = csv.DictReader(csvfile)\n for row in reader:\n csv_content_list.append(row)\n\n return csv_content_list"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef load_folder_files(folder_path, recursive=True):\n if isinstance(folder_path, (list, set)):\n files = []\n for path in set(folder_path):\n files.extend(load_folder_files(path, recursive))\n\n return files\n\n if not os.path.exists(folder_path):\n return []\n\n file_list = []\n\n for dirpath, dirnames, filenames in os.walk(folder_path):\n filenames_list = []\n\n for filename in filenames:\n if not filename.endswith(('.yml', '.yaml', '.json')):\n continue\n\n filenames_list.append(filename)\n\n for filename in filenames_list:\n file_path = os.path.join(dirpath, filename)\n file_list.append(file_path)\n\n if not recursive:\n break\n\n return file_list", "response": "Load all files in a folder and return all files endswith yml / yaml or json in list."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nloading environment variables from. env file.", "response": "def load_dot_env_file(dot_env_path):\n \"\"\" load .env file.\n\n Args:\n dot_env_path (str): .env file path\n\n Returns:\n dict: environment variables mapping\n\n {\n \"UserName\": \"debugtalk\",\n \"Password\": \"123456\",\n \"PROJECT_KEY\": \"ABCDEFGH\"\n }\n\n Raises:\n exceptions.FileFormatError: If .env file format is invalid.\n\n \"\"\"\n if not os.path.isfile(dot_env_path):\n return {}\n\n logger.log_info(\"Loading environment variables from {}\".format(dot_env_path))\n env_variables_mapping = {}\n\n with io.open(dot_env_path, 'r', encoding='utf-8') as fp:\n for line in fp:\n # maxsplit=1\n if \"=\" in line:\n variable, value = line.split(\"=\", 1)\n elif \":\" in line:\n variable, value = line.split(\":\", 1)\n else:\n raise exceptions.FileFormatError(\".env format error\")\n\n env_variables_mapping[variable.strip()] = value.strip()\n\n utils.set_os_environ(env_variables_mapping)\n return env_variables_mapping"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef locate_file(start_path, file_name):\n if os.path.isfile(start_path):\n start_dir_path = os.path.dirname(start_path)\n elif os.path.isdir(start_path):\n start_dir_path = start_path\n else:\n raise exceptions.FileNotFound(\"invalid path: {}\".format(start_path))\n\n file_path = os.path.join(start_dir_path, file_name)\n if os.path.isfile(file_path):\n return os.path.abspath(file_path)\n\n # current working directory\n if os.path.abspath(start_dir_path) in [os.getcwd(), os.path.abspath(os.sep)]:\n raise exceptions.FileNotFound(\"{} not found in {}\".format(file_name, start_path))\n\n # locate recursive upward\n return locate_file(os.path.dirname(start_dir_path), file_name)", "response": "locate filename and return absolute file path."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef load_module_functions(module):\n module_functions = {}\n\n for name, item in vars(module).items():\n if validator.is_function(item):\n module_functions[name] = item\n\n return module_functions", "response": "loads python module functions."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nextending with api reference", "response": "def __extend_with_api_ref(raw_testinfo):\n \"\"\" extend with api reference\n\n Raises:\n exceptions.ApiNotFound: api not found\n\n \"\"\"\n api_name = raw_testinfo[\"api\"]\n\n # api maybe defined in two types:\n # 1, individual file: each file is corresponding to one api definition\n # 2, api sets file: one file contains a list of api definitions\n if not os.path.isabs(api_name):\n # make compatible with Windows/Linux\n api_path = os.path.join(tests_def_mapping[\"PWD\"], *api_name.split(\"/\"))\n if os.path.isfile(api_path):\n # type 1: api is defined in individual file\n api_name = api_path\n\n try:\n block = tests_def_mapping[\"api\"][api_name]\n # NOTICE: avoid project_mapping been changed during iteration.\n raw_testinfo[\"api_def\"] = utils.deepcopy_dict(block)\n except KeyError:\n raise exceptions.ApiNotFound(\"{} not found!\".format(api_name))"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef load_folder_content(folder_path):\n items_mapping = {}\n\n for file_path in load_folder_files(folder_path):\n items_mapping[file_path] = load_file(file_path)\n\n return items_mapping", "response": "Loads all files in folder and returns a dict mapping file paths to api definitions."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nloading the api definitions from the given api folder.", "response": "def load_api_folder(api_folder_path):\n \"\"\" load api definitions from api folder.\n\n Args:\n api_folder_path (str): api files folder.\n\n api file should be in the following format:\n [\n {\n \"api\": {\n \"def\": \"api_login\",\n \"request\": {},\n \"validate\": []\n }\n },\n {\n \"api\": {\n \"def\": \"api_logout\",\n \"request\": {},\n \"validate\": []\n }\n }\n ]\n\n Returns:\n dict: api definition mapping.\n\n {\n \"api_login\": {\n \"function_meta\": {\"func_name\": \"api_login\", \"args\": [], \"kwargs\": {}}\n \"request\": {}\n },\n \"api_logout\": {\n \"function_meta\": {\"func_name\": \"api_logout\", \"args\": [], \"kwargs\": {}}\n \"request\": {}\n }\n }\n\n \"\"\"\n api_definition_mapping = {}\n\n api_items_mapping = load_folder_content(api_folder_path)\n\n for api_file_path, api_items in api_items_mapping.items():\n # TODO: add JSON schema validation\n if isinstance(api_items, list):\n for api_item in api_items:\n key, api_dict = api_item.popitem()\n api_id = api_dict.get(\"id\") or api_dict.get(\"def\") or api_dict.get(\"name\")\n if key != \"api\" or not api_id:\n raise exceptions.ParamsError(\n \"Invalid API defined in {}\".format(api_file_path))\n\n if api_id in api_definition_mapping:\n raise exceptions.ParamsError(\n \"Duplicated API ({}) defined in {}\".format(api_id, api_file_path))\n else:\n api_definition_mapping[api_id] = api_dict\n\n elif isinstance(api_items, dict):\n if api_file_path in api_definition_mapping:\n raise exceptions.ParamsError(\n \"Duplicated API defined: {}\".format(api_file_path))\n else:\n api_definition_mapping[api_file_path] = api_items\n\n return api_definition_mapping"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef main_hrun():\n import argparse\n from httprunner import logger\n from httprunner.__about__ import __description__, __version__\n from httprunner.api import HttpRunner\n from httprunner.compat import is_py2\n from httprunner.validator import validate_json_file\n from httprunner.utils import (create_scaffold, get_python2_retire_msg,\n prettify_json_file)\n\n parser = argparse.ArgumentParser(description=__description__)\n parser.add_argument(\n '-V', '--version', dest='version', action='store_true',\n help=\"show version\")\n parser.add_argument(\n 'testcase_paths', nargs='*',\n help=\"testcase file path\")\n parser.add_argument(\n '--log-level', default='INFO',\n help=\"Specify logging level, default is INFO.\")\n parser.add_argument(\n '--log-file',\n help=\"Write logs to specified file path.\")\n parser.add_argument(\n '--dot-env-path',\n help=\"Specify .env file path, which is useful for keeping sensitive data.\")\n parser.add_argument(\n '--report-template',\n help=\"specify report template path.\")\n parser.add_argument(\n '--report-dir',\n help=\"specify report save directory.\")\n parser.add_argument(\n '--failfast', action='store_true', default=False,\n help=\"Stop the test run on the first error or failure.\")\n parser.add_argument(\n '--save-tests', action='store_true', default=False,\n help=\"Save loaded tests and parsed tests to JSON file.\")\n parser.add_argument(\n '--startproject',\n help=\"Specify new project name.\")\n parser.add_argument(\n '--validate', nargs='*',\n help=\"Validate JSON testcase format.\")\n parser.add_argument(\n '--prettify', nargs='*',\n help=\"Prettify JSON testcase format.\")\n\n args = parser.parse_args()\n logger.setup_logger(args.log_level, args.log_file)\n\n if is_py2:\n logger.log_warning(get_python2_retire_msg())\n\n if args.version:\n logger.color_print(\"{}\".format(__version__), \"GREEN\")\n exit(0)\n\n if args.validate:\n validate_json_file(args.validate)\n exit(0)\n if args.prettify:\n prettify_json_file(args.prettify)\n exit(0)\n\n project_name = args.startproject\n if project_name:\n create_scaffold(project_name)\n exit(0)\n\n runner = HttpRunner(\n failfast=args.failfast,\n save_tests=args.save_tests,\n report_template=args.report_template,\n report_dir=args.report_dir\n )\n try:\n for path in args.testcase_paths:\n runner.run(path, dot_env_path=args.dot_env_path)\n except Exception:\n logger.log_error(\"!!!!!!!!!! exception stage: {} !!!!!!!!!!\".format(runner.exception_stage))\n raise\n\n return 0", "response": "Main function for httprunner API test."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef is_var_or_func_exist(content):\n if not isinstance(content, basestring):\n return False\n\n try:\n match_start_position = content.index(\"$\", 0)\n except ValueError:\n return False\n\n while match_start_position < len(content):\n dollar_match = dolloar_regex_compile.match(content, match_start_position)\n if dollar_match:\n match_start_position = dollar_match.end()\n continue\n\n func_match = function_regex_compile.match(content, match_start_position)\n if func_match:\n return True\n\n var_match = variable_regex_compile.match(content, match_start_position)\n if var_match:\n return True\n\n return False", "response": "check if variable or function exist"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nextract all variable names from content which is in format var1 var2", "response": "def regex_findall_variables(content):\n \"\"\" extract all variable names from content, which is in format $variable\n\n Args:\n content (str): string content\n\n Returns:\n list: variables list extracted from string content\n\n Examples:\n >>> regex_findall_variables(\"$variable\")\n [\"variable\"]\n\n >>> regex_findall_variables(\"/blog/$postid\")\n [\"postid\"]\n\n >>> regex_findall_variables(\"/$var1/$var2\")\n [\"var1\", \"var2\"]\n\n >>> regex_findall_variables(\"abc\")\n []\n\n \"\"\"\n try:\n vars_list = []\n for var_tuple in variable_regex_compile.findall(content):\n vars_list.append(\n var_tuple[0] or var_tuple[1]\n )\n return vars_list\n except TypeError:\n return []"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nparsing parameters and generate a list of items for the current language", "response": "def parse_parameters(parameters, variables_mapping=None, functions_mapping=None):\n \"\"\" parse parameters and generate cartesian product.\n\n Args:\n parameters (list) parameters: parameter name and value in list\n parameter value may be in three types:\n (1) data list, e.g. [\"iOS/10.1\", \"iOS/10.2\", \"iOS/10.3\"]\n (2) call built-in parameterize function, \"${parameterize(account.csv)}\"\n (3) call custom function in debugtalk.py, \"${gen_app_version()}\"\n\n variables_mapping (dict): variables mapping loaded from testcase config\n functions_mapping (dict): functions mapping loaded from debugtalk.py\n\n Returns:\n list: cartesian product list\n\n Examples:\n >>> parameters = [\n {\"user_agent\": [\"iOS/10.1\", \"iOS/10.2\", \"iOS/10.3\"]},\n {\"username-password\": \"${parameterize(account.csv)}\"},\n {\"app_version\": \"${gen_app_version()}\"}\n ]\n >>> parse_parameters(parameters)\n\n \"\"\"\n variables_mapping = variables_mapping or {}\n functions_mapping = functions_mapping or {}\n parsed_parameters_list = []\n\n parameters = utils.ensure_mapping_format(parameters)\n for parameter_name, parameter_content in parameters.items():\n parameter_name_list = parameter_name.split(\"-\")\n\n if isinstance(parameter_content, list):\n # (1) data list\n # e.g. {\"app_version\": [\"2.8.5\", \"2.8.6\"]}\n # => [{\"app_version\": \"2.8.5\", \"app_version\": \"2.8.6\"}]\n # e.g. {\"username-password\": [[\"user1\", \"111111\"], [\"test2\", \"222222\"]}\n # => [{\"username\": \"user1\", \"password\": \"111111\"}, {\"username\": \"user2\", \"password\": \"222222\"}]\n parameter_content_list = []\n for parameter_item in parameter_content:\n if not isinstance(parameter_item, (list, tuple)):\n # \"2.8.5\" => [\"2.8.5\"]\n parameter_item = [parameter_item]\n\n # [\"app_version\"], [\"2.8.5\"] => {\"app_version\": \"2.8.5\"}\n # [\"username\", \"password\"], [\"user1\", \"111111\"] => {\"username\": \"user1\", \"password\": \"111111\"}\n parameter_content_dict = dict(zip(parameter_name_list, parameter_item))\n\n parameter_content_list.append(parameter_content_dict)\n else:\n # (2) & (3)\n parsed_variables_mapping = parse_variables_mapping(\n variables_mapping\n )\n parsed_parameter_content = eval_lazy_data(\n parameter_content,\n parsed_variables_mapping,\n functions_mapping\n )\n if not isinstance(parsed_parameter_content, list):\n raise exceptions.ParamsError(\"parameters syntax error!\")\n\n parameter_content_list = []\n for parameter_item in parsed_parameter_content:\n if isinstance(parameter_item, dict):\n # get subset by parameter name\n # {\"app_version\": \"${gen_app_version()}\"}\n # gen_app_version() => [{'app_version': '2.8.5'}, {'app_version': '2.8.6'}]\n # {\"username-password\": \"${get_account()}\"}\n # get_account() => [\n # {\"username\": \"user1\", \"password\": \"111111\"},\n # {\"username\": \"user2\", \"password\": \"222222\"}\n # ]\n parameter_dict = {key: parameter_item[key] for key in parameter_name_list}\n elif isinstance(parameter_item, (list, tuple)):\n # {\"username-password\": \"${get_account()}\"}\n # get_account() => [(\"user1\", \"111111\"), (\"user2\", \"222222\")]\n parameter_dict = dict(zip(parameter_name_list, parameter_item))\n elif len(parameter_name_list) == 1:\n # {\"user_agent\": \"${get_user_agent()}\"}\n # get_user_agent() => [\"iOS/10.1\", \"iOS/10.2\"]\n parameter_dict = {\n parameter_name_list[0]: parameter_item\n }\n\n parameter_content_list.append(parameter_dict)\n\n parsed_parameters_list.append(parameter_content_list)\n\n return utils.gen_cartesian_product(*parsed_parameters_list)"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_mapping_variable(variable_name, variables_mapping):\n try:\n return variables_mapping[variable_name]\n except KeyError:\n raise exceptions.VariableNotFound(\"{} is not found.\".format(variable_name))", "response": "get variable from variables_mapping."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_mapping_function(function_name, functions_mapping):\n if function_name in functions_mapping:\n return functions_mapping[function_name]\n\n elif function_name in [\"parameterize\", \"P\"]:\n from httprunner import loader\n return loader.load_csv_file\n\n elif function_name in [\"environ\", \"ENV\"]:\n return utils.get_os_environ\n\n try:\n # check if HttpRunner builtin functions\n from httprunner import loader\n built_in_functions = loader.load_builtin_functions()\n return built_in_functions[function_name]\n except KeyError:\n pass\n\n try:\n # check if Python builtin functions\n item_func = eval(function_name)\n if callable(item_func):\n # is builtin function\n return item_func\n except (NameError, TypeError):\n # is not builtin function\n raise exceptions.FunctionNotFound(\"{} is not found.\".format(function_name))", "response": "get function from functions_mapping and returns object"} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nparse function params to args and kwargs", "response": "def parse_function_params(params):\n \"\"\" parse function params to args and kwargs.\n\n Args:\n params (str): function param in string\n\n Returns:\n dict: function meta dict\n\n {\n \"args\": [],\n \"kwargs\": {}\n }\n\n Examples:\n >>> parse_function_params(\"\")\n {'args': [], 'kwargs': {}}\n\n >>> parse_function_params(\"5\")\n {'args': [5], 'kwargs': {}}\n\n >>> parse_function_params(\"1, 2\")\n {'args': [1, 2], 'kwargs': {}}\n\n >>> parse_function_params(\"a=1, b=2\")\n {'args': [], 'kwargs': {'a': 1, 'b': 2}}\n\n >>> parse_function_params(\"1, 2, a=3, b=4\")\n {'args': [1, 2], 'kwargs': {'a':3, 'b':4}}\n\n \"\"\"\n function_meta = {\n \"args\": [],\n \"kwargs\": {}\n }\n\n params_str = params.strip()\n if params_str == \"\":\n return function_meta\n\n args_list = params_str.split(',')\n for arg in args_list:\n arg = arg.strip()\n if '=' in arg:\n key, value = arg.split('=')\n function_meta[\"kwargs\"][key.strip()] = parse_string_value(value.strip())\n else:\n function_meta[\"args\"].append(parse_string_value(arg))\n\n return function_meta"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nmake string in content as lazy object with functions_mapping check_variables_set and cached", "response": "def prepare_lazy_data(content, functions_mapping=None, check_variables_set=None, cached=False):\n \"\"\" make string in content as lazy object with functions_mapping\n\n Raises:\n exceptions.VariableNotFound: if any variable undefined in check_variables_set\n\n \"\"\"\n # TODO: refactor type check\n if content is None or isinstance(content, (numeric_types, bool, type)):\n return content\n\n elif isinstance(content, (list, set, tuple)):\n return [\n prepare_lazy_data(\n item,\n functions_mapping,\n check_variables_set,\n cached\n )\n for item in content\n ]\n\n elif isinstance(content, dict):\n parsed_content = {}\n for key, value in content.items():\n parsed_key = prepare_lazy_data(\n key,\n functions_mapping,\n check_variables_set,\n cached\n )\n parsed_value = prepare_lazy_data(\n value,\n functions_mapping,\n check_variables_set,\n cached\n )\n parsed_content[parsed_key] = parsed_value\n\n return parsed_content\n\n elif isinstance(content, basestring):\n # content is in string format here\n if not is_var_or_func_exist(content):\n # content is neither variable nor function\n # replace $$ notation with $ and consider it as normal char.\n # e.g. abc => abc, abc$$def => abc$def, abc$$$$def$$h => abc$$def$h\n return content.replace(\"$$\", \"$\")\n\n functions_mapping = functions_mapping or {}\n check_variables_set = check_variables_set or set()\n content = content.strip()\n content = LazyString(content, functions_mapping, check_variables_set, cached)\n\n return content"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nparse lazy data with evaluated variables mapping.", "response": "def parse_lazy_data(content, variables_mapping=None):\n \"\"\" parse lazy data with evaluated variables mapping.\n Notice: variables_mapping should not contain any variable or function.\n \"\"\"\n # TODO: refactor type check\n if content is None or isinstance(content, (numeric_types, bool, type)):\n return content\n\n elif isinstance(content, LazyString):\n variables_mapping = utils.ensure_mapping_format(variables_mapping or {})\n return content.to_value(variables_mapping)\n\n elif isinstance(content, (list, set, tuple)):\n return [\n parse_lazy_data(item, variables_mapping)\n for item in content\n ]\n\n elif isinstance(content, dict):\n parsed_content = {}\n for key, value in content.items():\n parsed_key = parse_lazy_data(key, variables_mapping)\n parsed_value = parse_lazy_data(value, variables_mapping)\n parsed_content[parsed_key] = parsed_value\n\n return parsed_content\n\n return content"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nevaluate lazy data instantly.", "response": "def eval_lazy_data(content, variables_mapping=None, functions_mapping=None):\n \"\"\" evaluate data instantly.\n Notice: variables_mapping should not contain any variable or function.\n \"\"\"\n variables_mapping = variables_mapping or {}\n check_variables_set = set(variables_mapping.keys())\n return parse_lazy_data(\n prepare_lazy_data(\n content,\n functions_mapping,\n check_variables_set\n ),\n variables_mapping\n )"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef extract_variables(content):\n if isinstance(content, (list, set, tuple)):\n variables = set()\n for item in content:\n variables = variables | extract_variables(item)\n return variables\n\n elif isinstance(content, dict):\n variables = set()\n for key, value in content.items():\n variables = variables | extract_variables(value)\n return variables\n\n elif isinstance(content, LazyString):\n return set(regex_findall_variables(content.raw_string))\n\n return set()", "response": "extract all variables in content recursively."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef parse_variables_mapping(variables_mapping, ignore=False):\n run_times = 0\n parsed_variables_mapping = {}\n\n while len(parsed_variables_mapping) != len(variables_mapping):\n for var_name in variables_mapping:\n\n run_times += 1\n if run_times > len(variables_mapping) * 4:\n not_found_variables = {\n key: variables_mapping[key]\n for key in variables_mapping\n if key not in parsed_variables_mapping\n }\n raise exceptions.VariableNotFound(not_found_variables)\n\n if var_name in parsed_variables_mapping:\n continue\n\n value = variables_mapping[var_name]\n variables = extract_variables(value)\n\n # check if reference variable itself\n if var_name in variables:\n # e.g.\n # var_name = \"token\"\n # variables_mapping = {\"token\": LazyString($token)}\n # var_name = \"key\"\n # variables_mapping = {\"key\": [LazyString($key), 2]}\n if ignore:\n parsed_variables_mapping[var_name] = value\n continue\n raise exceptions.VariableNotFound(var_name)\n\n if variables:\n # reference other variable, or function call with other variable\n # e.g. {\"varA\": \"123$varB\", \"varB\": \"456$varC\"}\n # e.g. {\"varC\": \"${sum_two($a, $b)}\"}\n if any([_var_name not in parsed_variables_mapping for _var_name in variables]):\n # reference variable not parsed\n continue\n\n parsed_value = parse_lazy_data(value, parsed_variables_mapping)\n parsed_variables_mapping[var_name] = parsed_value\n\n return parsed_variables_mapping", "response": "Parse the variables_mapping and return a dictionary of the variables that are found in the base base."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nextend test with api definition", "response": "def _extend_with_api(test_dict, api_def_dict):\n \"\"\" extend test with api definition, test will merge and override api definition.\n\n Args:\n test_dict (dict): test block, this will override api_def_dict\n api_def_dict (dict): api definition\n\n Examples:\n >>> api_def_dict = {\n \"name\": \"get token 1\",\n \"request\": {...},\n \"validate\": [{'eq': ['status_code', 200]}]\n }\n >>> test_dict = {\n \"name\": \"get token 2\",\n \"extract\": {\"token\": \"content.token\"},\n \"validate\": [{'eq': ['status_code', 201]}, {'len_eq': ['content.token', 16]}]\n }\n >>> _extend_with_api(test_dict, api_def_dict)\n >>> print(test_dict)\n {\n \"name\": \"get token 2\",\n \"request\": {...},\n \"extract\": {\"token\": \"content.token\"},\n \"validate\": [{'eq': ['status_code', 201]}, {'len_eq': ['content.token', 16]}]\n }\n\n \"\"\"\n # override api name\n test_dict.setdefault(\"name\", api_def_dict.pop(\"name\", \"api name undefined\"))\n\n # override variables\n def_variables = api_def_dict.pop(\"variables\", [])\n test_dict[\"variables\"] = utils.extend_variables(\n def_variables,\n test_dict.get(\"variables\", {})\n )\n\n # merge & override validators TODO: relocate\n def_raw_validators = api_def_dict.pop(\"validate\", [])\n def_validators = [\n validator.uniform_validator(_validator)\n for _validator in def_raw_validators\n ]\n ref_validators = test_dict.pop(\"validate\", [])\n test_dict[\"validate\"] = validator.extend_validators(\n def_validators,\n ref_validators\n )\n\n # merge & override extractors\n def_extrators = api_def_dict.pop(\"extract\", {})\n test_dict[\"extract\"] = utils.extend_variables(\n def_extrators,\n test_dict.get(\"extract\", {})\n )\n\n # merge & override request\n test_dict[\"request\"] = api_def_dict.pop(\"request\", {})\n\n # base_url & verify: priority api_def_dict > test_dict\n if api_def_dict.get(\"base_url\"):\n test_dict[\"base_url\"] = api_def_dict[\"base_url\"]\n\n if \"verify\" in api_def_dict:\n test_dict[\"request\"][\"verify\"] = api_def_dict[\"verify\"]\n\n # merge & override setup_hooks\n def_setup_hooks = api_def_dict.pop(\"setup_hooks\", [])\n ref_setup_hooks = test_dict.get(\"setup_hooks\", [])\n extended_setup_hooks = list(set(def_setup_hooks + ref_setup_hooks))\n if extended_setup_hooks:\n test_dict[\"setup_hooks\"] = extended_setup_hooks\n # merge & override teardown_hooks\n def_teardown_hooks = api_def_dict.pop(\"teardown_hooks\", [])\n ref_teardown_hooks = test_dict.get(\"teardown_hooks\", [])\n extended_teardown_hooks = list(set(def_teardown_hooks + ref_teardown_hooks))\n if extended_teardown_hooks:\n test_dict[\"teardown_hooks\"] = extended_teardown_hooks\n\n # TODO: extend with other api definition items, e.g. times\n test_dict.update(api_def_dict)"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nparsing lazy functon instance", "response": "def __parse(self, function_meta):\n \"\"\" init func as lazy functon instance\n\n Args:\n function_meta (dict): function meta including name, args and kwargs\n \"\"\"\n self._func = get_mapping_function(\n function_meta[\"func_name\"],\n self.functions_mapping\n )\n self.func_name = self._func.__name__\n self._args = prepare_lazy_data(\n function_meta.get(\"args\", []),\n self.functions_mapping,\n self.check_variables_set\n )\n self._kwargs = prepare_lazy_data(\n function_meta.get(\"kwargs\", {}),\n self.functions_mapping,\n self.check_variables_set\n )\n\n if self.func_name == \"load_csv_file\":\n if len(self._args) != 1 or self._kwargs:\n raise exceptions.ParamsError(\"P() should only pass in one argument!\")\n self._args = [self._args[0]]\n elif self.func_name == \"get_os_environ\":\n if len(self._args) != 1 or self._kwargs:\n raise exceptions.ParamsError(\"ENV() should only pass in one argument!\")\n self._args = [self._args[0]]"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef to_value(self, variables_mapping=None):\n variables_mapping = variables_mapping or {}\n args = parse_lazy_data(self._args, variables_mapping)\n kwargs = parse_lazy_data(self._kwargs, variables_mapping)\n self.cache_key = self.__prepare_cache_key(args, kwargs)\n return self._func(*args, **kwargs)", "response": "parse lazy data with evaluated variables mapping."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef __parse(self, raw_string):\n self._args = []\n\n def escape_braces(origin_string):\n return origin_string.replace(\"{\", \"{{\").replace(\"}\", \"}}\")\n\n try:\n match_start_position = raw_string.index(\"$\", 0)\n begin_string = raw_string[0:match_start_position]\n self._string = escape_braces(begin_string)\n except ValueError:\n self._string = escape_braces(raw_string)\n return\n\n while match_start_position < len(raw_string):\n\n # Notice: notation priority\n # $$ > ${func($a, $b)} > $var\n\n # search $$\n dollar_match = dolloar_regex_compile.match(raw_string, match_start_position)\n if dollar_match:\n match_start_position = dollar_match.end()\n self._string += \"$\"\n continue\n\n # search function like ${func($a, $b)}\n func_match = function_regex_compile.match(raw_string, match_start_position)\n if func_match:\n function_meta = parse_function_params(func_match.group(1))\n function_meta = {\n \"func_name\": func_match.group(1)\n }\n function_meta.update(parse_function_params(func_match.group(2)))\n lazy_func = LazyFunction(\n function_meta,\n self.functions_mapping,\n self.check_variables_set\n )\n self._args.append(lazy_func)\n match_start_position = func_match.end()\n self._string += \"{}\"\n continue\n\n # search variable like ${var} or $var\n var_match = variable_regex_compile.match(raw_string, match_start_position)\n if var_match:\n var_name = var_match.group(1) or var_match.group(2)\n # check if any variable undefined in check_variables_set\n if var_name not in self.check_variables_set:\n raise exceptions.VariableNotFound(var_name)\n\n self._args.append(var_name)\n match_start_position = var_match.end()\n self._string += \"{}\"\n continue\n\n curr_position = match_start_position\n try:\n # find next $ location\n match_start_position = raw_string.index(\"$\", curr_position+1)\n remain_string = raw_string[curr_position:match_start_position]\n except ValueError:\n remain_string = raw_string[curr_position:]\n # break while loop\n match_start_position = len(raw_string)\n\n self._string += escape_braces(remain_string)", "response": "Parse the raw string and return a tuple of the base class and the string with the function and arguments."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nparsing lazy data with evaluated variables mapping.", "response": "def to_value(self, variables_mapping=None):\n \"\"\" parse lazy data with evaluated variables mapping.\n Notice: variables_mapping should not contain any variable or function.\n \"\"\"\n variables_mapping = variables_mapping or {}\n\n args = []\n for arg in self._args:\n if isinstance(arg, LazyFunction):\n if self.cached and arg.cache_key and arg.cache_key in cached_functions_mapping:\n value = cached_functions_mapping[arg.cache_key]\n else:\n value = arg.to_value(variables_mapping)\n cached_functions_mapping[arg.cache_key] = value\n args.append(value)\n else:\n # variable\n var_value = get_mapping_variable(arg, variables_mapping)\n args.append(var_value)\n\n if self._string == \"{}\":\n return args[0]\n else:\n return self._string.format(*args)"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_req_resp_record(self, resp_obj):\n def log_print(req_resp_dict, r_type):\n msg = \"\\n================== {} details ==================\\n\".format(r_type)\n for key, value in req_resp_dict[r_type].items():\n msg += \"{:<16} : {}\\n\".format(key, repr(value))\n logger.log_debug(msg)\n\n req_resp_dict = {\n \"request\": {},\n \"response\": {}\n }\n\n # record actual request info\n req_resp_dict[\"request\"][\"url\"] = resp_obj.request.url\n req_resp_dict[\"request\"][\"method\"] = resp_obj.request.method\n req_resp_dict[\"request\"][\"headers\"] = dict(resp_obj.request.headers)\n\n request_body = resp_obj.request.body\n if request_body:\n request_content_type = lower_dict_keys(\n req_resp_dict[\"request\"][\"headers\"]\n ).get(\"content-type\")\n if request_content_type and \"multipart/form-data\" in request_content_type:\n # upload file type\n req_resp_dict[\"request\"][\"body\"] = \"upload file stream (OMITTED)\"\n else:\n req_resp_dict[\"request\"][\"body\"] = request_body\n\n # log request details in debug mode\n log_print(req_resp_dict, \"request\")\n\n # record response info\n req_resp_dict[\"response\"][\"ok\"] = resp_obj.ok\n req_resp_dict[\"response\"][\"url\"] = resp_obj.url\n req_resp_dict[\"response\"][\"status_code\"] = resp_obj.status_code\n req_resp_dict[\"response\"][\"reason\"] = resp_obj.reason\n req_resp_dict[\"response\"][\"cookies\"] = resp_obj.cookies or {}\n req_resp_dict[\"response\"][\"encoding\"] = resp_obj.encoding\n resp_headers = dict(resp_obj.headers)\n req_resp_dict[\"response\"][\"headers\"] = resp_headers\n\n lower_resp_headers = lower_dict_keys(resp_headers)\n content_type = lower_resp_headers.get(\"content-type\", \"\")\n req_resp_dict[\"response\"][\"content_type\"] = content_type\n\n if \"image\" in content_type:\n # response is image type, record bytes content only\n req_resp_dict[\"response\"][\"content\"] = resp_obj.content\n else:\n try:\n # try to record json data\n req_resp_dict[\"response\"][\"json\"] = resp_obj.json()\n except ValueError:\n # only record at most 512 text charactors\n resp_text = resp_obj.text\n req_resp_dict[\"response\"][\"text\"] = omit_long_data(resp_text)\n\n # log response details in debug mode\n log_print(req_resp_dict, \"response\")\n\n return req_resp_dict", "response": "get request and response info from Response object."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef request(self, method, url, name=None, **kwargs):\n self.init_meta_data()\n\n # record test name\n self.meta_data[\"name\"] = name\n\n # record original request info\n self.meta_data[\"data\"][0][\"request\"][\"method\"] = method\n self.meta_data[\"data\"][0][\"request\"][\"url\"] = url\n kwargs.setdefault(\"timeout\", 120)\n self.meta_data[\"data\"][0][\"request\"].update(kwargs)\n\n start_timestamp = time.time()\n response = self._send_request_safe_mode(method, url, **kwargs)\n response_time_ms = round((time.time() - start_timestamp) * 1000, 2)\n\n # get the length of the content, but if the argument stream is set to True, we take\n # the size from the content-length header, in order to not trigger fetching of the body\n if kwargs.get(\"stream\", False):\n content_size = int(dict(response.headers).get(\"content-length\") or 0)\n else:\n content_size = len(response.content or \"\")\n\n # record the consumed time\n self.meta_data[\"stat\"] = {\n \"response_time_ms\": response_time_ms,\n \"elapsed_ms\": response.elapsed.microseconds / 1000.0,\n \"content_size\": content_size\n }\n\n # record request and response histories, include 30X redirection\n response_list = response.history + [response]\n self.meta_data[\"data\"] = [\n self.get_req_resp_record(resp_obj)\n for resp_obj in response_list\n ]\n\n try:\n response.raise_for_status()\n except RequestException as e:\n logger.log_error(u\"{exception}\".format(exception=str(e)))\n else:\n logger.log_info(\n \"\"\"status_code: {}, response_time(ms): {} ms, response_length: {} bytes\\n\"\"\".format(\n response.status_code,\n response_time_ms,\n content_size\n )\n )\n\n return response", "response": "Construct and sends a new HTTP request."} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nsending a HTTP request and catch any exception that might occur due to connection problems.", "response": "def _send_request_safe_mode(self, method, url, **kwargs):\n \"\"\"\n Send a HTTP request, and catch any exception that might occur due to connection problems.\n Safe mode has been removed from requests 1.x.\n \"\"\"\n try:\n msg = \"processed request:\\n\"\n msg += \"> {method} {url}\\n\".format(method=method, url=url)\n msg += \"> kwargs: {kwargs}\".format(kwargs=kwargs)\n logger.log_debug(msg)\n return requests.Session.request(self, method, url, **kwargs)\n except (MissingSchema, InvalidSchema, InvalidURL):\n raise\n except RequestException as ex:\n resp = ApiResponse()\n resp.error = ex\n resp.status_code = 0 # with this status_code, content returns None\n resp.request = Request(method, url).prepare()\n return resp"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nconverts comparator alias to uniform name", "response": "def get_uniform_comparator(comparator):\n \"\"\" convert comparator alias to uniform name\n \"\"\"\n if comparator in [\"eq\", \"equals\", \"==\", \"is\"]:\n return \"equals\"\n elif comparator in [\"lt\", \"less_than\"]:\n return \"less_than\"\n elif comparator in [\"le\", \"less_than_or_equals\"]:\n return \"less_than_or_equals\"\n elif comparator in [\"gt\", \"greater_than\"]:\n return \"greater_than\"\n elif comparator in [\"ge\", \"greater_than_or_equals\"]:\n return \"greater_than_or_equals\"\n elif comparator in [\"ne\", \"not_equals\"]:\n return \"not_equals\"\n elif comparator in [\"str_eq\", \"string_equals\"]:\n return \"string_equals\"\n elif comparator in [\"len_eq\", \"length_equals\", \"count_eq\"]:\n return \"length_equals\"\n elif comparator in [\"len_gt\", \"count_gt\", \"length_greater_than\", \"count_greater_than\"]:\n return \"length_greater_than\"\n elif comparator in [\"len_ge\", \"count_ge\", \"length_greater_than_or_equals\", \\\n \"count_greater_than_or_equals\"]:\n return \"length_greater_than_or_equals\"\n elif comparator in [\"len_lt\", \"count_lt\", \"length_less_than\", \"count_less_than\"]:\n return \"length_less_than\"\n elif comparator in [\"len_le\", \"count_le\", \"length_less_than_or_equals\", \\\n \"count_less_than_or_equals\"]:\n return \"length_less_than_or_equals\"\n else:\n return comparator"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef uniform_validator(validator):\n if not isinstance(validator, dict):\n raise exceptions.ParamsError(\"invalid validator: {}\".format(validator))\n\n if \"check\" in validator and \"expect\" in validator:\n # format1\n check_item = validator[\"check\"]\n expect_value = validator[\"expect\"]\n comparator = validator.get(\"comparator\", \"eq\")\n\n elif len(validator) == 1:\n # format2\n comparator = list(validator.keys())[0]\n compare_values = validator[comparator]\n\n if not isinstance(compare_values, list) or len(compare_values) != 2:\n raise exceptions.ParamsError(\"invalid validator: {}\".format(validator))\n\n check_item, expect_value = compare_values\n\n else:\n raise exceptions.ParamsError(\"invalid validator: {}\".format(validator))\n\n # uniform comparator, e.g. lt => less_than, eq => equals\n comparator = get_uniform_comparator(comparator)\n\n return {\n \"check\": check_item,\n \"expect\": expect_value,\n \"comparator\": comparator\n }", "response": "unify a validator for the current version of the current node"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _convert_validators_to_mapping(validators):\n validators_mapping = {}\n\n for validator in validators:\n if not isinstance(validator[\"check\"], collections.Hashable):\n check = json.dumps(validator[\"check\"])\n else:\n check = validator[\"check\"]\n\n key = (check, validator[\"comparator\"])\n validators_mapping[key] = validator\n\n return validators_mapping", "response": "Convert validators list to dict."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef extend_validators(raw_validators, override_validators):\n\n if not raw_validators:\n return override_validators\n\n elif not override_validators:\n return raw_validators\n\n else:\n def_validators_mapping = _convert_validators_to_mapping(raw_validators)\n ref_validators_mapping = _convert_validators_to_mapping(override_validators)\n\n def_validators_mapping.update(ref_validators_mapping)\n return list(def_validators_mapping.values())", "response": "extend raw_validators with override_validators."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef is_variable(tup):\n name, item = tup\n if callable(item):\n # function or class\n return False\n\n if isinstance(item, types.ModuleType):\n # imported module\n return False\n\n if name.startswith(\"_\"):\n # private property\n return False\n\n return True", "response": "Returns True if the given tuple is a variable."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nvalidating JSON testcase file", "response": "def validate_json_file(file_list):\n \"\"\" validate JSON testcase format\n \"\"\"\n for json_file in set(file_list):\n if not json_file.endswith(\".json\"):\n logger.log_warning(\"Only JSON file format can be validated, skip: {}\".format(json_file))\n continue\n\n logger.color_print(\"Start to validate JSON file: {}\".format(json_file), \"GREEN\")\n\n with io.open(json_file) as stream:\n try:\n json.load(stream)\n except ValueError as e:\n raise SystemExit(e)\n\n print(\"OK\")"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef gen_random_string(str_len):\n return ''.join(\n random.choice(string.ascii_letters + string.digits) for _ in range(str_len))", "response": "generate random string with specified length"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_timestamp(str_len=13):\n if isinstance(str_len, integer_types) and 0 < str_len < 17:\n return builtin_str(time.time()).replace(\".\", \"\")[:str_len]\n\n raise ParamsError(\"timestamp length can only between 0 and 16.\")", "response": "get timestamp string, length can only between 0 and 16"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef multipart_encoder(**kwargs):\n def get_filetype(file_path):\n file_type = filetype.guess(file_path)\n if file_type:\n return file_type.mime\n else:\n return \"text/html\"\n\n fields_dict = {}\n for key, value in kwargs.items():\n\n if os.path.isabs(value):\n _file_path = value\n is_file = True\n else:\n global PWD\n _file_path = os.path.join(PWD, value)\n is_file = os.path.isfile(_file_path)\n\n if is_file:\n filename = os.path.basename(_file_path)\n with open(_file_path, 'rb') as f:\n mime_type = get_filetype(_file_path)\n fields_dict[key] = (filename, f.read(), mime_type)\n else:\n fields_dict[key] = value\n\n return MultipartEncoder(fields=fields_dict)", "response": "initialize MultipartEncoder with uploading fields."} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nparses testcase file and return locustfile path.", "response": "def parse_locustfile(file_path):\n \"\"\" parse testcase file and return locustfile path.\n if file_path is a Python file, assume it is a locustfile\n if file_path is a YAML/JSON file, convert it to locustfile\n \"\"\"\n if not os.path.isfile(file_path):\n color_print(\"file path invalid, exit.\", \"RED\")\n sys.exit(1)\n\n file_suffix = os.path.splitext(file_path)[1]\n if file_suffix == \".py\":\n locustfile_path = file_path\n elif file_suffix in ['.yaml', '.yml', '.json']:\n locustfile_path = gen_locustfile(file_path)\n else:\n # '' or other suffix\n color_print(\"file type should be YAML/JSON/Python, exit.\", \"RED\")\n sys.exit(1)\n\n return locustfile_path"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef gen_locustfile(testcase_file_path):\n locustfile_path = 'locustfile.py'\n template_path = os.path.join(\n os.path.dirname(os.path.realpath(__file__)),\n \"templates\",\n \"locustfile_template\"\n )\n\n with io.open(template_path, encoding='utf-8') as template:\n with io.open(locustfile_path, 'w', encoding='utf-8') as locustfile:\n template_content = template.read()\n template_content = template_content.replace(\"$TESTCASE_FILE\", testcase_file_path)\n locustfile.write(template_content)\n\n return locustfile_path", "response": "generate locustfile from template."} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nupdate session with extracted variables mapping.", "response": "def update_session_variables(self, variables_mapping):\n \"\"\" update session with extracted variables mapping.\n these variables are valid in the whole running session.\n \"\"\"\n variables_mapping = utils.ensure_mapping_format(variables_mapping)\n self.session_variables_mapping.update(variables_mapping)\n self.test_variables_mapping.update(self.session_variables_mapping)"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef __eval_validator_check(self, check_item, resp_obj):\n if isinstance(check_item, (dict, list)) \\\n or isinstance(check_item, parser.LazyString):\n # format 1/2/3\n check_value = self.eval_content(check_item)\n else:\n # format 4/5\n check_value = resp_obj.extract_field(check_item)\n\n return check_value", "response": "Evaluate check item in validator.\n "} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nvalidate the current state of the object.", "response": "def validate(self, validators, resp_obj):\n \"\"\" make validation with comparators\n \"\"\"\n self.validation_results = []\n if not validators:\n return\n\n logger.log_debug(\"start to validate.\")\n\n validate_pass = True\n failures = []\n\n for validator in validators:\n # validator should be LazyFunction object\n if not isinstance(validator, parser.LazyFunction):\n raise exceptions.ValidationFailure(\n \"validator should be parsed first: {}\".format(validators))\n\n # evaluate validator args with context variable mapping.\n validator_args = validator.get_args()\n check_item, expect_item = validator_args\n check_value = self.__eval_validator_check(\n check_item,\n resp_obj\n )\n expect_value = self.__eval_validator_expect(expect_item)\n validator.update_args([check_value, expect_value])\n\n comparator = validator.func_name\n validator_dict = {\n \"comparator\": comparator,\n \"check\": check_item,\n \"check_value\": check_value,\n \"expect\": expect_item,\n \"expect_value\": expect_value\n }\n validate_msg = \"\\nvalidate: {} {} {}({})\".format(\n check_item,\n comparator,\n expect_value,\n type(expect_value).__name__\n )\n\n try:\n validator.to_value(self.test_variables_mapping)\n validator_dict[\"check_result\"] = \"pass\"\n validate_msg += \"\\t==> pass\"\n logger.log_debug(validate_msg)\n except (AssertionError, TypeError):\n validate_pass = False\n validator_dict[\"check_result\"] = \"fail\"\n validate_msg += \"\\t==> fail\"\n validate_msg += \"\\n{}({}) {} {}({})\".format(\n check_value,\n type(check_value).__name__,\n comparator,\n expect_value,\n type(expect_value).__name__\n )\n logger.log_error(validate_msg)\n failures.append(validate_msg)\n\n self.validation_results.append(validator_dict)\n\n # restore validator args, in case of running multiple times\n validator.update_args(validator_args)\n\n if not validate_pass:\n failures_string = \"\\n\".join([failure for failure in failures])\n raise exceptions.ValidationFailure(failures_string)"} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nget summary from test result", "response": "def get_summary(result):\n \"\"\" get summary from test result\n\n Args:\n result (instance): HtmlTestResult() instance\n\n Returns:\n dict: summary extracted from result.\n\n {\n \"success\": True,\n \"stat\": {},\n \"time\": {},\n \"records\": []\n }\n\n \"\"\"\n summary = {\n \"success\": result.wasSuccessful(),\n \"stat\": {\n 'total': result.testsRun,\n 'failures': len(result.failures),\n 'errors': len(result.errors),\n 'skipped': len(result.skipped),\n 'expectedFailures': len(result.expectedFailures),\n 'unexpectedSuccesses': len(result.unexpectedSuccesses)\n }\n }\n summary[\"stat\"][\"successes\"] = summary[\"stat\"][\"total\"] \\\n - summary[\"stat\"][\"failures\"] \\\n - summary[\"stat\"][\"errors\"] \\\n - summary[\"stat\"][\"skipped\"] \\\n - summary[\"stat\"][\"expectedFailures\"] \\\n - summary[\"stat\"][\"unexpectedSuccesses\"]\n\n summary[\"time\"] = {\n 'start_at': result.start_at,\n 'duration': result.duration\n }\n summary[\"records\"] = result.records\n\n return summary"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\naggregating new_stat to origin_stat.", "response": "def aggregate_stat(origin_stat, new_stat):\n \"\"\" aggregate new_stat to origin_stat.\n\n Args:\n origin_stat (dict): origin stat dict, will be updated with new_stat dict.\n new_stat (dict): new stat dict.\n\n \"\"\"\n for key in new_stat:\n if key not in origin_stat:\n origin_stat[key] = new_stat[key]\n elif key == \"start_at\":\n # start datetime\n origin_stat[key] = min(origin_stat[key], new_stat[key])\n else:\n origin_stat[key] += new_stat[key]"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef stringify_summary(summary):\n for index, suite_summary in enumerate(summary[\"details\"]):\n\n if not suite_summary.get(\"name\"):\n suite_summary[\"name\"] = \"testcase {}\".format(index)\n\n for record in suite_summary.get(\"records\"):\n meta_datas = record['meta_datas']\n __stringify_meta_datas(meta_datas)\n meta_datas_expanded = []\n __expand_meta_datas(meta_datas, meta_datas_expanded)\n record[\"meta_datas_expanded\"] = meta_datas_expanded\n record[\"response_time\"] = __get_total_response_time(meta_datas_expanded)", "response": "stringify summary in order to dump json file and generate html report."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef __stringify_response(response_data):\n for key, value in response_data.items():\n\n if isinstance(value, list):\n value = json.dumps(value, indent=2, ensure_ascii=False)\n\n elif isinstance(value, bytes):\n try:\n encoding = response_data.get(\"encoding\")\n if not encoding or encoding == \"None\":\n encoding = \"utf-8\"\n\n if key == \"content\" and \"image\" in response_data[\"content_type\"]:\n # display image\n value = \"data:{};base64,{}\".format(\n response_data[\"content_type\"],\n b64encode(value).decode(encoding)\n )\n else:\n value = escape(value.decode(encoding))\n except UnicodeDecodeError:\n pass\n\n elif not isinstance(value, (basestring, numeric_types, Iterable)):\n # class instance, e.g. MultipartEncoder()\n value = repr(value)\n\n elif isinstance(value, requests.cookies.RequestsCookieJar):\n value = value.get_dict()\n\n response_data[key] = value", "response": "stringfy HTTP response data"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef __expand_meta_datas(meta_datas, meta_datas_expanded):\n if isinstance(meta_datas, dict):\n meta_datas_expanded.append(meta_datas)\n elif isinstance(meta_datas, list):\n for meta_data in meta_datas:\n __expand_meta_datas(meta_data, meta_datas_expanded)", "response": "Expand meta_datas to one level\n "} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef __get_total_response_time(meta_datas_expanded):\n try:\n response_time = 0\n for meta_data in meta_datas_expanded:\n response_time += meta_data[\"stat\"][\"response_time_ms\"]\n\n return \"{:.2f}\".format(response_time)\n\n except TypeError:\n # failure exists\n return \"N/A\"", "response": "caculate total response time of all meta_datas"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nrendering html report with specified report name and template", "response": "def render_html_report(summary, report_template=None, report_dir=None):\n \"\"\" render html report with specified report name and template\n\n Args:\n report_template (str): specify html report template path\n report_dir (str): specify html report save directory\n\n \"\"\"\n if not report_template:\n report_template = os.path.join(\n os.path.abspath(os.path.dirname(__file__)),\n \"templates\",\n \"report_template.html\"\n )\n logger.log_debug(\"No html report template specified, use default.\")\n else:\n logger.log_info(\"render with html report template: {}\".format(report_template))\n\n logger.log_info(\"Start to render Html report ...\")\n\n report_dir = report_dir or os.path.join(os.getcwd(), \"reports\")\n if not os.path.isdir(report_dir):\n os.makedirs(report_dir)\n\n start_at_timestamp = int(summary[\"time\"][\"start_at\"])\n summary[\"time\"][\"start_datetime\"] = datetime.fromtimestamp(start_at_timestamp).strftime('%Y-%m-%d %H:%M:%S')\n\n report_path = os.path.join(report_dir, \"{}.html\".format(start_at_timestamp))\n\n with io.open(report_template, \"r\", encoding='utf-8') as fp_r:\n template_content = fp_r.read()\n with io.open(report_path, 'w', encoding='utf-8') as fp_w:\n rendered_content = Template(\n template_content,\n extensions=[\"jinja2.ext.loopcontrols\"]\n ).render(summary)\n fp_w.write(rendered_content)\n\n logger.log_info(\"Generated Html report: {}\".format(report_path))\n\n return report_path"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _extract_field_with_regex(self, field):\n matched = re.search(field, self.text)\n if not matched:\n err_msg = u\"Failed to extract data with regex! => {}\\n\".format(field)\n err_msg += u\"response body: {}\\n\".format(self.text)\n logger.log_error(err_msg)\n raise exceptions.ExtractFailure(err_msg)\n\n return matched.group(1)", "response": "extract field from response body with regex."} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nextracting the value of a field from the response content.", "response": "def _extract_field_with_delimiter(self, field):\n \"\"\" response content could be json or html text.\n\n Args:\n field (str): string joined by delimiter.\n e.g.\n \"status_code\"\n \"headers\"\n \"cookies\"\n \"content\"\n \"headers.content-type\"\n \"content.person.name.first_name\"\n\n \"\"\"\n # string.split(sep=None, maxsplit=-1) -> list of strings\n # e.g. \"content.person.name\" => [\"content\", \"person.name\"]\n try:\n top_query, sub_query = field.split('.', 1)\n except ValueError:\n top_query = field\n sub_query = None\n\n # status_code\n if top_query in [\"status_code\", \"encoding\", \"ok\", \"reason\", \"url\"]:\n if sub_query:\n # status_code.XX\n err_msg = u\"Failed to extract: {}\\n\".format(field)\n logger.log_error(err_msg)\n raise exceptions.ParamsError(err_msg)\n\n return getattr(self, top_query)\n\n # cookies\n elif top_query == \"cookies\":\n cookies = self.cookies\n if not sub_query:\n # extract cookies\n return cookies\n\n try:\n return cookies[sub_query]\n except KeyError:\n err_msg = u\"Failed to extract cookie! => {}\\n\".format(field)\n err_msg += u\"response cookies: {}\\n\".format(cookies)\n logger.log_error(err_msg)\n raise exceptions.ExtractFailure(err_msg)\n\n # elapsed\n elif top_query == \"elapsed\":\n available_attributes = u\"available attributes: days, seconds, microseconds, total_seconds\"\n if not sub_query:\n err_msg = u\"elapsed is datetime.timedelta instance, attribute should also be specified!\\n\"\n err_msg += available_attributes\n logger.log_error(err_msg)\n raise exceptions.ParamsError(err_msg)\n elif sub_query in [\"days\", \"seconds\", \"microseconds\"]:\n return getattr(self.elapsed, sub_query)\n elif sub_query == \"total_seconds\":\n return self.elapsed.total_seconds()\n else:\n err_msg = \"{} is not valid datetime.timedelta attribute.\\n\".format(sub_query)\n err_msg += available_attributes\n logger.log_error(err_msg)\n raise exceptions.ParamsError(err_msg)\n\n # headers\n elif top_query == \"headers\":\n headers = self.headers\n if not sub_query:\n # extract headers\n return headers\n\n try:\n return headers[sub_query]\n except KeyError:\n err_msg = u\"Failed to extract header! => {}\\n\".format(field)\n err_msg += u\"response headers: {}\\n\".format(headers)\n logger.log_error(err_msg)\n raise exceptions.ExtractFailure(err_msg)\n\n # response body\n elif top_query in [\"content\", \"text\", \"json\"]:\n try:\n body = self.json\n except exceptions.JSONDecodeError:\n body = self.text\n\n if not sub_query:\n # extract response body\n return body\n\n if isinstance(body, (dict, list)):\n # content = {\"xxx\": 123}, content.xxx\n return utils.query_json(body, sub_query)\n elif sub_query.isdigit():\n # content = \"abcdefg\", content.3 => d\n return utils.query_json(body, sub_query)\n else:\n # content = \"abcdefg\", content.xxx\n err_msg = u\"Failed to extract attribute from response body! => {}\\n\".format(field)\n err_msg += u\"response body: {}\\n\".format(body)\n logger.log_error(err_msg)\n raise exceptions.ExtractFailure(err_msg)\n\n # new set response attributes in teardown_hooks\n elif top_query in self.__dict__:\n attributes = self.__dict__[top_query]\n\n if not sub_query:\n # extract response attributes\n return attributes\n\n if isinstance(attributes, (dict, list)):\n # attributes = {\"xxx\": 123}, content.xxx\n return utils.query_json(attributes, sub_query)\n elif sub_query.isdigit():\n # attributes = \"abcdefg\", attributes.3 => d\n return utils.query_json(attributes, sub_query)\n else:\n # content = \"attributes.new_attribute_not_exist\"\n err_msg = u\"Failed to extract cumstom set attribute from teardown hooks! => {}\\n\".format(field)\n err_msg += u\"response set attributes: {}\\n\".format(attributes)\n logger.log_error(err_msg)\n raise exceptions.TeardownHooksFailure(err_msg)\n\n # others\n else:\n err_msg = u\"Failed to extract attribute from response! => {}\\n\".format(field)\n err_msg += u\"available response attributes: status_code, cookies, elapsed, headers, content, text, json, encoding, ok, reason, url.\\n\\n\"\n err_msg += u\"If you want to set attribute in teardown_hooks, take the following example as reference:\\n\"\n err_msg += u\"response.new_attribute = 'new_attribute_value'\\n\"\n logger.log_error(err_msg)\n raise exceptions.ParamsError(err_msg)"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nextracting value from requests. Response. field", "response": "def extract_field(self, field):\n \"\"\" extract value from requests.Response.\n \"\"\"\n if not isinstance(field, basestring):\n err_msg = u\"Invalid extractor! => {}\\n\".format(field)\n logger.log_error(err_msg)\n raise exceptions.ParamsError(err_msg)\n\n msg = \"extract: {}\".format(field)\n\n if text_extractor_regexp_compile.match(field):\n value = self._extract_field_with_regex(field)\n else:\n value = self._extract_field_with_delimiter(field)\n\n if is_py2 and isinstance(value, unicode):\n value = value.encode(\"utf-8\")\n\n msg += \"\\t=> {}\".format(value)\n logger.log_debug(msg)\n\n return value"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef extract_response(self, extractors):\n if not extractors:\n return {}\n\n logger.log_debug(\"start to extract from response object.\")\n extracted_variables_mapping = OrderedDict()\n extract_binds_order_dict = utils.ensure_mapping_format(extractors)\n\n for key, field in extract_binds_order_dict.items():\n extracted_variables_mapping[key] = self.extract_field(field)\n\n return extracted_variables_mapping", "response": "extract value from requests. Response and store in OrderedDict."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nuses this method to request a callback answer from bots.", "response": "def request_callback_answer(\n self,\n chat_id: Union[int, str],\n message_id: int,\n callback_data: bytes\n ):\n \"\"\"Use this method to request a callback answer from bots.\n This is the equivalent of clicking an inline button containing callback data.\n\n Args:\n chat_id (``int`` | ``str``):\n Unique identifier (int) or username (str) of the target chat.\n For your personal cloud (Saved Messages) you can simply use \"me\" or \"self\".\n For a contact that exists in your Telegram address book you can use his phone number (str).\n\n message_id (``int``):\n The message id the inline keyboard is attached on.\n\n callback_data (``bytes``):\n Callback data associated with the inline button you want to get the answer from.\n\n Returns:\n The answer containing info useful for clients to display a notification at the top of the chat screen\n or as an alert.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n ``TimeoutError`` if the bot fails to answer within 10 seconds.\n \"\"\"\n return self.send(\n functions.messages.GetBotCallbackAnswer(\n peer=self.resolve_peer(chat_id),\n msg_id=message_id,\n data=callback_data\n ),\n retries=0,\n timeout=10\n )"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef promote_chat_member(\n self,\n chat_id: Union[int, str],\n user_id: Union[int, str],\n can_change_info: bool = True,\n can_post_messages: bool = False,\n can_edit_messages: bool = False,\n can_delete_messages: bool = True,\n can_restrict_members: bool = True,\n can_invite_users: bool = True,\n can_pin_messages: bool = False,\n can_promote_members: bool = False\n ) -> bool:\n \"\"\"Use this method to promote or demote a user in a supergroup or a channel.\n\n You must be an administrator in the chat for this to work and must have the appropriate admin rights.\n Pass False for all boolean parameters to demote a user.\n\n Args:\n chat_id (``int`` | ``str``):\n Unique identifier (int) or username (str) of the target chat.\n\n user_id (``int`` | ``str``):\n Unique identifier (int) or username (str) of the target user.\n For a contact that exists in your Telegram address book you can use his phone number (str).\n\n can_change_info (``bool``, *optional*):\n Pass True, if the administrator can change chat title, photo and other settings.\n\n can_post_messages (``bool``, *optional*):\n Pass True, if the administrator can create channel posts, channels only.\n\n can_edit_messages (``bool``, *optional*):\n Pass True, if the administrator can edit messages of other users and can pin messages, channels only.\n\n can_delete_messages (``bool``, *optional*):\n Pass True, if the administrator can delete messages of other users.\n\n can_restrict_members (``bool``, *optional*):\n Pass True, if the administrator can restrict, ban or unban chat members.\n\n can_invite_users (``bool``, *optional*):\n Pass True, if the administrator can invite new users to the chat.\n\n can_pin_messages (``bool``, *optional*):\n Pass True, if the administrator can pin messages, supergroups only.\n\n can_promote_members (``bool``, *optional*):\n Pass True, if the administrator can add new administrators with a subset of his own privileges or\n demote administrators that he has promoted, directly or indirectly (promoted by administrators that\n were appointed by him).\n\n Returns:\n True on success.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n self.send(\n functions.channels.EditAdmin(\n channel=self.resolve_peer(chat_id),\n user_id=self.resolve_peer(user_id),\n admin_rights=types.ChatAdminRights(\n change_info=can_change_info or None,\n post_messages=can_post_messages or None,\n edit_messages=can_edit_messages or None,\n delete_messages=can_delete_messages or None,\n ban_users=can_restrict_members or None,\n invite_users=can_invite_users or None,\n pin_messages=can_pin_messages or None,\n add_admins=can_promote_members or None,\n )\n )\n )\n\n return True", "response": "Promote a user in a chat or a channel."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nuse this method to set a new profile photo on the local machine.", "response": "def set_user_profile_photo(\n self,\n photo: str\n ) -> bool:\n \"\"\"Use this method to set a new profile photo.\n\n This method only works for Users.\n Bots profile photos must be set using BotFather.\n\n Args:\n photo (``str``):\n Profile photo to set.\n Pass a file path as string to upload a new photo that exists on your local machine.\n\n Returns:\n True on success.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n\n return bool(\n self.send(\n functions.photos.UploadProfilePhoto(\n file=self.save_file(photo)\n )\n )\n )"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef create(self):\n retries_left = self.MAX_RETRIES\n\n # The server may close the connection at any time, causing the auth key creation to fail.\n # If that happens, just try again up to MAX_RETRIES times.\n while True:\n self.connection = Connection(self.dc_id, self.test_mode, self.ipv6, self.proxy)\n\n try:\n log.info(\"Start creating a new auth key on DC{}\".format(self.dc_id))\n\n self.connection.connect()\n\n # Step 1; Step 2\n nonce = int.from_bytes(urandom(16), \"little\", signed=True)\n log.debug(\"Send req_pq: {}\".format(nonce))\n res_pq = self.send(functions.ReqPqMulti(nonce=nonce))\n log.debug(\"Got ResPq: {}\".format(res_pq.server_nonce))\n log.debug(\"Server public key fingerprints: {}\".format(res_pq.server_public_key_fingerprints))\n\n for i in res_pq.server_public_key_fingerprints:\n if i in RSA.server_public_keys:\n log.debug(\"Using fingerprint: {}\".format(i))\n public_key_fingerprint = i\n break\n else:\n log.debug(\"Fingerprint unknown: {}\".format(i))\n else:\n raise Exception(\"Public key not found\")\n\n # Step 3\n pq = int.from_bytes(res_pq.pq, \"big\")\n log.debug(\"Start PQ factorization: {}\".format(pq))\n start = time.time()\n g = Prime.decompose(pq)\n p, q = sorted((g, pq // g)) # p < q\n log.debug(\"Done PQ factorization ({}s): {} {}\".format(round(time.time() - start, 3), p, q))\n\n # Step 4\n server_nonce = res_pq.server_nonce\n new_nonce = int.from_bytes(urandom(32), \"little\", signed=True)\n\n data = types.PQInnerData(\n pq=res_pq.pq,\n p=p.to_bytes(4, \"big\"),\n q=q.to_bytes(4, \"big\"),\n nonce=nonce,\n server_nonce=server_nonce,\n new_nonce=new_nonce,\n ).write()\n\n sha = sha1(data).digest()\n padding = urandom(- (len(data) + len(sha)) % 255)\n data_with_hash = sha + data + padding\n encrypted_data = RSA.encrypt(data_with_hash, public_key_fingerprint)\n\n log.debug(\"Done encrypt data with RSA\")\n\n # Step 5. TODO: Handle \"server_DH_params_fail\". Code assumes response is ok\n log.debug(\"Send req_DH_params\")\n server_dh_params = self.send(\n functions.ReqDHParams(\n nonce=nonce,\n server_nonce=server_nonce,\n p=p.to_bytes(4, \"big\"),\n q=q.to_bytes(4, \"big\"),\n public_key_fingerprint=public_key_fingerprint,\n encrypted_data=encrypted_data\n )\n )\n\n encrypted_answer = server_dh_params.encrypted_answer\n\n server_nonce = server_nonce.to_bytes(16, \"little\", signed=True)\n new_nonce = new_nonce.to_bytes(32, \"little\", signed=True)\n\n tmp_aes_key = (\n sha1(new_nonce + server_nonce).digest()\n + sha1(server_nonce + new_nonce).digest()[:12]\n )\n\n tmp_aes_iv = (\n sha1(server_nonce + new_nonce).digest()[12:]\n + sha1(new_nonce + new_nonce).digest() + new_nonce[:4]\n )\n\n server_nonce = int.from_bytes(server_nonce, \"little\", signed=True)\n\n answer_with_hash = AES.ige256_decrypt(encrypted_answer, tmp_aes_key, tmp_aes_iv)\n answer = answer_with_hash[20:]\n\n server_dh_inner_data = Object.read(BytesIO(answer))\n\n log.debug(\"Done decrypting answer\")\n\n dh_prime = int.from_bytes(server_dh_inner_data.dh_prime, \"big\")\n delta_time = server_dh_inner_data.server_time - time.time()\n\n log.debug(\"Delta time: {}\".format(round(delta_time, 3)))\n\n # Step 6\n g = server_dh_inner_data.g\n b = int.from_bytes(urandom(256), \"big\")\n g_b = pow(g, b, dh_prime).to_bytes(256, \"big\")\n\n retry_id = 0\n\n data = types.ClientDHInnerData(\n nonce=nonce,\n server_nonce=server_nonce,\n retry_id=retry_id,\n g_b=g_b\n ).write()\n\n sha = sha1(data).digest()\n padding = urandom(- (len(data) + len(sha)) % 16)\n data_with_hash = sha + data + padding\n encrypted_data = AES.ige256_encrypt(data_with_hash, tmp_aes_key, tmp_aes_iv)\n\n log.debug(\"Send set_client_DH_params\")\n set_client_dh_params_answer = self.send(\n functions.SetClientDHParams(\n nonce=nonce,\n server_nonce=server_nonce,\n encrypted_data=encrypted_data\n )\n )\n\n # TODO: Handle \"auth_key_aux_hash\" if the previous step fails\n\n # Step 7; Step 8\n g_a = int.from_bytes(server_dh_inner_data.g_a, \"big\")\n auth_key = pow(g_a, b, dh_prime).to_bytes(256, \"big\")\n server_nonce = server_nonce.to_bytes(16, \"little\", signed=True)\n\n # TODO: Handle errors\n\n #######################\n # Security checks\n #######################\n\n assert dh_prime == Prime.CURRENT_DH_PRIME\n log.debug(\"DH parameters check: OK\")\n\n # https://core.telegram.org/mtproto/security_guidelines#g-a-and-g-b-validation\n g_b = int.from_bytes(g_b, \"big\")\n assert 1 < g < dh_prime - 1\n assert 1 < g_a < dh_prime - 1\n assert 1 < g_b < dh_prime - 1\n assert 2 ** (2048 - 64) < g_a < dh_prime - 2 ** (2048 - 64)\n assert 2 ** (2048 - 64) < g_b < dh_prime - 2 ** (2048 - 64)\n log.debug(\"g_a and g_b validation: OK\")\n\n # https://core.telegram.org/mtproto/security_guidelines#checking-sha1-hash-values\n answer = server_dh_inner_data.write() # Call .write() to remove padding\n assert answer_with_hash[:20] == sha1(answer).digest()\n log.debug(\"SHA1 hash values check: OK\")\n\n # https://core.telegram.org/mtproto/security_guidelines#checking-nonce-server-nonce-and-new-nonce-fields\n # 1st message\n assert nonce == res_pq.nonce\n # 2nd message\n server_nonce = int.from_bytes(server_nonce, \"little\", signed=True)\n assert nonce == server_dh_params.nonce\n assert server_nonce == server_dh_params.server_nonce\n # 3rd message\n assert nonce == set_client_dh_params_answer.nonce\n assert server_nonce == set_client_dh_params_answer.server_nonce\n server_nonce = server_nonce.to_bytes(16, \"little\", signed=True)\n log.debug(\"Nonce fields check: OK\")\n\n # Step 9\n server_salt = AES.xor(new_nonce[:8], server_nonce[:8])\n\n log.debug(\"Server salt: {}\".format(int.from_bytes(server_salt, \"little\")))\n\n log.info(\n \"Done auth key exchange: {}\".format(\n set_client_dh_params_answer.__class__.__name__\n )\n )\n except Exception as e:\n if retries_left:\n retries_left -= 1\n else:\n raise e\n\n time.sleep(1)\n continue\n else:\n return auth_key\n finally:\n self.connection.close()", "response": "Create an auth key for the current user."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nuse this method to iterate through a chat history sequentially. This convenience method does the same as repeatedly calling :meth:`get_history` in a loop, thus saving you from the hassle of setting up boilerplate code. It is useful for getting the whole chat history with a single call. Args: chat_id (``int`` | ``str``): Unique identifier (int) or username (str) of the target chat. For your personal cloud (Saved Messages) you can simply use \"me\" or \"self\". For a contact that exists in your Telegram address book you can use his phone number (str). limit (``int``, *optional*): Limits the number of messages to be retrieved. By default, no limit is applied and all messages are returned. offset (``int``, *optional*): Sequential number of the first message to be returned.. Negative values are also accepted and become useful in case you set offset_id or offset_date. offset_id (``int``, *optional*): Identifier of the first message to be returned. offset_date (``int``, *optional*): Pass a date in Unix time as offset to retrieve only older messages starting from that date. reverse (``bool``, *optional*): Pass True to retrieve the messages in reversed order (from older to most recent). Returns: A generator yielding :obj:`Message ` objects. Raises: :class:`RPCError ` in case of a Telegram RPC error.", "response": "def iter_history(\n self,\n chat_id: Union[int, str],\n limit: int = 0,\n offset: int = 0,\n offset_id: int = 0,\n offset_date: int = 0,\n reverse: bool = False\n ) -> Generator[\"pyrogram.Message\", None, None]:\n \"\"\"Use this method to iterate through a chat history sequentially.\n\n This convenience method does the same as repeatedly calling :meth:`get_history` in a loop, thus saving you from\n the hassle of setting up boilerplate code. It is useful for getting the whole chat history with a single call.\n\n Args:\n chat_id (``int`` | ``str``):\n Unique identifier (int) or username (str) of the target chat.\n For your personal cloud (Saved Messages) you can simply use \"me\" or \"self\".\n For a contact that exists in your Telegram address book you can use his phone number (str).\n\n limit (``int``, *optional*):\n Limits the number of messages to be retrieved.\n By default, no limit is applied and all messages are returned.\n\n offset (``int``, *optional*):\n Sequential number of the first message to be returned..\n Negative values are also accepted and become useful in case you set offset_id or offset_date.\n\n offset_id (``int``, *optional*):\n Identifier of the first message to be returned.\n\n offset_date (``int``, *optional*):\n Pass a date in Unix time as offset to retrieve only older messages starting from that date.\n\n reverse (``bool``, *optional*):\n Pass True to retrieve the messages in reversed order (from older to most recent).\n\n Returns:\n A generator yielding :obj:`Message ` objects.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n offset_id = offset_id or (1 if reverse else 0)\n current = 0\n total = limit or (1 << 31) - 1\n limit = min(100, total)\n\n while True:\n messages = self.get_history(\n chat_id=chat_id,\n limit=limit,\n offset=offset,\n offset_id=offset_id,\n offset_date=offset_date,\n reverse=reverse\n ).messages\n\n if not messages:\n return\n\n offset_id = messages[-1].message_id + (1 if reverse else 0)\n\n for message in messages:\n yield message\n\n current += 1\n\n if current >= total:\n return"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nuse this method to send phone contacts.", "response": "def send_contact(\n self,\n chat_id: Union[int, str],\n phone_number: str,\n first_name: str,\n last_name: str = None,\n vcard: str = None,\n disable_notification: bool = None,\n reply_to_message_id: int = None,\n reply_markup: Union[\n \"pyrogram.InlineKeyboardMarkup\",\n \"pyrogram.ReplyKeyboardMarkup\",\n \"pyrogram.ReplyKeyboardRemove\",\n \"pyrogram.ForceReply\"\n ] = None\n ) -> \"pyrogram.Message\":\n \"\"\"Use this method to send phone contacts.\n\n Args:\n chat_id (``int`` | ``str``):\n Unique identifier (int) or username (str) of the target chat.\n For your personal cloud (Saved Messages) you can simply use \"me\" or \"self\".\n For a contact that exists in your Telegram address book you can use his phone number (str).\n\n phone_number (``str``):\n Contact's phone number.\n\n first_name (``str``):\n Contact's first name.\n\n last_name (``str``, *optional*):\n Contact's last name.\n\n vcard (``str``, *optional*):\n Additional data about the contact in the form of a vCard, 0-2048 bytes\n\n disable_notification (``bool``, *optional*):\n Sends the message silently.\n Users will receive a notification with no sound.\n\n reply_to_message_id (``int``, *optional*):\n If the message is a reply, ID of the original message.\n\n reply_markup (:obj:`InlineKeyboardMarkup` | :obj:`ReplyKeyboardMarkup` | :obj:`ReplyKeyboardRemove` | :obj:`ForceReply`, *optional*):\n Additional interface options. An object for an inline keyboard, custom reply keyboard,\n instructions to remove reply keyboard or to force a reply from the user.\n\n Returns:\n On success, the sent :obj:`Message ` is returned.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n r = self.send(\n functions.messages.SendMedia(\n peer=self.resolve_peer(chat_id),\n media=types.InputMediaContact(\n phone_number=phone_number,\n first_name=first_name,\n last_name=last_name or \"\",\n vcard=vcard or \"\"\n ),\n message=\"\",\n silent=disable_notification or None,\n reply_to_msg_id=reply_to_message_id,\n random_id=self.rnd_id(),\n reply_markup=reply_markup.write() if reply_markup else None\n )\n )\n\n for i in r.updates:\n if isinstance(i, (types.UpdateNewMessage, types.UpdateNewChannelMessage)):\n return pyrogram.Message._parse(\n self, i.message,\n {i.id: i for i in r.users},\n {i.id: i for i in r.chats}\n )"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef edit_message_text(\n self,\n chat_id: Union[int, str],\n message_id: int,\n text: str,\n parse_mode: str = \"\",\n disable_web_page_preview: bool = None,\n reply_markup: \"pyrogram.InlineKeyboardMarkup\" = None\n ) -> \"pyrogram.Message\":\n \"\"\"Use this method to edit text messages.\n\n Args:\n chat_id (``int`` | ``str``):\n Unique identifier (int) or username (str) of the target chat.\n For your personal cloud (Saved Messages) you can simply use \"me\" or \"self\".\n For a contact that exists in your Telegram address book you can use his phone number (str).\n\n message_id (``int``):\n Message identifier in the chat specified in chat_id.\n\n text (``str``):\n New text of the message.\n\n parse_mode (``str``, *optional*):\n Use :obj:`MARKDOWN ` or :obj:`HTML `\n if you want Telegram apps to show bold, italic, fixed-width text or inline URLs in your message.\n Defaults to Markdown.\n\n disable_web_page_preview (``bool``, *optional*):\n Disables link previews for links in this message.\n\n reply_markup (:obj:`InlineKeyboardMarkup`, *optional*):\n An InlineKeyboardMarkup object.\n\n Returns:\n On success, the edited :obj:`Message ` is returned.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n style = self.html if parse_mode.lower() == \"html\" else self.markdown\n\n r = self.send(\n functions.messages.EditMessage(\n peer=self.resolve_peer(chat_id),\n id=message_id,\n no_webpage=disable_web_page_preview or None,\n reply_markup=reply_markup.write() if reply_markup else None,\n **style.parse(text)\n )\n )\n\n for i in r.updates:\n if isinstance(i, (types.UpdateEditMessage, types.UpdateEditChannelMessage)):\n return pyrogram.Message._parse(\n self, i.message,\n {i.id: i for i in r.users},\n {i.id: i for i in r.chats}\n )", "response": "Use this method to edit text messages in Telegram."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nusing this method to retrieve a chunk of the history of a chat. You can get up to 100 messages at once. For a more convenient way of getting a chat history see :meth:`iter_history`. Args: chat_id (``int`` | ``str``): Unique identifier (int) or username (str) of the target chat. For your personal cloud (Saved Messages) you can simply use \"me\" or \"self\". For a contact that exists in your Telegram address book you can use his phone number (str). limit (``int``, *optional*): Limits the number of messages to be retrieved. By default, the first 100 messages are returned. offset (``int``, *optional*): Sequential number of the first message to be returned. Defaults to 0 (most recent message). Negative values are also accepted and become useful in case you set offset_id or offset_date. offset_id (``int``, *optional*): Pass a message identifier as offset to retrieve only older messages starting from that message. offset_date (``int``, *optional*): Pass a date in Unix time as offset to retrieve only older messages starting from that date. reverse (``bool``, *optional*): Pass True to retrieve the messages in reversed order (from older to most recent). Returns: On success, a :obj:`Messages ` object is returned. Raises: :class:`RPCError ` in case of a Telegram RPC error.", "response": "def get_history(\n self,\n chat_id: Union[int, str],\n limit: int = 100,\n offset: int = 0,\n offset_id: int = 0,\n offset_date: int = 0,\n reverse: bool = False\n ):\n \"\"\"Use this method to retrieve a chunk of the history of a chat.\n\n You can get up to 100 messages at once.\n For a more convenient way of getting a chat history see :meth:`iter_history`.\n\n Args:\n chat_id (``int`` | ``str``):\n Unique identifier (int) or username (str) of the target chat.\n For your personal cloud (Saved Messages) you can simply use \"me\" or \"self\".\n For a contact that exists in your Telegram address book you can use his phone number (str).\n\n limit (``int``, *optional*):\n Limits the number of messages to be retrieved.\n By default, the first 100 messages are returned.\n\n offset (``int``, *optional*):\n Sequential number of the first message to be returned. Defaults to 0 (most recent message).\n Negative values are also accepted and become useful in case you set offset_id or offset_date.\n\n offset_id (``int``, *optional*):\n Pass a message identifier as offset to retrieve only older messages starting from that message.\n\n offset_date (``int``, *optional*):\n Pass a date in Unix time as offset to retrieve only older messages starting from that date.\n\n reverse (``bool``, *optional*):\n Pass True to retrieve the messages in reversed order (from older to most recent).\n\n Returns:\n On success, a :obj:`Messages ` object is returned.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n\n while True:\n try:\n messages = pyrogram.Messages._parse(\n self,\n self.send(\n functions.messages.GetHistory(\n peer=self.resolve_peer(chat_id),\n offset_id=offset_id,\n offset_date=offset_date,\n add_offset=offset * (-1 if reverse else 1) - (limit if reverse else 0),\n limit=limit,\n max_id=0,\n min_id=0,\n hash=0\n )\n )\n )\n except FloodWait as e:\n log.warning(\"Sleeping for {}s\".format(e.x))\n time.sleep(e.x)\n else:\n break\n\n if reverse:\n messages.messages.reverse()\n\n return messages"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef on_user_status(\n self=None,\n filters=None,\n group: int = 0\n ) -> callable:\n \"\"\"Use this decorator to automatically register a function for handling user status updates.\n This does the same thing as :meth:`add_handler` using the :class:`UserStatusHandler`.\n\n Args:\n filters (:obj:`Filters `):\n Pass one or more filters to allow only a subset of UserStatus updated to be passed in your function.\n\n group (``int``, *optional*):\n The group identifier, defaults to 0.\n \"\"\"\n\n def decorator(func: callable) -> Tuple[Handler, int]:\n if isinstance(func, tuple):\n func = func[0].callback\n\n handler = pyrogram.UserStatusHandler(func, filters)\n\n if isinstance(self, Filter):\n return pyrogram.UserStatusHandler(func, self), group if filters is None else filters\n\n if self is not None:\n self.add_handler(handler, group)\n\n return handler, group\n\n return decorator", "response": "Decorator to automatically register a function for handling user status updates."} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nusing this method to get the number of members in a chat.", "response": "def get_chat_members_count(\n self,\n chat_id: Union[int, str]\n ) -> int:\n \"\"\"Use this method to get the number of members in a chat.\n\n Args:\n chat_id (``int`` | ``str``):\n Unique identifier (int) or username (str) of the target chat.\n\n Returns:\n On success, an integer is returned.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n ``ValueError`` if a chat_id belongs to user.\n \"\"\"\n peer = self.resolve_peer(chat_id)\n\n if isinstance(peer, types.InputPeerChat):\n return self.send(\n functions.messages.GetChats(\n id=[peer.chat_id]\n )\n ).chats[0].participants_count\n elif isinstance(peer, types.InputPeerChannel):\n return self.send(\n functions.channels.GetFullChannel(\n channel=peer\n )\n ).full_chat.participants_count\n else:\n raise ValueError(\"The chat_id \\\"{}\\\" belongs to a user\".format(chat_id))"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nuses this decorator to automatically register a function for handling disconnections.", "response": "def on_disconnect(self=None) -> callable:\n \"\"\"Use this decorator to automatically register a function for handling disconnections.\n This does the same thing as :meth:`add_handler` using the :class:`DisconnectHandler`.\n \"\"\"\n\n def decorator(func: callable) -> Handler:\n handler = pyrogram.DisconnectHandler(func)\n\n if self is not None:\n self.add_handler(handler)\n\n return handler\n\n return decorator"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef start(self):\n if self.is_started:\n raise ConnectionError(\"Client has already been started\")\n\n if self.BOT_TOKEN_RE.match(self.session_name):\n self.is_bot = True\n self.bot_token = self.session_name\n self.session_name = self.session_name.split(\":\")[0]\n log.warning('\\nWARNING: You are using a bot token as session name!\\n'\n 'This usage will be deprecated soon. Please use a session file name to load '\n 'an existing session and the bot_token argument to create new sessions.\\n'\n 'More info: https://docs.pyrogram.ml/start/Setup#bot-authorization\\n')\n\n self.load_config()\n self.load_session()\n self.load_plugins()\n\n self.session = Session(\n self,\n self.dc_id,\n self.auth_key\n )\n\n self.session.start()\n self.is_started = True\n\n try:\n if self.user_id is None:\n if self.bot_token is None:\n self.is_bot = False\n self.authorize_user()\n else:\n self.is_bot = True\n self.authorize_bot()\n\n self.save_session()\n\n if not self.is_bot:\n if self.takeout:\n self.takeout_id = self.send(functions.account.InitTakeoutSession()).id\n log.warning(\"Takeout session {} initiated\".format(self.takeout_id))\n\n now = time.time()\n\n if abs(now - self.date) > Client.OFFLINE_SLEEP:\n self.peers_by_username = {}\n self.peers_by_phone = {}\n\n self.get_initial_dialogs()\n self.get_contacts()\n else:\n self.send(functions.messages.GetPinnedDialogs())\n self.get_initial_dialogs_chunk()\n else:\n self.send(functions.updates.GetState())\n except Exception as e:\n self.is_started = False\n self.session.stop()\n raise e\n\n for i in range(self.UPDATES_WORKERS):\n self.updates_workers_list.append(\n Thread(\n target=self.updates_worker,\n name=\"UpdatesWorker#{}\".format(i + 1)\n )\n )\n\n self.updates_workers_list[-1].start()\n\n for i in range(self.DOWNLOAD_WORKERS):\n self.download_workers_list.append(\n Thread(\n target=self.download_worker,\n name=\"DownloadWorker#{}\".format(i + 1)\n )\n )\n\n self.download_workers_list[-1].start()\n\n self.dispatcher.start()\n\n mimetypes.init()\n Syncer.add(self)\n\n return self", "response": "Use this method to start the Client after creating it.\n Requires no parameters.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n ``ConnectionError`` in case you try to start an already started Client."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef stop(self):\n if not self.is_started:\n raise ConnectionError(\"Client is already stopped\")\n\n if self.takeout_id:\n self.send(functions.account.FinishTakeoutSession())\n log.warning(\"Takeout session {} finished\".format(self.takeout_id))\n\n Syncer.remove(self)\n self.dispatcher.stop()\n\n for _ in range(self.DOWNLOAD_WORKERS):\n self.download_queue.put(None)\n\n for i in self.download_workers_list:\n i.join()\n\n self.download_workers_list.clear()\n\n for _ in range(self.UPDATES_WORKERS):\n self.updates_queue.put(None)\n\n for i in self.updates_workers_list:\n i.join()\n\n self.updates_workers_list.clear()\n\n for i in self.media_sessions.values():\n i.stop()\n\n self.media_sessions.clear()\n\n self.is_started = False\n self.session.stop()\n\n return self", "response": "Stop the current takeout session."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef idle(self, stop_signals: tuple = (SIGINT, SIGTERM, SIGABRT)):\n\n def signal_handler(*args):\n self.is_idle = False\n\n for s in stop_signals:\n signal(s, signal_handler)\n\n self.is_idle = True\n\n while self.is_idle:\n time.sleep(1)\n\n self.stop()", "response": "Blocks the program execution until one of the signals are received."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef add_handler(self, handler: Handler, group: int = 0):\n if isinstance(handler, DisconnectHandler):\n self.disconnect_handler = handler.callback\n else:\n self.dispatcher.add_handler(handler, group)\n\n return handler, group", "response": "Register a new handler to the dispatcher."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nremoving a previously added update handler.", "response": "def remove_handler(self, handler: Handler, group: int = 0):\n \"\"\"Removes a previously-added update handler.\n\n Make sure to provide the right group that the handler was added in. You can use\n the return value of the :meth:`add_handler` method, a tuple of (handler, group), and\n pass it directly.\n\n Args:\n handler (``Handler``):\n The handler to be removed.\n\n group (``int``, *optional*):\n The group identifier, defaults to 0.\n \"\"\"\n if isinstance(handler, DisconnectHandler):\n self.disconnect_handler = None\n else:\n self.dispatcher.remove_handler(handler, group)"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef send(self,\n data: Object,\n retries: int = Session.MAX_RETRIES,\n timeout: float = Session.WAIT_TIMEOUT):\n \"\"\"Use this method to send Raw Function queries.\n\n This method makes possible to manually call every single Telegram API method in a low-level manner.\n Available functions are listed in the :obj:`functions ` package and may accept compound\n data types from :obj:`types ` as well as bare types such as ``int``, ``str``, etc...\n\n Args:\n data (``Object``):\n The API Schema function filled with proper arguments.\n\n retries (``int``):\n Number of retries.\n\n timeout (``float``):\n Timeout in seconds.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n if not self.is_started:\n raise ConnectionError(\"Client has not been started\")\n\n if self.no_updates:\n data = functions.InvokeWithoutUpdates(query=data)\n\n if self.takeout_id:\n data = functions.InvokeWithTakeout(takeout_id=self.takeout_id, query=data)\n\n r = self.session.send(data, retries, timeout)\n\n self.fetch_peers(getattr(r, \"users\", []))\n self.fetch_peers(getattr(r, \"chats\", []))\n\n return r", "response": "Send a raw function to the API."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nuse this method to get the InputPeer of a known peer_id. This is a utility method intended to be used **only** when working with Raw Functions (i.e: a Telegram API method you wish to use which is not available yet in the Client class as an easy-to-use method), whenever an InputPeer type is required. Args: peer_id (``int`` | ``str``): The peer id you want to extract the InputPeer from. Can be a direct id (int), a username (str) or a phone number (str). Returns: On success, the resolved peer id is returned in form of an InputPeer object. Raises: :class:`RPCError ` in case of a Telegram RPC error. ``KeyError`` in case the peer doesn't exist in the internal database.", "response": "def resolve_peer(self,\n peer_id: Union[int, str]):\n \"\"\"Use this method to get the InputPeer of a known peer_id.\n\n This is a utility method intended to be used **only** when working with Raw Functions (i.e: a Telegram API\n method you wish to use which is not available yet in the Client class as an easy-to-use method), whenever an\n InputPeer type is required.\n\n Args:\n peer_id (``int`` | ``str``):\n The peer id you want to extract the InputPeer from.\n Can be a direct id (int), a username (str) or a phone number (str).\n\n Returns:\n On success, the resolved peer id is returned in form of an InputPeer object.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n ``KeyError`` in case the peer doesn't exist in the internal database.\n \"\"\"\n try:\n return self.peers_by_id[peer_id]\n except KeyError:\n if type(peer_id) is str:\n if peer_id in (\"self\", \"me\"):\n return types.InputPeerSelf()\n\n peer_id = re.sub(r\"[@+\\s]\", \"\", peer_id.lower())\n\n try:\n int(peer_id)\n except ValueError:\n if peer_id not in self.peers_by_username:\n self.send(\n functions.contacts.ResolveUsername(\n username=peer_id\n )\n )\n\n return self.peers_by_username[peer_id]\n else:\n try:\n return self.peers_by_phone[peer_id]\n except KeyError:\n raise PeerIdInvalid\n\n if peer_id > 0:\n self.fetch_peers(\n self.send(\n functions.users.GetUsers(\n id=[types.InputUser(user_id=peer_id, access_hash=0)]\n )\n )\n )\n else:\n if str(peer_id).startswith(\"-100\"):\n self.send(\n functions.channels.GetChannels(\n id=[types.InputChannel(channel_id=int(str(peer_id)[4:]), access_hash=0)]\n )\n )\n else:\n self.send(\n functions.messages.GetChats(\n id=[-peer_id]\n )\n )\n\n try:\n return self.peers_by_id[peer_id]\n except KeyError:\n raise PeerIdInvalid"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nusing this method to upload a file onto Telegram servers, without actually sending the message to anyone. This is a utility method intended to be used **only** when working with Raw Functions (i.e: a Telegram API method you wish to use which is not available yet in the Client class as an easy-to-use method), whenever an InputFile type is required. Args: path (``str``): The path of the file you want to upload that exists on your local machine. file_id (``int``, *optional*): In case a file part expired, pass the file_id and the file_part to retry uploading that specific chunk. file_part (``int``, *optional*): In case a file part expired, pass the file_id and the file_part to retry uploading that specific chunk. progress (``callable``, *optional*): Pass a callback function to view the upload progress. The function must take *(client, current, total, \\*args)* as positional arguments (look at the section below for a detailed description). progress_args (``tuple``, *optional*): Extra custom arguments for the progress callback function. Useful, for example, if you want to pass a chat_id and a message_id in order to edit a message with the updated progress. Other Parameters: client (:obj:`Client `): The Client itself, useful when you want to call other API methods inside the callback function. current (``int``): The amount of bytes uploaded so far. total (``int``): The size of the file. *args (``tuple``, *optional*): Extra custom arguments as defined in the *progress_args* parameter. You can either keep *\\*args* or add every single extra argument in your function signature. Returns: On success, the uploaded file is returned in form of an InputFile object. Raises: :class:`RPCError ` in case of a Telegram RPC error.", "response": "def save_file(self,\n path: str,\n file_id: int = None,\n file_part: int = 0,\n progress: callable = None,\n progress_args: tuple = ()):\n \"\"\"Use this method to upload a file onto Telegram servers, without actually sending the message to anyone.\n\n This is a utility method intended to be used **only** when working with Raw Functions (i.e: a Telegram API\n method you wish to use which is not available yet in the Client class as an easy-to-use method), whenever an\n InputFile type is required.\n\n Args:\n path (``str``):\n The path of the file you want to upload that exists on your local machine.\n\n file_id (``int``, *optional*):\n In case a file part expired, pass the file_id and the file_part to retry uploading that specific chunk.\n\n file_part (``int``, *optional*):\n In case a file part expired, pass the file_id and the file_part to retry uploading that specific chunk.\n\n progress (``callable``, *optional*):\n Pass a callback function to view the upload progress.\n The function must take *(client, current, total, \\*args)* as positional arguments (look at the section\n below for a detailed description).\n\n progress_args (``tuple``, *optional*):\n Extra custom arguments for the progress callback function. Useful, for example, if you want to pass\n a chat_id and a message_id in order to edit a message with the updated progress.\n\n Other Parameters:\n client (:obj:`Client `):\n The Client itself, useful when you want to call other API methods inside the callback function.\n\n current (``int``):\n The amount of bytes uploaded so far.\n\n total (``int``):\n The size of the file.\n\n *args (``tuple``, *optional*):\n Extra custom arguments as defined in the *progress_args* parameter.\n You can either keep *\\*args* or add every single extra argument in your function signature.\n\n Returns:\n On success, the uploaded file is returned in form of an InputFile object.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n part_size = 512 * 1024\n file_size = os.path.getsize(path)\n\n if file_size == 0:\n raise ValueError(\"File size equals to 0 B\")\n\n if file_size > 1500 * 1024 * 1024:\n raise ValueError(\"Telegram doesn't support uploading files bigger than 1500 MiB\")\n\n file_total_parts = int(math.ceil(file_size / part_size))\n is_big = True if file_size > 10 * 1024 * 1024 else False\n is_missing_part = True if file_id is not None else False\n file_id = file_id or self.rnd_id()\n md5_sum = md5() if not is_big and not is_missing_part else None\n\n session = Session(self, self.dc_id, self.auth_key, is_media=True)\n session.start()\n\n try:\n with open(path, \"rb\") as f:\n f.seek(part_size * file_part)\n\n while True:\n chunk = f.read(part_size)\n\n if not chunk:\n if not is_big:\n md5_sum = \"\".join([hex(i)[2:].zfill(2) for i in md5_sum.digest()])\n break\n\n for _ in range(3):\n if is_big:\n rpc = functions.upload.SaveBigFilePart(\n file_id=file_id,\n file_part=file_part,\n file_total_parts=file_total_parts,\n bytes=chunk\n )\n else:\n rpc = functions.upload.SaveFilePart(\n file_id=file_id,\n file_part=file_part,\n bytes=chunk\n )\n\n if session.send(rpc):\n break\n else:\n raise AssertionError(\"Telegram didn't accept chunk #{} of {}\".format(file_part, path))\n\n if is_missing_part:\n return\n\n if not is_big:\n md5_sum.update(chunk)\n\n file_part += 1\n\n if progress:\n progress(self, min(file_part * part_size, file_size), file_size, *progress_args)\n except Client.StopTransmission:\n raise\n except Exception as e:\n log.error(e, exc_info=True)\n else:\n if is_big:\n return types.InputFileBig(\n id=file_id,\n parts=file_total_parts,\n name=os.path.basename(path),\n\n )\n else:\n return types.InputFile(\n id=file_id,\n parts=file_total_parts,\n name=os.path.basename(path),\n md5_checksum=md5_sum\n )\n finally:\n session.stop()"} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nsends a chat action to the other party.", "response": "def send_chat_action(\n self,\n chat_id: Union[int, str],\n action: Union[ChatAction, str],\n progress: int = 0\n ):\n \"\"\"Use this method when you need to tell the other party that something is happening on your side.\n\n Args:\n chat_id (``int`` | ``str``):\n Unique identifier (int) or username (str) of the target chat.\n For your personal cloud (Saved Messages) you can simply use \"me\" or \"self\".\n For a contact that exists in your Telegram address book you can use his phone number (str).\n\n action (:obj:`ChatAction ` | ``str``):\n Type of action to broadcast.\n Choose one from the :class:`ChatAction ` enumeration,\n depending on what the user is about to receive.\n You can also provide a string (e.g. \"typing\", \"upload_photo\", \"record_audio\", ...).\n\n progress (``int``, *optional*):\n Progress of the upload process.\n Currently useless because official clients don't seem to be handling this.\n\n Returns:\n On success, True is returned.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n ``ValueError`` if the provided string is not a valid ChatAction.\n \"\"\"\n\n # Resolve Enum type\n if isinstance(action, str):\n action = ChatAction.from_string(action).value\n elif isinstance(action, ChatAction):\n action = action.value\n\n if \"Upload\" in action.__name__:\n action = action(progress=progress)\n else:\n action = action()\n\n return self.send(\n functions.messages.SetTyping(\n peer=self.resolve_peer(chat_id),\n action=action\n )\n )"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef answer_inline_query(\n self,\n inline_query_id: str,\n results: List[InlineQueryResult],\n cache_time: int = 300,\n is_personal: bool = None,\n next_offset: str = \"\",\n switch_pm_text: str = \"\",\n switch_pm_parameter: str = \"\"\n ):\n \"\"\"Use this method to send answers to an inline query.\n No more than 50 results per query are allowed.\n\n Args:\n inline_query_id (``str``):\n Unique identifier for the answered query.\n\n results (List of :obj:`InlineQueryResult `):\n A list of results for the inline query.\n\n cache_time (``int``, *optional*):\n The maximum amount of time in seconds that the result of the inline query may be cached on the server.\n Defaults to 300.\n\n is_personal (``bool``, *optional*):\n Pass True, if results may be cached on the server side only for the user that sent the query.\n By default, results may be returned to any user who sends the same query.\n\n next_offset (``str``, *optional*):\n Pass the offset that a client should send in the next query with the same text to receive more results.\n Pass an empty string if there are no more results or if you don\u2018t support pagination.\n Offset length can\u2019t exceed 64 bytes.\n\n switch_pm_text (``str``, *optional*):\n If passed, clients will display a button with specified text that switches the user to a private chat\n with the bot and sends the bot a start message with the parameter switch_pm_parameter\n\n switch_pm_parameter (``str``, *optional*):\n `Deep-linking `_ parameter for the /start message sent to\n the bot when user presses the switch button. 1-64 characters, only A-Z, a-z, 0-9, _ and - are allowed.\n\n Example: An inline bot that sends YouTube videos can ask the user to connect the bot to their YouTube\n account to adapt search results accordingly. To do this, it displays a \"Connect your YouTube account\"\n button above the results, or even before showing any. The user presses the button, switches to a private\n chat with the bot and, in doing so, passes a start parameter that instructs the bot to return an oauth\n link. Once done, the bot can offer a switch_inline button so that the user can easily return to the chat\n where they wanted to use the bot's inline capabilities.\n\n Returns:\n On success, True is returned.\n \"\"\"\n return self.send(\n functions.messages.SetInlineBotResults(\n query_id=int(inline_query_id),\n results=[r.write() for r in results],\n cache_time=cache_time,\n gallery=None,\n private=is_personal or None,\n next_offset=next_offset or None,\n switch_pm=types.InlineBotSwitchPM(\n text=switch_pm_text,\n start_param=switch_pm_parameter\n ) if switch_pm_text else None\n )\n )", "response": "This method sends an answer to an inline query."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nsending a location to the user.", "response": "def send_location(\n self,\n chat_id: Union[int, str],\n latitude: float,\n longitude: float,\n disable_notification: bool = None,\n reply_to_message_id: int = None,\n reply_markup: Union[\n \"pyrogram.InlineKeyboardMarkup\",\n \"pyrogram.ReplyKeyboardMarkup\",\n \"pyrogram.ReplyKeyboardRemove\",\n \"pyrogram.ForceReply\"\n ] = None\n ) -> \"pyrogram.Message\":\n \"\"\"Use this method to send points on the map.\n\n Args:\n chat_id (``int`` | ``str``):\n Unique identifier (int) or username (str) of the target chat.\n For your personal cloud (Saved Messages) you can simply use \"me\" or \"self\".\n For a contact that exists in your Telegram address book you can use his phone number (str).\n\n latitude (``float``):\n Latitude of the location.\n\n longitude (``float``):\n Longitude of the location.\n\n disable_notification (``bool``, *optional*):\n Sends the message silently.\n Users will receive a notification with no sound.\n\n reply_to_message_id (``int``, *optional*):\n If the message is a reply, ID of the original message\n\n reply_markup (:obj:`InlineKeyboardMarkup` | :obj:`ReplyKeyboardMarkup` | :obj:`ReplyKeyboardRemove` | :obj:`ForceReply`, *optional*):\n Additional interface options. An object for an inline keyboard, custom reply keyboard,\n instructions to remove reply keyboard or to force a reply from the user.\n\n Returns:\n On success, the sent :obj:`Message ` is returned.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n r = self.send(\n functions.messages.SendMedia(\n peer=self.resolve_peer(chat_id),\n media=types.InputMediaGeoPoint(\n geo_point=types.InputGeoPoint(\n lat=latitude,\n long=longitude\n )\n ),\n message=\"\",\n silent=disable_notification or None,\n reply_to_msg_id=reply_to_message_id,\n random_id=self.rnd_id(),\n reply_markup=reply_markup.write() if reply_markup else None\n )\n )\n\n for i in r.updates:\n if isinstance(i, (types.UpdateNewMessage, types.UpdateNewChannelMessage)):\n return pyrogram.Message._parse(\n self, i.message,\n {i.id: i for i in r.users},\n {i.id: i for i in r.chats}\n )"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef retract_vote(\n self,\n chat_id: Union[int, str],\n message_id: id\n ) -> bool:\n \"\"\"Use this method to retract your vote in a poll.\n\n Args:\n chat_id (``int`` | ``str``):\n Unique identifier (int) or username (str) of the target chat.\n For your personal cloud (Saved Messages) you can simply use \"me\" or \"self\".\n For a contact that exists in your Telegram address book you can use his phone number (str).\n\n message_id (``int``):\n Unique poll message identifier inside this chat.\n\n Returns:\n On success, True is returned.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n self.send(\n functions.messages.SendVote(\n peer=self.resolve_peer(chat_id),\n msg_id=message_id,\n options=[]\n )\n )\n\n return True", "response": "Use this method to retract a vote in a poll."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nusing this method to get a list of profile pictures for a user.", "response": "def get_user_profile_photos(\n self,\n user_id: Union[int, str],\n offset: int = 0,\n limit: int = 100\n ) -> \"pyrogram.UserProfilePhotos\":\n \"\"\"Use this method to get a list of profile pictures for a user.\n\n Args:\n user_id (``int`` | ``str``):\n Unique identifier (int) or username (str) of the target chat.\n For your personal cloud (Saved Messages) you can simply use \"me\" or \"self\".\n For a contact that exists in your Telegram address book you can use his phone number (str).\n\n offset (``int``, *optional*):\n Sequential number of the first photo to be returned.\n By default, all photos are returned.\n\n limit (``int``, *optional*):\n Limits the number of photos to be retrieved.\n Values between 1\u2014100 are accepted. Defaults to 100.\n\n Returns:\n On success, a :obj:`UserProfilePhotos` object is returned.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n return pyrogram.UserProfilePhotos._parse(\n self,\n self.send(\n functions.photos.GetUserPhotos(\n user_id=self.resolve_peer(user_id),\n offset=offset,\n max_id=0,\n limit=limit\n )\n )\n )"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nusing this method to update the username of a user in the chat.", "response": "def update_username(\n self,\n username: Union[str, None]\n ) -> bool:\n \"\"\"Use this method to update your own username.\n \n This method only works for users, not bots. Bot usernames must be changed via Bot Support or by recreating\n them from scratch using BotFather. To update a channel or supergroup username you can use\n :meth:`update_chat_username`.\n\n Args:\n username (``str`` | ``None``):\n Username to set. \"\" (empty string) or None to remove the username.\n\n Returns:\n True on success.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n\n return bool(\n self.send(\n functions.account.UpdateUsername(\n username=username or \"\"\n )\n )\n )"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef reply(\n self,\n text: str,\n quote: bool = None,\n parse_mode: str = \"\",\n disable_web_page_preview: bool = None,\n disable_notification: bool = None,\n reply_to_message_id: int = None,\n reply_markup=None\n ) -> \"Message\":\n \"\"\"Bound method *reply* of :obj:`Message `.\n\n Use as a shortcut for:\n\n .. code-block:: python\n\n client.send_message(\n chat_id=message.chat.id,\n text=\"hello\",\n reply_to_message_id=message.message_id\n )\n\n Example:\n .. code-block:: python\n\n message.reply(\"hello\", quote=True)\n\n Args:\n text (``str``):\n Text of the message to be sent.\n\n quote (``bool``, *optional*):\n If ``True``, the message will be sent as a reply to this message.\n If *reply_to_message_id* is passed, this parameter will be ignored.\n Defaults to ``True`` in group chats and ``False`` in private chats.\n\n parse_mode (``str``, *optional*):\n Use :obj:`MARKDOWN ` or :obj:`HTML `\n if you want Telegram apps to show bold, italic, fixed-width text or inline URLs in your message.\n Defaults to Markdown.\n\n disable_web_page_preview (``bool``, *optional*):\n Disables link previews for links in this message.\n\n disable_notification (``bool``, *optional*):\n Sends the message silently.\n Users will receive a notification with no sound.\n\n reply_to_message_id (``int``, *optional*):\n If the message is a reply, ID of the original message.\n\n reply_markup (:obj:`InlineKeyboardMarkup` | :obj:`ReplyKeyboardMarkup` | :obj:`ReplyKeyboardRemove` | :obj:`ForceReply`, *optional*):\n Additional interface options. An object for an inline keyboard, custom reply keyboard,\n instructions to remove reply keyboard or to force a reply from the user.\n\n Returns:\n On success, the sent Message is returned.\n\n Raises:\n :class:`RPCError `\n \"\"\"\n if quote is None:\n quote = self.chat.type != \"private\"\n\n if reply_to_message_id is None and quote:\n reply_to_message_id = self.message_id\n\n return self._client.send_message(\n chat_id=self.chat.id,\n text=text,\n parse_mode=parse_mode,\n disable_web_page_preview=disable_web_page_preview,\n disable_notification=disable_notification,\n reply_to_message_id=reply_to_message_id,\n reply_markup=reply_markup\n )", "response": "Reply to the specified message."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef reply_animation(\n self,\n animation: str,\n quote: bool = None,\n caption: str = \"\",\n parse_mode: str = \"\",\n duration: int = 0,\n width: int = 0,\n height: int = 0,\n thumb: str = None,\n disable_notification: bool = None,\n reply_markup: Union[\n \"pyrogram.InlineKeyboardMarkup\",\n \"pyrogram.ReplyKeyboardMarkup\",\n \"pyrogram.ReplyKeyboardRemove\",\n \"pyrogram.ForceReply\"\n ] = None,\n reply_to_message_id: int = None,\n progress: callable = None,\n progress_args: tuple = ()\n ) -> \"Message\":\n \"\"\"Bound method *reply_animation* of :obj:`Message `.\n\n Use as a shortcut for:\n\n .. code-block:: python\n\n client.send_animation(\n chat_id=message.chat.id,\n animation=animation\n )\n\n Example:\n .. code-block:: python\n\n message.reply_animation(animation)\n\n Args:\n animation (``str``):\n Animation to send.\n Pass a file_id as string to send an animation that exists on the Telegram servers,\n pass an HTTP URL as a string for Telegram to get an animation from the Internet, or\n pass a file path as string to upload a new animation that exists on your local machine.\n\n quote (``bool``, *optional*):\n If ``True``, the message will be sent as a reply to this message.\n If *reply_to_message_id* is passed, this parameter will be ignored.\n Defaults to ``True`` in group chats and ``False`` in private chats.\n\n caption (``str``, *optional*):\n Animation caption, 0-1024 characters.\n\n parse_mode (``str``, *optional*):\n Use :obj:`MARKDOWN ` or :obj:`HTML `\n if you want Telegram apps to show bold, italic, fixed-width text or inline URLs in your caption.\n Defaults to Markdown.\n\n duration (``int``, *optional*):\n Duration of sent animation in seconds.\n\n width (``int``, *optional*):\n Animation width.\n\n height (``int``, *optional*):\n Animation height.\n\n thumb (``str``, *optional*):\n Thumbnail of the animation file sent.\n The thumbnail should be in JPEG format and less than 200 KB in size.\n A thumbnail's width and height should not exceed 90 pixels.\n Thumbnails can't be reused and can be only uploaded as a new file.\n\n disable_notification (``bool``, *optional*):\n Sends the message silently.\n Users will receive a notification with no sound.\n\n reply_to_message_id (``int``, *optional*):\n If the message is a reply, ID of the original message.\n\n reply_markup (:obj:`InlineKeyboardMarkup` | :obj:`ReplyKeyboardMarkup` | :obj:`ReplyKeyboardRemove` | :obj:`ForceReply`, *optional*):\n Additional interface options. An object for an inline keyboard, custom reply keyboard,\n instructions to remove reply keyboard or to force a reply from the user.\n\n progress (``callable``, *optional*):\n Pass a callback function to view the upload progress.\n The function must take *(client, current, total, \\*args)* as positional arguments (look at the section\n below for a detailed description).\n\n progress_args (``tuple``, *optional*):\n Extra custom arguments for the progress callback function. Useful, for example, if you want to pass\n a chat_id and a message_id in order to edit a message with the updated progress.\n\n Other Parameters:\n client (:obj:`Client `):\n The Client itself, useful when you want to call other API methods inside the callback function.\n\n current (``int``):\n The amount of bytes uploaded so far.\n\n total (``int``):\n The size of the file.\n\n *args (``tuple``, *optional*):\n Extra custom arguments as defined in the *progress_args* parameter.\n You can either keep *\\*args* or add every single extra argument in your function signature.\n\n Returns:\n On success, the sent :obj:`Message ` is returned.\n In case the upload is deliberately stopped with :meth:`stop_transmission`, None is returned instead.\n\n Raises:\n :class:`RPCError `\n \"\"\"\n if quote is None:\n quote = self.chat.type != \"private\"\n\n if reply_to_message_id is None and quote:\n reply_to_message_id = self.message_id\n\n return self._client.send_animation(\n chat_id=self.chat.id,\n animation=animation,\n caption=caption,\n parse_mode=parse_mode,\n duration=duration,\n width=width,\n height=height,\n thumb=thumb,\n disable_notification=disable_notification,\n reply_to_message_id=reply_to_message_id,\n reply_markup=reply_markup,\n progress=progress,\n progress_args=progress_args\n )", "response": "Reply to a message in Telegram servers."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreply to an audio file.", "response": "def reply_audio(\n self,\n audio: str,\n quote: bool = None,\n caption: str = \"\",\n parse_mode: str = \"\",\n duration: int = 0,\n performer: str = None,\n title: str = None,\n thumb: str = None,\n disable_notification: bool = None,\n reply_to_message_id: int = None,\n reply_markup: Union[\n \"pyrogram.InlineKeyboardMarkup\",\n \"pyrogram.ReplyKeyboardMarkup\",\n \"pyrogram.ReplyKeyboardRemove\",\n \"pyrogram.ForceReply\"\n ] = None,\n progress: callable = None,\n progress_args: tuple = ()\n ) -> \"Message\":\n \"\"\"Bound method *reply_audio* of :obj:`Message `.\n\n Use as a shortcut for:\n\n .. code-block:: python\n\n client.send_audio(\n chat_id=message.chat.id,\n audio=audio\n )\n\n Example:\n .. code-block:: python\n\n message.reply_audio(audio)\n\n Args:\n audio (``str``):\n Audio file to send.\n Pass a file_id as string to send an audio file that exists on the Telegram servers,\n pass an HTTP URL as a string for Telegram to get an audio file from the Internet, or\n pass a file path as string to upload a new audio file that exists on your local machine.\n\n quote (``bool``, *optional*):\n If ``True``, the message will be sent as a reply to this message.\n If *reply_to_message_id* is passed, this parameter will be ignored.\n Defaults to ``True`` in group chats and ``False`` in private chats.\n\n caption (``str``, *optional*):\n Audio caption, 0-1024 characters.\n\n parse_mode (``str``, *optional*):\n Use :obj:`MARKDOWN ` or :obj:`HTML `\n if you want Telegram apps to show bold, italic, fixed-width text or inline URLs in your caption.\n Defaults to Markdown.\n\n duration (``int``, *optional*):\n Duration of the audio in seconds.\n\n performer (``str``, *optional*):\n Performer.\n\n title (``str``, *optional*):\n Track name.\n\n thumb (``str``, *optional*):\n Thumbnail of the music file album cover.\n The thumbnail should be in JPEG format and less than 200 KB in size.\n A thumbnail's width and height should not exceed 90 pixels.\n Thumbnails can't be reused and can be only uploaded as a new file.\n\n disable_notification (``bool``, *optional*):\n Sends the message silently.\n Users will receive a notification with no sound.\n\n reply_to_message_id (``int``, *optional*):\n If the message is a reply, ID of the original message.\n\n reply_markup (:obj:`InlineKeyboardMarkup` | :obj:`ReplyKeyboardMarkup` | :obj:`ReplyKeyboardRemove` | :obj:`ForceReply`, *optional*):\n Additional interface options. An object for an inline keyboard, custom reply keyboard,\n instructions to remove reply keyboard or to force a reply from the user.\n\n progress (``callable``, *optional*):\n Pass a callback function to view the upload progress.\n The function must take *(client, current, total, \\*args)* as positional arguments (look at the section\n below for a detailed description).\n\n progress_args (``tuple``, *optional*):\n Extra custom arguments for the progress callback function. Useful, for example, if you want to pass\n a chat_id and a message_id in order to edit a message with the updated progress.\n\n Other Parameters:\n client (:obj:`Client `):\n The Client itself, useful when you want to call other API methods inside the callback function.\n\n current (``int``):\n The amount of bytes uploaded so far.\n\n total (``int``):\n The size of the file.\n\n *args (``tuple``, *optional*):\n Extra custom arguments as defined in the *progress_args* parameter.\n You can either keep *\\*args* or add every single extra argument in your function signature.\n\n Returns:\n On success, the sent :obj:`Message ` is returned.\n In case the upload is deliberately stopped with :meth:`stop_transmission`, None is returned instead.\n\n Raises:\n :class:`RPCError `\n \"\"\"\n if quote is None:\n quote = self.chat.type != \"private\"\n\n if reply_to_message_id is None and quote:\n reply_to_message_id = self.message_id\n\n return self._client.send_audio(\n chat_id=self.chat.id,\n audio=audio,\n caption=caption,\n parse_mode=parse_mode,\n duration=duration,\n performer=performer,\n title=title,\n thumb=thumb,\n disable_notification=disable_notification,\n reply_to_message_id=reply_to_message_id,\n reply_markup=reply_markup,\n progress=progress,\n progress_args=progress_args\n )"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef reply_cached_media(\n self,\n file_id: str,\n quote: bool = None,\n caption: str = \"\",\n parse_mode: str = \"\",\n disable_notification: bool = None,\n reply_to_message_id: int = None,\n reply_markup: Union[\n \"pyrogram.InlineKeyboardMarkup\",\n \"pyrogram.ReplyKeyboardMarkup\",\n \"pyrogram.ReplyKeyboardRemove\",\n \"pyrogram.ForceReply\"\n ] = None\n ) -> \"Message\":\n \"\"\"Bound method *reply_cached_media* of :obj:`Message `.\n\n Use as a shortcut for:\n\n .. code-block:: python\n\n client.send_cached_media(\n chat_id=message.chat.id,\n file_id=file_id\n )\n\n Example:\n .. code-block:: python\n\n message.reply_cached_media(file_id)\n\n Args:\n file_id (``str``):\n Media to send.\n Pass a file_id as string to send a media that exists on the Telegram servers.\n\n quote (``bool``, *optional*):\n If ``True``, the message will be sent as a reply to this message.\n If *reply_to_message_id* is passed, this parameter will be ignored.\n Defaults to ``True`` in group chats and ``False`` in private chats.\n\n caption (``bool``, *optional*):\n Media caption, 0-1024 characters.\n\n parse_mode (``str``, *optional*):\n Use :obj:`MARKDOWN ` or :obj:`HTML `\n if you want Telegram apps to show bold, italic, fixed-width text or inline URLs in your caption.\n Defaults to Markdown.\n\n disable_notification (``bool``, *optional*):\n Sends the message silently.\n Users will receive a notification with no sound.\n\n reply_to_message_id (``int``, *optional*):\n If the message is a reply, ID of the original message.\n\n reply_markup (:obj:`InlineKeyboardMarkup` | :obj:`ReplyKeyboardMarkup` | :obj:`ReplyKeyboardRemove` | :obj:`ForceReply`, *optional*):\n Additional interface options. An object for an inline keyboard, custom reply keyboard,\n instructions to remove reply keyboard or to force a reply from the user.\n\n Returns:\n On success, the sent :obj:`Message ` is returned.\n\n Raises:\n :class:`RPCError `\n \"\"\"\n if quote is None:\n quote = self.chat.type != \"private\"\n\n if reply_to_message_id is None and quote:\n reply_to_message_id = self.message_id\n\n return self._client.send_cached_media(\n chat_id=self.chat.id,\n file_id=file_id,\n caption=caption,\n parse_mode=parse_mode,\n disable_notification=disable_notification,\n reply_to_message_id=reply_to_message_id,\n reply_markup=reply_markup\n )", "response": "This method sends a cached media to the Telegram servers."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nsends a chat action to the user.", "response": "def reply_chat_action(\n self,\n action: Union[ChatAction, str],\n progress: int = 0\n ) -> \"Message\":\n \"\"\"Bound method *reply_chat_action* of :obj:`Message `.\n\n Use as a shortcut for:\n\n .. code-block:: python\n\n client.send_chat_action(\n chat_id=message.chat.id,\n action=\"typing\"\n )\n\n Example:\n .. code-block:: python\n\n message.reply_chat_action(\"typing\")\n\n Args:\n action (:obj:`ChatAction ` | ``str``):\n Type of action to broadcast.\n Choose one from the :class:`ChatAction ` enumeration,\n depending on what the user is about to receive.\n You can also provide a string (e.g. \"typing\", \"upload_photo\", \"record_audio\", ...).\n\n progress (``int``, *optional*):\n Progress of the upload process.\n Currently useless because official clients don't seem to be handling this.\n\n Returns:\n On success, True is returned.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n ``ValueError`` if the provided string is not a valid ChatAction.\n \"\"\"\n return self._client.send_chat_action(\n chat_id=self.chat.id,\n action=action,\n progress=progress\n )"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef reply_contact(\n self,\n phone_number: str,\n first_name: str,\n quote: bool = None,\n last_name: str = \"\",\n vcard: str = \"\",\n disable_notification: bool = None,\n reply_to_message_id: int = None,\n reply_markup: Union[\n \"pyrogram.InlineKeyboardMarkup\",\n \"pyrogram.ReplyKeyboardMarkup\",\n \"pyrogram.ReplyKeyboardRemove\",\n \"pyrogram.ForceReply\"\n ] = None\n ) -> \"Message\":\n \"\"\"Bound method *reply_contact* of :obj:`Message `.\n\n Use as a shortcut for:\n\n .. code-block:: python\n\n client.send_contact(\n chat_id=message.chat.id,\n phone_number=phone_number,\n first_name=first_name\n )\n\n Example:\n .. code-block:: python\n\n message.reply_contact(phone_number, \"Dan\")\n\n Args:\n phone_number (``str``):\n Contact's phone number.\n\n first_name (``str``):\n Contact's first name.\n\n quote (``bool``, *optional*):\n If ``True``, the message will be sent as a reply to this message.\n If *reply_to_message_id* is passed, this parameter will be ignored.\n Defaults to ``True`` in group chats and ``False`` in private chats.\n\n last_name (``str``, *optional*):\n Contact's last name.\n\n vcard (``str``, *optional*):\n Additional data about the contact in the form of a vCard, 0-2048 bytes\n\n disable_notification (``bool``, *optional*):\n Sends the message silently.\n Users will receive a notification with no sound.\n\n reply_to_message_id (``int``, *optional*):\n If the message is a reply, ID of the original message.\n\n reply_markup (:obj:`InlineKeyboardMarkup` | :obj:`ReplyKeyboardMarkup` | :obj:`ReplyKeyboardRemove` | :obj:`ForceReply`, *optional*):\n Additional interface options. An object for an inline keyboard, custom reply keyboard,\n instructions to remove reply keyboard or to force a reply from the user.\n\n Returns:\n On success, the sent :obj:`Message ` is returned.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n if quote is None:\n quote = self.chat.type != \"private\"\n\n if reply_to_message_id is None and quote:\n reply_to_message_id = self.message_id\n\n return self._client.send_contact(\n chat_id=self.chat.id,\n phone_number=phone_number,\n first_name=first_name,\n last_name=last_name,\n vcard=vcard,\n disable_notification=disable_notification,\n reply_to_message_id=reply_to_message_id,\n reply_markup=reply_markup\n )", "response": "This method sends a contact to the user in the chat."} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreplies to a specific document on the Telegram servers.", "response": "def reply_document(\n self,\n document: str,\n quote: bool = None,\n thumb: str = None,\n caption: str = \"\",\n parse_mode: str = \"\",\n disable_notification: bool = None,\n reply_to_message_id: int = None,\n reply_markup: Union[\n \"pyrogram.InlineKeyboardMarkup\",\n \"pyrogram.ReplyKeyboardMarkup\",\n \"pyrogram.ReplyKeyboardRemove\",\n \"pyrogram.ForceReply\"\n ] = None,\n progress: callable = None,\n progress_args: tuple = ()\n ) -> \"Message\":\n \"\"\"Bound method *reply_document* of :obj:`Message `.\n\n Use as a shortcut for:\n\n .. code-block:: python\n\n client.send_document(\n chat_id=message.chat.id,\n document=document\n )\n\n Example:\n .. code-block:: python\n\n message.reply_document(document)\n\n Args:\n document (``str``):\n File to send.\n Pass a file_id as string to send a file that exists on the Telegram servers,\n pass an HTTP URL as a string for Telegram to get a file from the Internet, or\n pass a file path as string to upload a new file that exists on your local machine.\n\n quote (``bool``, *optional*):\n If ``True``, the message will be sent as a reply to this message.\n If *reply_to_message_id* is passed, this parameter will be ignored.\n Defaults to ``True`` in group chats and ``False`` in private chats.\n\n thumb (``str``, *optional*):\n Thumbnail of the file sent.\n The thumbnail should be in JPEG format and less than 200 KB in size.\n A thumbnail's width and height should not exceed 90 pixels.\n Thumbnails can't be reused and can be only uploaded as a new file.\n\n caption (``str``, *optional*):\n Document caption, 0-1024 characters.\n\n parse_mode (``str``, *optional*):\n Use :obj:`MARKDOWN ` or :obj:`HTML `\n if you want Telegram apps to show bold, italic, fixed-width text or inline URLs in your caption.\n Defaults to Markdown.\n\n disable_notification (``bool``, *optional*):\n Sends the message silently.\n Users will receive a notification with no sound.\n\n reply_to_message_id (``int``, *optional*):\n If the message is a reply, ID of the original message.\n\n reply_markup (:obj:`InlineKeyboardMarkup` | :obj:`ReplyKeyboardMarkup` | :obj:`ReplyKeyboardRemove` | :obj:`ForceReply`, *optional*):\n Additional interface options. An object for an inline keyboard, custom reply keyboard,\n instructions to remove reply keyboard or to force a reply from the user.\n\n progress (``callable``, *optional*):\n Pass a callback function to view the upload progress.\n The function must take *(client, current, total, \\*args)* as positional arguments (look at the section\n below for a detailed description).\n\n progress_args (``tuple``, *optional*):\n Extra custom arguments for the progress callback function. Useful, for example, if you want to pass\n a chat_id and a message_id in order to edit a message with the updated progress.\n\n Other Parameters:\n client (:obj:`Client `):\n The Client itself, useful when you want to call other API methods inside the callback function.\n\n current (``int``):\n The amount of bytes uploaded so far.\n\n total (``int``):\n The size of the file.\n\n *args (``tuple``, *optional*):\n Extra custom arguments as defined in the *progress_args* parameter.\n You can either keep *\\*args* or add every single extra argument in your function signature.\n\n Returns:\n On success, the sent :obj:`Message ` is returned.\n In case the upload is deliberately stopped with :meth:`stop_transmission`, None is returned instead.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n if quote is None:\n quote = self.chat.type != \"private\"\n\n if reply_to_message_id is None and quote:\n reply_to_message_id = self.message_id\n\n return self._client.send_document(\n chat_id=self.chat.id,\n document=document,\n thumb=thumb,\n caption=caption,\n parse_mode=parse_mode,\n disable_notification=disable_notification,\n reply_to_message_id=reply_to_message_id,\n reply_markup=reply_markup,\n progress=progress,\n progress_args=progress_args\n )"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nbind method *reply_game* of :obj:`Message `. Use as a shortcut for: .. code-block:: python client.send_game( chat_id=message.chat.id, game_short_name=\"lumberjack\" ) Example: .. code-block:: python message.reply_game(\"lumberjack\") Args: game_short_name (``str``): Short name of the game, serves as the unique identifier for the game. Set up your games via Botfather. quote (``bool``, *optional*): If ``True``, the message will be sent as a reply to this message. If *reply_to_message_id* is passed, this parameter will be ignored. Defaults to ``True`` in group chats and ``False`` in private chats. disable_notification (``bool``, *optional*): Sends the message silently. Users will receive a notification with no sound. reply_to_message_id (``int``, *optional*): If the message is a reply, ID of the original message. reply_markup (:obj:`InlineKeyboardMarkup`, *optional*): An object for an inline keyboard. If empty, one \u2018Play game_title\u2019 button will be shown automatically. If not empty, the first button must launch the game. Returns: On success, the sent :obj:`Message` is returned. Raises: :class:`RPCError ` in case of a Telegram RPC error.", "response": "def reply_game(\n self,\n game_short_name: str,\n quote: bool = None,\n disable_notification: bool = None,\n reply_to_message_id: int = None,\n reply_markup: Union[\n \"pyrogram.InlineKeyboardMarkup\",\n \"pyrogram.ReplyKeyboardMarkup\",\n \"pyrogram.ReplyKeyboardRemove\",\n \"pyrogram.ForceReply\"\n ] = None\n ) -> \"Message\":\n \"\"\"Bound method *reply_game* of :obj:`Message `.\n\n Use as a shortcut for:\n\n .. code-block:: python\n\n client.send_game(\n chat_id=message.chat.id,\n game_short_name=\"lumberjack\"\n )\n\n Example:\n .. code-block:: python\n\n message.reply_game(\"lumberjack\")\n\n Args:\n game_short_name (``str``):\n Short name of the game, serves as the unique identifier for the game. Set up your games via Botfather.\n\n quote (``bool``, *optional*):\n If ``True``, the message will be sent as a reply to this message.\n If *reply_to_message_id* is passed, this parameter will be ignored.\n Defaults to ``True`` in group chats and ``False`` in private chats.\n\n disable_notification (``bool``, *optional*):\n Sends the message silently.\n Users will receive a notification with no sound.\n\n reply_to_message_id (``int``, *optional*):\n If the message is a reply, ID of the original message.\n\n reply_markup (:obj:`InlineKeyboardMarkup`, *optional*):\n An object for an inline keyboard. If empty, one \u2018Play game_title\u2019 button will be shown automatically.\n If not empty, the first button must launch the game.\n\n Returns:\n On success, the sent :obj:`Message` is returned.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n if quote is None:\n quote = self.chat.type != \"private\"\n\n if reply_to_message_id is None and quote:\n reply_to_message_id = self.message_id\n\n return self._client.send_game(\n chat_id=self.chat.id,\n game_short_name=game_short_name,\n disable_notification=disable_notification,\n reply_to_message_id=reply_to_message_id,\n reply_markup=reply_markup\n )"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nbinding method *reply_inline_bot_result* of :obj:`Message `. Use as a shortcut for: .. code-block:: python client.send_inline_bot_result( chat_id=message.chat.id, query_id=query_id, result_id=result_id ) Example: .. code-block:: python message.reply_inline_bot_result(query_id, result_id) Args: query_id (``int``): Unique identifier for the answered query. result_id (``str``): Unique identifier for the result that was chosen. quote (``bool``, *optional*): If ``True``, the message will be sent as a reply to this message. If *reply_to_message_id* is passed, this parameter will be ignored. Defaults to ``True`` in group chats and ``False`` in private chats. disable_notification (``bool``, *optional*): Sends the message silently. Users will receive a notification with no sound. reply_to_message_id (``bool``, *optional*): If the message is a reply, ID of the original message. hide_via (``bool``): Sends the message with *via @bot* hidden. Returns: On success, the sent Message is returned. Raises: :class:`RPCError ` in case of a Telegram RPC error.", "response": "def reply_inline_bot_result(\n self,\n query_id: int,\n result_id: str,\n quote: bool = None,\n disable_notification: bool = None,\n reply_to_message_id: int = None,\n hide_via: bool = None\n ) -> \"Message\":\n \"\"\"Bound method *reply_inline_bot_result* of :obj:`Message `.\n\n Use as a shortcut for:\n\n .. code-block:: python\n\n client.send_inline_bot_result(\n chat_id=message.chat.id,\n query_id=query_id,\n result_id=result_id\n )\n\n Example:\n .. code-block:: python\n\n message.reply_inline_bot_result(query_id, result_id)\n\n Args:\n query_id (``int``):\n Unique identifier for the answered query.\n\n result_id (``str``):\n Unique identifier for the result that was chosen.\n\n quote (``bool``, *optional*):\n If ``True``, the message will be sent as a reply to this message.\n If *reply_to_message_id* is passed, this parameter will be ignored.\n Defaults to ``True`` in group chats and ``False`` in private chats.\n\n disable_notification (``bool``, *optional*):\n Sends the message silently.\n Users will receive a notification with no sound.\n\n reply_to_message_id (``bool``, *optional*):\n If the message is a reply, ID of the original message.\n\n hide_via (``bool``):\n Sends the message with *via @bot* hidden.\n\n Returns:\n On success, the sent Message is returned.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n if quote is None:\n quote = self.chat.type != \"private\"\n\n if reply_to_message_id is None and quote:\n reply_to_message_id = self.message_id\n\n return self._client.send_inline_bot_result(\n chat_id=self.chat.id,\n query_id=query_id,\n result_id=result_id,\n disable_notification=disable_notification,\n reply_to_message_id=reply_to_message_id,\n hide_via=hide_via\n )"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nbinds method *reply_location* of :obj:`Message `. Use as a shortcut for: .. code-block:: python client.send_location( chat_id=message.chat.id, latitude=41.890251, longitude=12.492373 ) Example: .. code-block:: python message.reply_location(41.890251, 12.492373) Args: latitude (``float``): Latitude of the location. longitude (``float``): Longitude of the location. quote (``bool``, *optional*): If ``True``, the message will be sent as a reply to this message. If *reply_to_message_id* is passed, this parameter will be ignored. Defaults to ``True`` in group chats and ``False`` in private chats. disable_notification (``bool``, *optional*): Sends the message silently. Users will receive a notification with no sound. reply_to_message_id (``int``, *optional*): If the message is a reply, ID of the original message reply_markup (:obj:`InlineKeyboardMarkup` | :obj:`ReplyKeyboardMarkup` | :obj:`ReplyKeyboardRemove` | :obj:`ForceReply`, *optional*): Additional interface options. An object for an inline keyboard, custom reply keyboard, instructions to remove reply keyboard or to force a reply from the user. Returns: On success, the sent :obj:`Message ` is returned. Raises: :class:`RPCError ` in case of a Telegram RPC error.", "response": "def reply_location(\n self,\n latitude: float,\n longitude: float,\n quote: bool = None,\n disable_notification: bool = None,\n reply_to_message_id: int = None,\n reply_markup: Union[\n \"pyrogram.InlineKeyboardMarkup\",\n \"pyrogram.ReplyKeyboardMarkup\",\n \"pyrogram.ReplyKeyboardRemove\",\n \"pyrogram.ForceReply\"\n ] = None\n ) -> \"Message\":\n \"\"\"Bound method *reply_location* of :obj:`Message `.\n\n Use as a shortcut for:\n\n .. code-block:: python\n\n client.send_location(\n chat_id=message.chat.id,\n latitude=41.890251,\n longitude=12.492373\n )\n\n Example:\n .. code-block:: python\n\n message.reply_location(41.890251, 12.492373)\n\n Args:\n latitude (``float``):\n Latitude of the location.\n\n longitude (``float``):\n Longitude of the location.\n\n quote (``bool``, *optional*):\n If ``True``, the message will be sent as a reply to this message.\n If *reply_to_message_id* is passed, this parameter will be ignored.\n Defaults to ``True`` in group chats and ``False`` in private chats.\n\n disable_notification (``bool``, *optional*):\n Sends the message silently.\n Users will receive a notification with no sound.\n\n reply_to_message_id (``int``, *optional*):\n If the message is a reply, ID of the original message\n\n reply_markup (:obj:`InlineKeyboardMarkup` | :obj:`ReplyKeyboardMarkup` | :obj:`ReplyKeyboardRemove` | :obj:`ForceReply`, *optional*):\n Additional interface options. An object for an inline keyboard, custom reply keyboard,\n instructions to remove reply keyboard or to force a reply from the user.\n\n Returns:\n On success, the sent :obj:`Message ` is returned.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n if quote is None:\n quote = self.chat.type != \"private\"\n\n if reply_to_message_id is None and quote:\n reply_to_message_id = self.message_id\n\n return self._client.send_location(\n chat_id=self.chat.id,\n latitude=latitude,\n longitude=longitude,\n disable_notification=disable_notification,\n reply_to_message_id=reply_to_message_id,\n reply_markup=reply_markup\n )"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef reply_media_group(\n self,\n media: List[Union[\"pyrogram.InputMediaPhoto\", \"pyrogram.InputMediaVideo\"]],\n quote: bool = None,\n disable_notification: bool = None,\n reply_to_message_id: int = None\n ) -> \"Message\":\n \"\"\"Bound method *reply_media_group* of :obj:`Message `.\n\n Use as a shortcut for:\n\n .. code-block:: python\n\n client.send_media_group(\n chat_id=message.chat.id,\n media=list_of_media\n )\n\n Example:\n .. code-block:: python\n\n message.reply_media_group(list_of_media)\n\n Args:\n media (``list``):\n A list containing either :obj:`InputMediaPhoto ` or\n :obj:`InputMediaVideo ` objects\n describing photos and videos to be sent, must include 2\u201310 items.\n\n quote (``bool``, *optional*):\n If ``True``, the message will be sent as a reply to this message.\n If *reply_to_message_id* is passed, this parameter will be ignored.\n Defaults to ``True`` in group chats and ``False`` in private chats.\n\n disable_notification (``bool``, *optional*):\n Sends the message silently.\n Users will receive a notification with no sound.\n\n reply_to_message_id (``int``, *optional*):\n If the message is a reply, ID of the original message.\n\n Returns:\n On success, a :obj:`Messages ` object is returned containing all the\n single messages sent.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n if quote is None:\n quote = self.chat.type != \"private\"\n\n if reply_to_message_id is None and quote:\n reply_to_message_id = self.message_id\n\n return self._client.send_media_group(\n chat_id=self.chat.id,\n media=media,\n disable_notification=disable_notification,\n reply_to_message_id=reply_to_message_id\n )", "response": "This method sends a list of photos and videos to the chat."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef reply_photo(\n self,\n photo: str,\n quote: bool = None,\n caption: str = \"\",\n parse_mode: str = \"\",\n ttl_seconds: int = None,\n disable_notification: bool = None,\n reply_to_message_id: int = None,\n reply_markup: Union[\n \"pyrogram.InlineKeyboardMarkup\",\n \"pyrogram.ReplyKeyboardMarkup\",\n \"pyrogram.ReplyKeyboardRemove\",\n \"pyrogram.ForceReply\"\n ] = None,\n progress: callable = None,\n progress_args: tuple = ()\n ) -> \"Message\":\n \"\"\"Bound method *reply_photo* of :obj:`Message `.\n\n Use as a shortcut for:\n\n .. code-block:: python\n\n client.send_photo(\n chat_id=message.chat.id,\n photo=photo\n )\n\n Example:\n .. code-block:: python\n\n message.reply_photo(photo)\n\n Args:\n photo (``str``):\n Photo to send.\n Pass a file_id as string to send a photo that exists on the Telegram servers,\n pass an HTTP URL as a string for Telegram to get a photo from the Internet, or\n pass a file path as string to upload a new photo that exists on your local machine.\n\n quote (``bool``, *optional*):\n If ``True``, the message will be sent as a reply to this message.\n If *reply_to_message_id* is passed, this parameter will be ignored.\n Defaults to ``True`` in group chats and ``False`` in private chats.\n\n caption (``bool``, *optional*):\n Photo caption, 0-1024 characters.\n\n parse_mode (``str``, *optional*):\n Use :obj:`MARKDOWN ` or :obj:`HTML `\n if you want Telegram apps to show bold, italic, fixed-width text or inline URLs in your caption.\n Defaults to Markdown.\n\n ttl_seconds (``int``, *optional*):\n Self-Destruct Timer.\n If you set a timer, the photo will self-destruct in *ttl_seconds*\n seconds after it was viewed.\n\n disable_notification (``bool``, *optional*):\n Sends the message silently.\n Users will receive a notification with no sound.\n\n reply_to_message_id (``int``, *optional*):\n If the message is a reply, ID of the original message.\n\n reply_markup (:obj:`InlineKeyboardMarkup` | :obj:`ReplyKeyboardMarkup` | :obj:`ReplyKeyboardRemove` | :obj:`ForceReply`, *optional*):\n Additional interface options. An object for an inline keyboard, custom reply keyboard,\n instructions to remove reply keyboard or to force a reply from the user.\n\n progress (``callable``, *optional*):\n Pass a callback function to view the upload progress.\n The function must take *(client, current, total, \\*args)* as positional arguments (look at the section\n below for a detailed description).\n\n progress_args (``tuple``, *optional*):\n Extra custom arguments for the progress callback function. Useful, for example, if you want to pass\n a chat_id and a message_id in order to edit a message with the updated progress.\n\n Other Parameters:\n client (:obj:`Client `):\n The Client itself, useful when you want to call other API methods inside the callback function.\n\n current (``int``):\n The amount of bytes uploaded so far.\n\n total (``int``):\n The size of the file.\n\n *args (``tuple``, *optional*):\n Extra custom arguments as defined in the *progress_args* parameter.\n You can either keep *\\*args* or add every single extra argument in your function signature.\n\n Returns:\n On success, the sent :obj:`Message ` is returned.\n In case the upload is deliberately stopped with :meth:`stop_transmission`, None is returned instead.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n if quote is None:\n quote = self.chat.type != \"private\"\n\n if reply_to_message_id is None and quote:\n reply_to_message_id = self.message_id\n\n return self._client.send_photo(\n chat_id=self.chat.id,\n photo=photo,\n caption=caption,\n parse_mode=parse_mode,\n ttl_seconds=ttl_seconds,\n disable_notification=disable_notification,\n reply_to_message_id=reply_to_message_id,\n reply_markup=reply_markup,\n progress=progress,\n progress_args=progress_args\n )", "response": "Reply to a photo."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef reply_poll(\n self,\n question: str,\n options: List[str],\n quote: bool = None,\n disable_notification: bool = None,\n reply_to_message_id: int = None,\n reply_markup: Union[\n \"pyrogram.InlineKeyboardMarkup\",\n \"pyrogram.ReplyKeyboardMarkup\",\n \"pyrogram.ReplyKeyboardRemove\",\n \"pyrogram.ForceReply\"\n ] = None\n ) -> \"Message\":\n \"\"\"Bound method *reply_poll* of :obj:`Message `.\n\n Use as a shortcut for:\n\n .. code-block:: python\n\n client.send_poll(\n chat_id=message.chat.id,\n question=\"Is Pyrogram the best?\",\n options=[\"Yes\", \"Yes\"]\n )\n\n Example:\n .. code-block:: python\n\n message.reply_poll(\"Is Pyrogram the best?\", [\"Yes\", \"Yes\"])\n\n Args:\n question (``str``):\n The poll question, as string.\n\n options (List of ``str``):\n The poll options, as list of strings (2 to 10 options are allowed).\n\n quote (``bool``, *optional*):\n If ``True``, the message will be sent as a reply to this message.\n If *reply_to_message_id* is passed, this parameter will be ignored.\n Defaults to ``True`` in group chats and ``False`` in private chats.\n\n disable_notification (``bool``, *optional*):\n Sends the message silently.\n Users will receive a notification with no sound.\n\n reply_to_message_id (``int``, *optional*):\n If the message is a reply, ID of the original message.\n\n reply_markup (:obj:`InlineKeyboardMarkup` | :obj:`ReplyKeyboardMarkup` | :obj:`ReplyKeyboardRemove` | :obj:`ForceReply`, *optional*):\n Additional interface options. An object for an inline keyboard, custom reply keyboard,\n instructions to remove reply keyboard or to force a reply from the user.\n\n Returns:\n On success, the sent :obj:`Message ` is returned.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n if quote is None:\n quote = self.chat.type != \"private\"\n\n if reply_to_message_id is None and quote:\n reply_to_message_id = self.message_id\n\n return self._client.send_poll(\n chat_id=self.chat.id,\n question=question,\n options=options,\n disable_notification=disable_notification,\n reply_to_message_id=reply_to_message_id,\n reply_markup=reply_markup\n )", "response": "This method sends a poll to the user."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef reply_sticker(\n self,\n sticker: str,\n quote: bool = None,\n disable_notification: bool = None,\n reply_to_message_id: int = None,\n reply_markup: Union[\n \"pyrogram.InlineKeyboardMarkup\",\n \"pyrogram.ReplyKeyboardMarkup\",\n \"pyrogram.ReplyKeyboardRemove\",\n \"pyrogram.ForceReply\"\n ] = None,\n progress: callable = None,\n progress_args: tuple = ()\n ) -> \"Message\":\n \"\"\"Bound method *reply_sticker* of :obj:`Message `.\n\n Use as a shortcut for:\n\n .. code-block:: python\n\n client.send_sticker(\n chat_id=message.chat.id,\n sticker=sticker\n )\n\n Example:\n .. code-block:: python\n\n message.reply_sticker(sticker)\n\n Args:\n sticker (``str``):\n Sticker to send.\n Pass a file_id as string to send a sticker that exists on the Telegram servers,\n pass an HTTP URL as a string for Telegram to get a .webp sticker file from the Internet, or\n pass a file path as string to upload a new sticker that exists on your local machine.\n\n quote (``bool``, *optional*):\n If ``True``, the message will be sent as a reply to this message.\n If *reply_to_message_id* is passed, this parameter will be ignored.\n Defaults to ``True`` in group chats and ``False`` in private chats.\n\n disable_notification (``bool``, *optional*):\n Sends the message silently.\n Users will receive a notification with no sound.\n\n reply_to_message_id (``int``, *optional*):\n If the message is a reply, ID of the original message.\n\n reply_markup (:obj:`InlineKeyboardMarkup` | :obj:`ReplyKeyboardMarkup` | :obj:`ReplyKeyboardRemove` | :obj:`ForceReply`, *optional*):\n Additional interface options. An object for an inline keyboard, custom reply keyboard,\n instructions to remove reply keyboard or to force a reply from the user.\n\n progress (``callable``, *optional*):\n Pass a callback function to view the upload progress.\n The function must take *(client, current, total, \\*args)* as positional arguments (look at the section\n below for a detailed description).\n\n progress_args (``tuple``, *optional*):\n Extra custom arguments for the progress callback function. Useful, for example, if you want to pass\n a chat_id and a message_id in order to edit a message with the updated progress.\n\n Other Parameters:\n client (:obj:`Client `):\n The Client itself, useful when you want to call other API methods inside the callback function.\n\n current (``int``):\n The amount of bytes uploaded so far.\n\n total (``int``):\n The size of the file.\n\n *args (``tuple``, *optional*):\n Extra custom arguments as defined in the *progress_args* parameter.\n You can either keep *\\*args* or add every single extra argument in your function signature.\n\n Returns:\n On success, the sent :obj:`Message ` is returned.\n In case the upload is deliberately stopped with :meth:`stop_transmission`, None is returned instead.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n if quote is None:\n quote = self.chat.type != \"private\"\n\n if reply_to_message_id is None and quote:\n reply_to_message_id = self.message_id\n\n return self._client.send_sticker(\n chat_id=self.chat.id,\n sticker=sticker,\n disable_notification=disable_notification,\n reply_to_message_id=reply_to_message_id,\n reply_markup=reply_markup,\n progress=progress,\n progress_args=progress_args\n )", "response": "This method sends a new sticker to the Telegram servers and sends it as a reply to the original message."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nbind method *reply_venue* of :obj:`Message `. Use as a shortcut for: .. code-block:: python client.send_venue( chat_id=message.chat.id, latitude=41.890251, longitude=12.492373, title=\"Coliseum\", address=\"Piazza del Colosseo, 1, 00184 Roma RM\" ) Example: .. code-block:: python message.reply_venue(41.890251, 12.492373, \"Coliseum\", \"Piazza del Colosseo, 1, 00184 Roma RM\") Args: latitude (``float``): Latitude of the venue. longitude (``float``): Longitude of the venue. title (``str``): Name of the venue. address (``str``): Address of the venue. quote (``bool``, *optional*): If ``True``, the message will be sent as a reply to this message. If *reply_to_message_id* is passed, this parameter will be ignored. Defaults to ``True`` in group chats and ``False`` in private chats. foursquare_id (``str``, *optional*): Foursquare identifier of the venue. foursquare_type (``str``, *optional*): Foursquare type of the venue, if known. (For example, \"arts_entertainment/default\", \"arts_entertainment/aquarium\" or \"food/icecream\".) disable_notification (``bool``, *optional*): Sends the message silently. Users will receive a notification with no sound. reply_to_message_id (``int``, *optional*): If the message is a reply, ID of the original message reply_markup (:obj:`InlineKeyboardMarkup` | :obj:`ReplyKeyboardMarkup` | :obj:`ReplyKeyboardRemove` | :obj:`ForceReply`, *optional*): Additional interface options. An object for an inline keyboard, custom reply keyboard, instructions to remove reply keyboard or to force a reply from the user. Returns: On success, the sent :obj:`Message ` is returned. Raises: :class:`RPCError ` in case of a Telegram RPC error.", "response": "def reply_venue(\n self,\n latitude: float,\n longitude: float,\n title: str,\n address: str,\n quote: bool = None,\n foursquare_id: str = \"\",\n foursquare_type: str = \"\",\n disable_notification: bool = None,\n reply_to_message_id: int = None,\n reply_markup: Union[\n \"pyrogram.InlineKeyboardMarkup\",\n \"pyrogram.ReplyKeyboardMarkup\",\n \"pyrogram.ReplyKeyboardRemove\",\n \"pyrogram.ForceReply\"\n ] = None\n ) -> \"Message\":\n \"\"\"Bound method *reply_venue* of :obj:`Message `.\n\n Use as a shortcut for:\n\n .. code-block:: python\n\n client.send_venue(\n chat_id=message.chat.id,\n latitude=41.890251,\n longitude=12.492373,\n title=\"Coliseum\",\n address=\"Piazza del Colosseo, 1, 00184 Roma RM\"\n )\n\n Example:\n .. code-block:: python\n\n message.reply_venue(41.890251, 12.492373, \"Coliseum\", \"Piazza del Colosseo, 1, 00184 Roma RM\")\n\n Args:\n latitude (``float``):\n Latitude of the venue.\n\n longitude (``float``):\n Longitude of the venue.\n\n title (``str``):\n Name of the venue.\n\n address (``str``):\n Address of the venue.\n\n quote (``bool``, *optional*):\n If ``True``, the message will be sent as a reply to this message.\n If *reply_to_message_id* is passed, this parameter will be ignored.\n Defaults to ``True`` in group chats and ``False`` in private chats.\n\n foursquare_id (``str``, *optional*):\n Foursquare identifier of the venue.\n\n foursquare_type (``str``, *optional*):\n Foursquare type of the venue, if known.\n (For example, \"arts_entertainment/default\", \"arts_entertainment/aquarium\" or \"food/icecream\".)\n\n disable_notification (``bool``, *optional*):\n Sends the message silently.\n Users will receive a notification with no sound.\n\n reply_to_message_id (``int``, *optional*):\n If the message is a reply, ID of the original message\n\n reply_markup (:obj:`InlineKeyboardMarkup` | :obj:`ReplyKeyboardMarkup` | :obj:`ReplyKeyboardRemove` | :obj:`ForceReply`, *optional*):\n Additional interface options. An object for an inline keyboard, custom reply keyboard,\n instructions to remove reply keyboard or to force a reply from the user.\n\n Returns:\n On success, the sent :obj:`Message ` is returned.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n if quote is None:\n quote = self.chat.type != \"private\"\n\n if reply_to_message_id is None and quote:\n reply_to_message_id = self.message_id\n\n return self._client.send_venue(\n chat_id=self.chat.id,\n latitude=latitude,\n longitude=longitude,\n title=title,\n address=address,\n foursquare_id=foursquare_id,\n foursquare_type=foursquare_type,\n disable_notification=disable_notification,\n reply_to_message_id=reply_to_message_id,\n reply_markup=reply_markup\n )"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef reply_video(\n self,\n video: str,\n quote: bool = None,\n caption: str = \"\",\n parse_mode: str = \"\",\n duration: int = 0,\n width: int = 0,\n height: int = 0,\n thumb: str = None,\n supports_streaming: bool = True,\n disable_notification: bool = None,\n reply_to_message_id: int = None,\n reply_markup: Union[\n \"pyrogram.InlineKeyboardMarkup\",\n \"pyrogram.ReplyKeyboardMarkup\",\n \"pyrogram.ReplyKeyboardRemove\",\n \"pyrogram.ForceReply\"\n ] = None,\n progress: callable = None,\n progress_args: tuple = ()\n ) -> \"Message\":\n \"\"\"Bound method *reply_video* of :obj:`Message `.\n\n Use as a shortcut for:\n\n .. code-block:: python\n\n client.send_video(\n chat_id=message.chat.id,\n video=video\n )\n\n Example:\n .. code-block:: python\n\n message.reply_video(video)\n\n Args:\n video (``str``):\n Video to send.\n Pass a file_id as string to send a video that exists on the Telegram servers,\n pass an HTTP URL as a string for Telegram to get a video from the Internet, or\n pass a file path as string to upload a new video that exists on your local machine.\n\n quote (``bool``, *optional*):\n If ``True``, the message will be sent as a reply to this message.\n If *reply_to_message_id* is passed, this parameter will be ignored.\n Defaults to ``True`` in group chats and ``False`` in private chats.\n\n caption (``str``, *optional*):\n Video caption, 0-1024 characters.\n\n parse_mode (``str``, *optional*):\n Use :obj:`MARKDOWN ` or :obj:`HTML `\n if you want Telegram apps to show bold, italic, fixed-width text or inline URLs in your caption.\n Defaults to Markdown.\n\n duration (``int``, *optional*):\n Duration of sent video in seconds.\n\n width (``int``, *optional*):\n Video width.\n\n height (``int``, *optional*):\n Video height.\n\n thumb (``str``, *optional*):\n Thumbnail of the video sent.\n The thumbnail should be in JPEG format and less than 200 KB in size.\n A thumbnail's width and height should not exceed 90 pixels.\n Thumbnails can't be reused and can be only uploaded as a new file.\n\n supports_streaming (``bool``, *optional*):\n Pass True, if the uploaded video is suitable for streaming.\n\n disable_notification (``bool``, *optional*):\n Sends the message silently.\n Users will receive a notification with no sound.\n\n reply_to_message_id (``int``, *optional*):\n If the message is a reply, ID of the original message.\n\n reply_markup (:obj:`InlineKeyboardMarkup` | :obj:`ReplyKeyboardMarkup` | :obj:`ReplyKeyboardRemove` | :obj:`ForceReply`, *optional*):\n Additional interface options. An object for an inline keyboard, custom reply keyboard,\n instructions to remove reply keyboard or to force a reply from the user.\n\n progress (``callable``, *optional*):\n Pass a callback function to view the upload progress.\n The function must take *(client, current, total, \\*args)* as positional arguments (look at the section\n below for a detailed description).\n\n progress_args (``tuple``, *optional*):\n Extra custom arguments for the progress callback function. Useful, for example, if you want to pass\n a chat_id and a message_id in order to edit a message with the updated progress.\n\n Other Parameters:\n client (:obj:`Client `):\n The Client itself, useful when you want to call other API methods inside the callback function.\n\n current (``int``):\n The amount of bytes uploaded so far.\n\n total (``int``):\n The size of the file.\n\n *args (``tuple``, *optional*):\n Extra custom arguments as defined in the *progress_args* parameter.\n You can either keep *\\*args* or add every single extra argument in your function signature.\n\n Returns:\n On success, the sent :obj:`Message ` is returned.\n In case the upload is deliberately stopped with :meth:`stop_transmission`, None is returned instead.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n if quote is None:\n quote = self.chat.type != \"private\"\n\n if reply_to_message_id is None and quote:\n reply_to_message_id = self.message_id\n\n return self._client.send_video(\n chat_id=self.chat.id,\n video=video,\n caption=caption,\n parse_mode=parse_mode,\n duration=duration,\n width=width,\n height=height,\n thumb=thumb,\n supports_streaming=supports_streaming,\n disable_notification=disable_notification,\n reply_to_message_id=reply_to_message_id,\n reply_markup=reply_markup,\n progress=progress,\n progress_args=progress_args\n )", "response": "Reply to a video."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef reply_video_note(\n self,\n video_note: str,\n quote: bool = None,\n duration: int = 0,\n length: int = 1,\n thumb: str = None,\n disable_notification: bool = None,\n reply_to_message_id: int = None,\n reply_markup: Union[\n \"pyrogram.InlineKeyboardMarkup\",\n \"pyrogram.ReplyKeyboardMarkup\",\n \"pyrogram.ReplyKeyboardRemove\",\n \"pyrogram.ForceReply\"\n ] = None,\n progress: callable = None,\n progress_args: tuple = ()\n ) -> \"Message\":\n \"\"\"Bound method *reply_video_note* of :obj:`Message `.\n\n Use as a shortcut for:\n\n .. code-block:: python\n\n client.send_video_note(\n chat_id=message.chat.id,\n video_note=video_note\n )\n\n Example:\n .. code-block:: python\n\n message.reply_video_note(video_note)\n\n Args:\n video_note (``str``):\n Video note to send.\n Pass a file_id as string to send a video note that exists on the Telegram servers, or\n pass a file path as string to upload a new video note that exists on your local machine.\n Sending video notes by a URL is currently unsupported.\n\n quote (``bool``, *optional*):\n If ``True``, the message will be sent as a reply to this message.\n If *reply_to_message_id* is passed, this parameter will be ignored.\n Defaults to ``True`` in group chats and ``False`` in private chats.\n\n duration (``int``, *optional*):\n Duration of sent video in seconds.\n\n length (``int``, *optional*):\n Video width and height.\n\n thumb (``str``, *optional*):\n Thumbnail of the video sent.\n The thumbnail should be in JPEG format and less than 200 KB in size.\n A thumbnail's width and height should not exceed 90 pixels.\n Thumbnails can't be reused and can be only uploaded as a new file.\n\n disable_notification (``bool``, *optional*):\n Sends the message silently.\n Users will receive a notification with no sound.\n\n reply_to_message_id (``int``, *optional*):\n If the message is a reply, ID of the original message\n\n reply_markup (:obj:`InlineKeyboardMarkup` | :obj:`ReplyKeyboardMarkup` | :obj:`ReplyKeyboardRemove` | :obj:`ForceReply`, *optional*):\n Additional interface options. An object for an inline keyboard, custom reply keyboard,\n instructions to remove reply keyboard or to force a reply from the user.\n\n progress (``callable``, *optional*):\n Pass a callback function to view the upload progress.\n The function must take *(client, current, total, \\*args)* as positional arguments (look at the section\n below for a detailed description).\n\n progress_args (``tuple``, *optional*):\n Extra custom arguments for the progress callback function. Useful, for example, if you want to pass\n a chat_id and a message_id in order to edit a message with the updated progress.\n\n Other Parameters:\n client (:obj:`Client `):\n The Client itself, useful when you want to call other API methods inside the callback function.\n\n current (``int``):\n The amount of bytes uploaded so far.\n\n total (``int``):\n The size of the file.\n\n *args (``tuple``, *optional*):\n Extra custom arguments as defined in the *progress_args* parameter.\n You can either keep *\\*args* or add every single extra argument in your function signature.\n\n Returns:\n On success, the sent :obj:`Message ` is returned.\n In case the upload is deliberately stopped with :meth:`stop_transmission`, None is returned instead.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n if quote is None:\n quote = self.chat.type != \"private\"\n\n if reply_to_message_id is None and quote:\n reply_to_message_id = self.message_id\n\n return self._client.send_video_note(\n chat_id=self.chat.id,\n video_note=video_note,\n duration=duration,\n length=length,\n thumb=thumb,\n disable_notification=disable_notification,\n reply_to_message_id=reply_to_message_id,\n reply_markup=reply_markup,\n progress=progress,\n progress_args=progress_args\n )", "response": "Reply to a video note."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreply to a specific voice.", "response": "def reply_voice(\n self,\n voice: str,\n quote: bool = None,\n caption: str = \"\",\n parse_mode: str = \"\",\n duration: int = 0,\n disable_notification: bool = None,\n reply_to_message_id: int = None,\n reply_markup: Union[\n \"pyrogram.InlineKeyboardMarkup\",\n \"pyrogram.ReplyKeyboardMarkup\",\n \"pyrogram.ReplyKeyboardRemove\",\n \"pyrogram.ForceReply\"\n ] = None,\n progress: callable = None,\n progress_args: tuple = ()\n ) -> \"Message\":\n \"\"\"Bound method *reply_voice* of :obj:`Message `.\n\n Use as a shortcut for:\n\n .. code-block:: python\n\n client.send_voice(\n chat_id=message.chat.id,\n voice=voice\n )\n\n Example:\n .. code-block:: python\n\n message.reply_voice(voice)\n\n Args:\n voice (``str``):\n Audio file to send.\n Pass a file_id as string to send an audio that exists on the Telegram servers,\n pass an HTTP URL as a string for Telegram to get an audio from the Internet, or\n pass a file path as string to upload a new audio that exists on your local machine.\n\n quote (``bool``, *optional*):\n If ``True``, the message will be sent as a reply to this message.\n If *reply_to_message_id* is passed, this parameter will be ignored.\n Defaults to ``True`` in group chats and ``False`` in private chats.\n\n caption (``str``, *optional*):\n Voice message caption, 0-1024 characters.\n\n parse_mode (``str``, *optional*):\n Use :obj:`MARKDOWN ` or :obj:`HTML `\n if you want Telegram apps to show bold, italic, fixed-width text or inline URLs in your caption.\n Defaults to Markdown.\n\n duration (``int``, *optional*):\n Duration of the voice message in seconds.\n\n disable_notification (``bool``, *optional*):\n Sends the message silently.\n Users will receive a notification with no sound.\n\n reply_to_message_id (``int``, *optional*):\n If the message is a reply, ID of the original message\n\n reply_markup (:obj:`InlineKeyboardMarkup` | :obj:`ReplyKeyboardMarkup` | :obj:`ReplyKeyboardRemove` | :obj:`ForceReply`, *optional*):\n Additional interface options. An object for an inline keyboard, custom reply keyboard,\n instructions to remove reply keyboard or to force a reply from the user.\n\n progress (``callable``, *optional*):\n Pass a callback function to view the upload progress.\n The function must take *(client, current, total, \\*args)* as positional arguments (look at the section\n below for a detailed description).\n\n progress_args (``tuple``, *optional*):\n Extra custom arguments for the progress callback function. Useful, for example, if you want to pass\n a chat_id and a message_id in order to edit a message with the updated progress.\n\n Other Parameters:\n client (:obj:`Client `):\n The Client itself, useful when you want to call other API methods inside the callback function.\n\n current (``int``):\n The amount of bytes uploaded so far.\n\n total (``int``):\n The size of the file.\n\n *args (``tuple``, *optional*):\n Extra custom arguments as defined in the *progress_args* parameter.\n You can either keep *\\*args* or add every single extra argument in your function signature.\n\n Returns:\n On success, the sent :obj:`Message ` is returned.\n In case the upload is deliberately stopped with :meth:`stop_transmission`, None is returned instead.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n if quote is None:\n quote = self.chat.type != \"private\"\n\n if reply_to_message_id is None and quote:\n reply_to_message_id = self.message_id\n\n return self._client.send_voice(\n chat_id=self.chat.id,\n voice=voice,\n caption=caption,\n parse_mode=parse_mode,\n duration=duration,\n disable_notification=disable_notification,\n reply_to_message_id=reply_to_message_id,\n reply_markup=reply_markup,\n progress=progress,\n progress_args=progress_args\n )"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nediting the text of a message.", "response": "def edit(\n self,\n text: str,\n parse_mode: str = \"\",\n disable_web_page_preview: bool = None,\n reply_markup: Union[\n \"pyrogram.InlineKeyboardMarkup\",\n \"pyrogram.ReplyKeyboardMarkup\",\n \"pyrogram.ReplyKeyboardRemove\",\n \"pyrogram.ForceReply\"\n ] = None\n ) -> \"Message\":\n \"\"\"Bound method *edit* of :obj:`Message `\n\n Use as a shortcut for:\n\n .. code-block:: python\n\n client.edit_message_text(\n chat_id=message.chat.id,\n message_id=message.message_id,\n text=\"hello\"\n )\n\n Example:\n .. code-block:: python\n\n message.edit(\"hello\")\n\n Args:\n text (``str``):\n New text of the message.\n\n parse_mode (``str``, *optional*):\n Use :obj:`MARKDOWN ` or :obj:`HTML `\n if you want Telegram apps to show bold, italic, fixed-width text or inline URLs in your message.\n Defaults to Markdown.\n\n disable_web_page_preview (``bool``, *optional*):\n Disables link previews for links in this message.\n\n reply_markup (:obj:`InlineKeyboardMarkup`, *optional*):\n An InlineKeyboardMarkup object.\n\n Returns:\n On success, the edited :obj:`Message ` is returned.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n return self._client.edit_message_text(\n chat_id=self.chat.id,\n message_id=self.message_id,\n text=text,\n parse_mode=parse_mode,\n disable_web_page_preview=disable_web_page_preview,\n reply_markup=reply_markup\n )"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nediting the caption of a message.", "response": "def edit_caption(\n self,\n caption: str,\n parse_mode: str = \"\",\n reply_markup: Union[\n \"pyrogram.InlineKeyboardMarkup\",\n \"pyrogram.ReplyKeyboardMarkup\",\n \"pyrogram.ReplyKeyboardRemove\",\n \"pyrogram.ForceReply\"\n ] = None\n ) -> \"Message\":\n \"\"\"Bound method *edit_caption* of :obj:`Message `\n\n Use as a shortcut for:\n\n .. code-block:: python\n\n client.edit_message_caption(\n chat_id=message.chat.id,\n message_id=message.message_id,\n caption=\"hello\"\n )\n\n Example:\n .. code-block:: python\n\n message.edit_caption(\"hello\")\n\n Args:\n caption (``str``):\n New caption of the message.\n\n parse_mode (``str``, *optional*):\n Use :obj:`MARKDOWN ` or :obj:`HTML `\n if you want Telegram apps to show bold, italic, fixed-width text or inline URLs in your message.\n Defaults to Markdown.\n\n reply_markup (:obj:`InlineKeyboardMarkup`, *optional*):\n An InlineKeyboardMarkup object.\n\n Returns:\n On success, the edited :obj:`Message ` is returned.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n return self._client.edit_message_caption(\n chat_id=self.chat.id,\n message_id=self.message_id,\n caption=caption,\n parse_mode=parse_mode,\n reply_markup=reply_markup\n )"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef forward(\n self,\n chat_id: int or str,\n disable_notification: bool = None,\n as_copy: bool = False,\n remove_caption: bool = False\n ) -> \"Message\":\n \"\"\"Bound method *forward* of :obj:`Message `.\n\n Use as a shortcut for:\n\n .. code-block:: python\n\n client.forward_messages(\n chat_id=chat_id,\n from_chat_id=message.chat.id,\n message_ids=message.message_id\n )\n\n Example:\n .. code-block:: python\n\n message.forward(chat_id)\n\n Args:\n chat_id (``int`` | ``str``):\n Unique identifier (int) or username (str) of the target chat.\n For your personal cloud (Saved Messages) you can simply use \"me\" or \"self\".\n For a contact that exists in your Telegram address book you can use his phone number (str).\n\n disable_notification (``bool``, *optional*):\n Sends the message silently.\n Users will receive a notification with no sound.\n\n as_copy (``bool``, *optional*):\n Pass True to forward messages without the forward header (i.e.: send a copy of the message content).\n Defaults to False.\n\n remove_caption (``bool``, *optional*):\n If set to True and *as_copy* is enabled as well, media captions are not preserved when copying the\n message. Has no effect if *as_copy* is not enabled.\n Defaults to False.\n\n Returns:\n On success, the forwarded Message is returned.\n\n Raises:\n :class:`RPCError `\n \"\"\"\n if as_copy:\n if self.service:\n raise ValueError(\"Unable to copy service messages\")\n\n if self.game and not self._client.is_bot:\n raise ValueError(\"Users cannot send messages with Game media type\")\n\n # TODO: Improve markdown parser. Currently html appears to be more stable, thus we use it here because users\n # can\"t choose.\n\n if self.text:\n return self._client.send_message(\n chat_id,\n text=self.text.html,\n parse_mode=\"html\",\n disable_web_page_preview=not self.web_page,\n disable_notification=disable_notification\n )\n elif self.media:\n caption = self.caption.html if self.caption and not remove_caption else None\n\n send_media = partial(\n self._client.send_cached_media,\n chat_id=chat_id,\n disable_notification=disable_notification\n )\n\n if self.photo:\n file_id = self.photo.sizes[-1].file_id\n elif self.audio:\n file_id = self.audio.file_id\n elif self.document:\n file_id = self.document.file_id\n elif self.video:\n file_id = self.video.file_id\n elif self.animation:\n file_id = self.animation.file_id\n elif self.voice:\n file_id = self.voice.file_id\n elif self.sticker:\n file_id = self.sticker.file_id\n elif self.video_note:\n file_id = self.video_note.file_id\n elif self.contact:\n return self._client.send_contact(\n chat_id,\n phone_number=self.contact.phone_number,\n first_name=self.contact.first_name,\n last_name=self.contact.last_name,\n vcard=self.contact.vcard,\n disable_notification=disable_notification\n )\n elif self.location:\n return self._client.send_location(\n chat_id,\n latitude=self.location.latitude,\n longitude=self.location.longitude,\n disable_notification=disable_notification\n )\n elif self.venue:\n return self._client.send_venue(\n chat_id,\n latitude=self.venue.location.latitude,\n longitude=self.venue.location.longitude,\n title=self.venue.title,\n address=self.venue.address,\n foursquare_id=self.venue.foursquare_id,\n foursquare_type=self.venue.foursquare_type,\n disable_notification=disable_notification\n )\n elif self.poll:\n return self._client.send_poll(\n chat_id,\n question=self.poll.question,\n options=[opt.text for opt in self.poll.options],\n disable_notification=disable_notification\n )\n elif self.game:\n return self._client.send_game(\n chat_id,\n game_short_name=self.game.short_name,\n disable_notification=disable_notification\n )\n else:\n raise ValueError(\"Unknown media type\")\n\n if self.sticker or self.video_note: # Sticker and VideoNote should have no caption\n return send_media(file_id=file_id)\n else:\n return send_media(file_id=file_id, caption=caption, parse_mode=ParseMode.HTML)\n else:\n raise ValueError(\"Can't copy this message\")\n else:\n return self._client.forward_messages(\n chat_id=chat_id,\n from_chat_id=self.chat.id,\n message_ids=self.message_id,\n disable_notification=disable_notification\n )", "response": "Forward the message to the target chat."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef delete(self, revoke: bool = True):\n return self._client.delete_messages(\n chat_id=self.chat.id,\n message_ids=self.message_id,\n revoke=revoke\n )", "response": "Delete the message on the specified chat."} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nclick a button on the message.", "response": "def click(self, x: int or str, y: int = None, quote: bool = None):\n \"\"\"Bound method *click* of :obj:`Message `.\n\n Use as a shortcut for clicking a button attached to the message instead of.\n\n - Clicking inline buttons:\n\n .. code-block:: python\n\n client.request_callback_answer(\n chat_id=message.chat.id,\n message_id=message.message_id,\n callback_data=message.reply_markup[i][j].callback_data\n )\n\n - Clicking normal buttons:\n\n .. code-block:: python\n\n client.send_message(\n chat_id=message.chat.id,\n text=message.reply_markup[i][j].text\n )\n\n Example:\n This method can be used in three different ways:\n\n 1. Pass one integer argument only (e.g.: ``.click(2)``, to click a button at index 2).\n Buttons are counted left to right, starting from the top.\n\n 2. Pass two integer arguments (e.g.: ``.click(1, 0)``, to click a button at position (1, 0)).\n The origin (0, 0) is top-left.\n\n 3. Pass one string argument only (e.g.: ``.click(\"Settings\")``, to click a button by using its label).\n Only the first matching button will be pressed.\n\n Args:\n x (``int`` | ``str``):\n Used as integer index, integer abscissa (in pair with y) or as string label.\n\n y (``int``, *optional*):\n Used as ordinate only (in pair with x).\n\n quote (``bool``, *optional*):\n Useful for normal buttons only, where pressing it will result in a new message sent.\n If ``True``, the message will be sent as a reply to this message.\n Defaults to ``True`` in group chats and ``False`` in private chats.\n\n Returns:\n - The result of *request_callback_answer()* in case of inline callback button clicks.\n - The result of *reply()* in case of normal button clicks.\n - A string in case the inline button is an URL, switch_inline_query or switch_inline_query_current_chat\n button.\n\n Raises:\n :class:`RPCError `\n ``ValueError``: If the provided index or position is out of range or the button label was not found\n ``TimeoutError``: If, after clicking an inline button, the bot fails to answer within 10 seconds\n \"\"\"\n if isinstance(self.reply_markup, pyrogram.ReplyKeyboardMarkup):\n return self.reply(x, quote=quote)\n elif isinstance(self.reply_markup, pyrogram.InlineKeyboardMarkup):\n if isinstance(x, int) and y is None:\n try:\n button = [\n button\n for row in self.reply_markup.inline_keyboard\n for button in row\n ][x]\n except IndexError:\n raise ValueError(\"The button at index {} doesn't exist\".format(x)) from None\n elif isinstance(x, int) and isinstance(y, int):\n try:\n button = self.reply_markup.inline_keyboard[y][x]\n except IndexError:\n raise ValueError(\"The button at position ({}, {}) doesn't exist\".format(x, y)) from None\n elif isinstance(x, str):\n x = x.encode(\"utf-16\", \"surrogatepass\").decode(\"utf-16\")\n\n try:\n button = [\n button\n for row in self.reply_markup.inline_keyboard\n for button in row\n if x == button.text\n ][0]\n except IndexError:\n raise ValueError(\n \"The button with label '{}' doesn't exists\".format(\n x.encode(\"unicode_escape\").decode()\n )\n ) from None\n else:\n raise ValueError(\"Invalid arguments\")\n\n if button.callback_data:\n return self._client.request_callback_answer(\n chat_id=self.chat.id,\n message_id=self.message_id,\n callback_data=button.callback_data\n )\n elif button.url:\n return button.url\n elif button.switch_inline_query:\n return button.switch_inline_query\n elif button.switch_inline_query_current_chat:\n return button.switch_inline_query_current_chat\n else:\n raise ValueError(\"This button is not supported yet\")\n else:\n raise ValueError(\"The message doesn't contain any keyboard\")"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef download(\n self,\n file_name: str = \"\",\n block: bool = True,\n progress: callable = None,\n progress_args: tuple = ()\n ) -> \"Message\":\n \"\"\"Bound method *download* of :obj:`Message `.\n\n Use as a shortcut for:\n\n .. code-block:: python\n\n client.download_media(message)\n\n Example:\n .. code-block:: python\n\n message.download()\n\n Args:\n file_name (``str``, *optional*):\n A custom *file_name* to be used instead of the one provided by Telegram.\n By default, all files are downloaded in the *downloads* folder in your working directory.\n You can also specify a path for downloading files in a custom location: paths that end with \"/\"\n are considered directories. All non-existent folders will be created automatically.\n\n block (``bool``, *optional*):\n Blocks the code execution until the file has been downloaded.\n Defaults to True.\n\n progress (``callable``):\n Pass a callback function to view the download progress.\n The function must take *(client, current, total, \\*args)* as positional arguments (look at the section\n below for a detailed description).\n\n progress_args (``tuple``):\n Extra custom arguments for the progress callback function. Useful, for example, if you want to pass\n a chat_id and a message_id in order to edit a message with the updated progress.\n\n Returns:\n On success, the absolute path of the downloaded file as string is returned, None otherwise.\n\n Raises:\n :class:`RPCError `\n ``ValueError``: If the message doesn't contain any downloadable media\n \"\"\"\n return self._client.download_media(\n message=self,\n file_name=file_name,\n block=block,\n progress=progress,\n progress_args=progress_args,\n )", "response": "Download a file from the message."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef pin(self, disable_notification: bool = None) -> \"Message\":\n return self._client.pin_chat_message(\n chat_id=self.chat.id,\n message_id=self.message_id,\n disable_notification=disable_notification\n )", "response": "Pin the message to the chat."} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ngetting a user s dialogs.", "response": "def get_dialogs(\n self,\n offset_date: int = 0,\n limit: int = 100,\n pinned_only: bool = False\n ) -> \"pyrogram.Dialogs\":\n \"\"\"Use this method to get a chunk of the user's dialogs.\n\n You can get up to 100 dialogs at once.\n For a more convenient way of getting a user's dialogs see :meth:`iter_dialogs`.\n\n Args:\n offset_date (``int``):\n The offset date in Unix time taken from the top message of a :obj:`Dialog`.\n Defaults to 0. Valid for non-pinned dialogs only.\n\n limit (``str``, *optional*):\n Limits the number of dialogs to be retrieved.\n Defaults to 100. Valid for non-pinned dialogs only.\n\n pinned_only (``bool``, *optional*):\n Pass True if you want to get only pinned dialogs.\n Defaults to False.\n\n Returns:\n On success, a :obj:`Dialogs` object is returned.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n\n while True:\n try:\n if pinned_only:\n r = self.send(functions.messages.GetPinnedDialogs())\n else:\n r = self.send(\n functions.messages.GetDialogs(\n offset_date=offset_date,\n offset_id=0,\n offset_peer=types.InputPeerEmpty(),\n limit=limit,\n hash=0,\n exclude_pinned=True\n )\n )\n except FloodWait as e:\n log.warning(\"Sleeping {}s\".format(e.x))\n time.sleep(e.x)\n else:\n break\n\n return pyrogram.Dialogs._parse(self, r)"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_users(\n self,\n user_ids: Iterable[Union[int, str]]\n ) -> Union[\"pyrogram.User\", List[\"pyrogram.User\"]]:\n \"\"\"Use this method to get information about a user.\n You can retrieve up to 200 users at once.\n\n Args:\n user_ids (``iterable``):\n A list of User identifiers (id or username) or a single user id/username.\n For a contact that exists in your Telegram address book you can use his phone number (str).\n Iterators and Generators are also accepted.\n\n Returns:\n On success and in case *user_ids* was an iterable, the returned value will be a list of the requested\n :obj:`Users ` even if a list contains just one element, otherwise if\n *user_ids* was an integer or string, the single requested :obj:`User` is returned.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n is_iterable = not isinstance(user_ids, (int, str))\n user_ids = list(user_ids) if is_iterable else [user_ids]\n user_ids = [self.resolve_peer(i) for i in user_ids]\n\n r = self.send(\n functions.users.GetUsers(\n id=user_ids\n )\n )\n\n users = []\n\n for i in r:\n users.append(pyrogram.User._parse(self, i))\n\n return users if is_iterable else users[0]", "response": "Use this method to get information about a user."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef vote_poll(\n self,\n chat_id: Union[int, str],\n message_id: id,\n option: int\n ) -> bool:\n \"\"\"Use this method to vote a poll.\n\n Args:\n chat_id (``int`` | ``str``):\n Unique identifier (int) or username (str) of the target chat.\n For your personal cloud (Saved Messages) you can simply use \"me\" or \"self\".\n For a contact that exists in your Telegram address book you can use his phone number (str).\n\n message_id (``int``):\n Unique poll message identifier inside this chat.\n\n option (``int``):\n Index of the poll option you want to vote for (0 to 9).\n\n Returns:\n On success, True is returned.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n poll = self.get_messages(chat_id, message_id).poll\n\n self.send(\n functions.messages.SendVote(\n peer=self.resolve_peer(chat_id),\n msg_id=message_id,\n options=[poll.options[option].data]\n )\n )\n\n return True", "response": "Use this method to vote a poll."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef send_animation(\n self,\n chat_id: Union[int, str],\n animation: str,\n caption: str = \"\",\n parse_mode: str = \"\",\n duration: int = 0,\n width: int = 0,\n height: int = 0,\n thumb: str = None,\n disable_notification: bool = None,\n reply_to_message_id: int = None,\n reply_markup: Union[\n \"pyrogram.InlineKeyboardMarkup\",\n \"pyrogram.ReplyKeyboardMarkup\",\n \"pyrogram.ReplyKeyboardRemove\",\n \"pyrogram.ForceReply\"\n ] = None,\n progress: callable = None,\n progress_args: tuple = ()\n ) -> Union[\"pyrogram.Message\", None]:\n \"\"\"Use this method to send animation files (animation or H.264/MPEG-4 AVC video without sound).\n\n Args:\n chat_id (``int`` | ``str``):\n Unique identifier (int) or username (str) of the target chat.\n For your personal cloud (Saved Messages) you can simply use \"me\" or \"self\".\n For a contact that exists in your Telegram address book you can use his phone number (str).\n\n animation (``str``):\n Animation to send.\n Pass a file_id as string to send an animation that exists on the Telegram servers,\n pass an HTTP URL as a string for Telegram to get an animation from the Internet, or\n pass a file path as string to upload a new animation that exists on your local machine.\n\n caption (``str``, *optional*):\n Animation caption, 0-1024 characters.\n\n parse_mode (``str``, *optional*):\n Use :obj:`MARKDOWN ` or :obj:`HTML `\n if you want Telegram apps to show bold, italic, fixed-width text or inline URLs in your caption.\n Defaults to Markdown.\n\n duration (``int``, *optional*):\n Duration of sent animation in seconds.\n\n width (``int``, *optional*):\n Animation width.\n\n height (``int``, *optional*):\n Animation height.\n\n thumb (``str``, *optional*):\n Thumbnail of the animation file sent.\n The thumbnail should be in JPEG format and less than 200 KB in size.\n A thumbnail's width and height should not exceed 90 pixels.\n Thumbnails can't be reused and can be only uploaded as a new file.\n\n disable_notification (``bool``, *optional*):\n Sends the message silently.\n Users will receive a notification with no sound.\n\n reply_to_message_id (``int``, *optional*):\n If the message is a reply, ID of the original message.\n\n reply_markup (:obj:`InlineKeyboardMarkup` | :obj:`ReplyKeyboardMarkup` | :obj:`ReplyKeyboardRemove` | :obj:`ForceReply`, *optional*):\n Additional interface options. An object for an inline keyboard, custom reply keyboard,\n instructions to remove reply keyboard or to force a reply from the user.\n\n progress (``callable``, *optional*):\n Pass a callback function to view the upload progress.\n The function must take *(client, current, total, \\*args)* as positional arguments (look at the section\n below for a detailed description).\n\n progress_args (``tuple``, *optional*):\n Extra custom arguments for the progress callback function. Useful, for example, if you want to pass\n a chat_id and a message_id in order to edit a message with the updated progress.\n\n Other Parameters:\n client (:obj:`Client `):\n The Client itself, useful when you want to call other API methods inside the callback function.\n\n current (``int``):\n The amount of bytes uploaded so far.\n\n total (``int``):\n The size of the file.\n\n *args (``tuple``, *optional*):\n Extra custom arguments as defined in the *progress_args* parameter.\n You can either keep *\\*args* or add every single extra argument in your function signature.\n\n Returns:\n On success, the sent :obj:`Message ` is returned.\n In case the upload is deliberately stopped with :meth:`stop_transmission`, None is returned instead.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n file = None\n style = self.html if parse_mode.lower() == \"html\" else self.markdown\n\n try:\n if os.path.exists(animation):\n thumb = None if thumb is None else self.save_file(thumb)\n file = self.save_file(animation, progress=progress, progress_args=progress_args)\n media = types.InputMediaUploadedDocument(\n mime_type=self.guess_mime_type(animation) or \"video/mp4\",\n file=file,\n thumb=thumb,\n attributes=[\n types.DocumentAttributeVideo(\n supports_streaming=True,\n duration=duration,\n w=width,\n h=height\n ),\n types.DocumentAttributeFilename(file_name=os.path.basename(animation)),\n types.DocumentAttributeAnimated()\n ]\n )\n elif animation.startswith(\"http\"):\n media = types.InputMediaDocumentExternal(\n url=animation\n )\n else:\n try:\n decoded = utils.decode(animation)\n fmt = \" 24 else \" callable:\n \"\"\"Use this decorator to automatically register a function for handling raw updates.\n This does the same thing as :meth:`add_handler` using the :class:`RawUpdateHandler`.\n\n Args:\n group (``int``, *optional*):\n The group identifier, defaults to 0.\n \"\"\"\n\n def decorator(func: callable) -> Tuple[Handler, int]:\n if isinstance(func, tuple):\n func = func[0].callback\n\n handler = pyrogram.RawUpdateHandler(func)\n\n if isinstance(self, int):\n return handler, group if self is None else group\n\n if self is not None:\n self.add_handler(handler, group)\n\n return handler, group\n\n return decorator", "response": "Use this decorator to automatically register a function for handling raw updates."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef close_poll(\n self,\n chat_id: Union[int, str],\n message_id: id\n ) -> bool:\n \"\"\"Use this method to close (stop) a poll.\n\n Closed polls can't be reopened and nobody will be able to vote in it anymore.\n\n Args:\n chat_id (``int`` | ``str``):\n Unique identifier (int) or username (str) of the target chat.\n For your personal cloud (Saved Messages) you can simply use \"me\" or \"self\".\n For a contact that exists in your Telegram address book you can use his phone number (str).\n\n message_id (``int``):\n Unique poll message identifier inside this chat.\n\n Returns:\n On success, True is returned.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n poll = self.get_messages(chat_id, message_id).poll\n\n self.send(\n functions.messages.EditMessage(\n peer=self.resolve_peer(chat_id),\n id=message_id,\n media=types.InputMediaPoll(\n poll=types.Poll(\n id=poll.id,\n closed=True,\n question=\"\",\n answers=[]\n )\n )\n )\n )\n\n return True", "response": "Use this method to close a poll."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef send_inline_bot_result(\n self,\n chat_id: Union[int, str],\n query_id: int,\n result_id: str,\n disable_notification: bool = None,\n reply_to_message_id: int = None,\n hide_via: bool = None\n ):\n \"\"\"Use this method to send an inline bot result.\n Bot results can be retrieved using :obj:`get_inline_bot_results `\n\n Args:\n chat_id (``int`` | ``str``):\n Unique identifier (int) or username (str) of the target chat.\n For your personal cloud (Saved Messages) you can simply use \"me\" or \"self\".\n For a contact that exists in your Telegram address book you can use his phone number (str).\n\n query_id (``int``):\n Unique identifier for the answered query.\n\n result_id (``str``):\n Unique identifier for the result that was chosen.\n\n disable_notification (``bool``, *optional*):\n Sends the message silently.\n Users will receive a notification with no sound.\n\n reply_to_message_id (``bool``, *optional*):\n If the message is a reply, ID of the original message.\n\n hide_via (``bool``):\n Sends the message with *via @bot* hidden.\n\n Returns:\n On success, the sent Message is returned.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n return self.send(\n functions.messages.SendInlineBotResult(\n peer=self.resolve_peer(chat_id),\n query_id=query_id,\n id=result_id,\n random_id=self.rnd_id(),\n silent=disable_notification or None,\n reply_to_msg_id=reply_to_message_id,\n hide_via=hide_via or None\n )\n )", "response": "Use this method to send an inline bot result."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef send_cached_media(\n self,\n chat_id: Union[int, str],\n file_id: str,\n caption: str = \"\",\n parse_mode: str = \"\",\n disable_notification: bool = None,\n reply_to_message_id: int = None,\n reply_markup: Union[\n \"pyrogram.InlineKeyboardMarkup\",\n \"pyrogram.ReplyKeyboardMarkup\",\n \"pyrogram.ReplyKeyboardRemove\",\n \"pyrogram.ForceReply\"\n ] = None\n ) -> Union[\"pyrogram.Message\", None]:\n \"\"\"Use this method to send any media stored on the Telegram servers using a file_id.\n\n This convenience method works with any valid file_id only.\n It does the same as calling the relevant method for sending media using a file_id, thus saving you from the\n hassle of using the correct method for the media the file_id is pointing to.\n\n Args:\n chat_id (``int`` | ``str``):\n Unique identifier (int) or username (str) of the target chat.\n For your personal cloud (Saved Messages) you can simply use \"me\" or \"self\".\n For a contact that exists in your Telegram address book you can use his phone number (str).\n\n file_id (``str``):\n Media to send.\n Pass a file_id as string to send a media that exists on the Telegram servers.\n\n caption (``bool``, *optional*):\n Media caption, 0-1024 characters.\n\n parse_mode (``str``, *optional*):\n Use :obj:`MARKDOWN ` or :obj:`HTML `\n if you want Telegram apps to show bold, italic, fixed-width text or inline URLs in your caption.\n Defaults to Markdown.\n\n disable_notification (``bool``, *optional*):\n Sends the message silently.\n Users will receive a notification with no sound.\n\n reply_to_message_id (``int``, *optional*):\n If the message is a reply, ID of the original message.\n\n reply_markup (:obj:`InlineKeyboardMarkup` | :obj:`ReplyKeyboardMarkup` | :obj:`ReplyKeyboardRemove` | :obj:`ForceReply`, *optional*):\n Additional interface options. An object for an inline keyboard, custom reply keyboard,\n instructions to remove reply keyboard or to force a reply from the user.\n\n Returns:\n On success, the sent :obj:`Message ` is returned.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n style = self.html if parse_mode.lower() == \"html\" else self.markdown\n\n try:\n decoded = utils.decode(file_id)\n fmt = \" 24 else \"`.\n\n Args:\n chat_id (``int`` | ``str``):\n Unique identifier (int) or username (str) of the target chat.\n For your personal cloud (Saved Messages) you can simply use \"me\" or \"self\".\n For a contact that exists in your Telegram address book you can use his phone number (str).\n\n disable_notification (``bool``, *optional*):\n Sends messages silently.\n Users will receive a notification with no sound.\n\n as_copy (``bool``, *optional*):\n Pass True to forward messages without the forward header (i.e.: send a copy of the message content).\n Defaults to False.\n\n remove_caption (``bool``, *optional*):\n If set to True and *as_copy* is enabled as well, media captions are not preserved when copying the\n message. Has no effect if *as_copy* is not enabled.\n Defaults to False.\n\n Returns:\n On success, a :class:`Messages ` containing forwarded messages is returned.\n\n Raises:\n :class:`RPCError `\n \"\"\"\n forwarded_messages = []\n\n for message in self.messages:\n forwarded_messages.append(\n message.forward(\n chat_id=chat_id,\n as_copy=as_copy,\n disable_notification=disable_notification,\n remove_caption=remove_caption\n )\n )\n\n return Messages(\n total_count=len(forwarded_messages),\n messages=forwarded_messages,\n client=self._client\n )", "response": "Returns a new instance of the base class with forwarded messages."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef delete_messages(\n self,\n chat_id: Union[int, str],\n message_ids: Iterable[int],\n revoke: bool = True\n ) -> bool:\n \"\"\"Use this method to delete messages, including service messages.\n\n Args:\n chat_id (``int`` | ``str``):\n Unique identifier (int) or username (str) of the target chat.\n For your personal cloud (Saved Messages) you can simply use \"me\" or \"self\".\n For a contact that exists in your Telegram address book you can use his phone number (str).\n\n message_ids (``iterable``):\n A list of Message identifiers to delete or a single message id.\n Iterators and Generators are also accepted.\n\n revoke (``bool``, *optional*):\n Deletes messages on both parts.\n This is only for private cloud chats and normal groups, messages on\n channels and supergroups are always revoked (i.e.: deleted for everyone).\n Defaults to True.\n\n Returns:\n True on success, False otherwise.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n peer = self.resolve_peer(chat_id)\n message_ids = list(message_ids) if not isinstance(message_ids, int) else [message_ids]\n\n if isinstance(peer, types.InputPeerChannel):\n r = self.send(\n functions.channels.DeleteMessages(\n channel=peer,\n id=message_ids\n )\n )\n else:\n r = self.send(\n functions.messages.DeleteMessages(\n id=message_ids,\n revoke=revoke or None\n )\n )\n\n # Deleting messages you don't have right onto, won't raise any error.\n # Check for pts_count, which is 0 in case deletes fail.\n return bool(r.pts_count)", "response": "Use this method to delete messages from a chat."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nusing this method to change the description of a supergroup or a channel.", "response": "def set_chat_description(\n self,\n chat_id: Union[int, str],\n description: str\n ) -> bool:\n \"\"\"Use this method to change the description of a supergroup or a channel.\n You must be an administrator in the chat for this to work and must have the appropriate admin rights.\n\n Args:\n chat_id (``int`` | ``str``):\n Unique identifier (int) or username (str) of the target chat.\n\n description (``str``):\n New chat description, 0-255 characters.\n\n Returns:\n True on success.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n ``ValueError`` if a chat_id doesn't belong to a supergroup or a channel.\n \"\"\"\n peer = self.resolve_peer(chat_id)\n\n if isinstance(peer, (types.InputPeerChannel, types.InputPeerChat)):\n self.send(\n functions.messages.EditChatAbout(\n peer=peer,\n about=description\n )\n )\n else:\n raise ValueError(\"The chat_id \\\"{}\\\" belongs to a user\".format(chat_id))\n\n return True"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nuse this method to unban a previously kicked user in a supergroup or channel. The user will **not** return to the group or channel automatically, but will be able to join via link, etc. You must be an administrator for this to work. Args: chat_id (``int`` | ``str``): Unique identifier (int) or username (str) of the target chat. user_id (``int`` | ``str``): Unique identifier (int) or username (str) of the target user. For a contact that exists in your Telegram address book you can use his phone number (str). Returns: True on success. Raises: :class:`RPCError ` in case of a Telegram RPC error.", "response": "def unban_chat_member(\n self,\n chat_id: Union[int, str],\n user_id: Union[int, str]\n ) -> bool:\n \"\"\"Use this method to unban a previously kicked user in a supergroup or channel.\n The user will **not** return to the group or channel automatically, but will be able to join via link, etc.\n You must be an administrator for this to work.\n\n Args:\n chat_id (``int`` | ``str``):\n Unique identifier (int) or username (str) of the target chat.\n\n user_id (``int`` | ``str``):\n Unique identifier (int) or username (str) of the target user.\n For a contact that exists in your Telegram address book you can use his phone number (str).\n\n Returns:\n True on success.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n self.send(\n functions.channels.EditBanned(\n channel=self.resolve_peer(chat_id),\n user_id=self.resolve_peer(user_id),\n banned_rights=types.ChatBannedRights(\n until_date=0\n )\n )\n )\n\n return True"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef restrict_chat(\n self,\n chat_id: Union[int, str],\n can_send_messages: bool = False,\n can_send_media_messages: bool = False,\n can_send_other_messages: bool = False,\n can_add_web_page_previews: bool = False,\n can_send_polls: bool = False,\n can_change_info: bool = False,\n can_invite_users: bool = False,\n can_pin_messages: bool = False\n ) -> Chat:\n \"\"\"Use this method to restrict a chat.\n Pass True for all boolean parameters to lift restrictions from a chat.\n\n Args:\n chat_id (``int`` | ``str``):\n Unique identifier (int) or username (str) of the target chat.\n\n can_send_messages (``bool``, *optional*):\n Pass True, if the user can send text messages, contacts, locations and venues.\n\n can_send_media_messages (``bool``, *optional*):\n Pass True, if the user can send audios, documents, photos, videos, video notes and voice notes,\n implies can_send_messages.\n\n can_send_other_messages (``bool``, *optional*):\n Pass True, if the user can send animations, games, stickers and use inline bots,\n implies can_send_media_messages.\n\n can_add_web_page_previews (``bool``, *optional*):\n Pass True, if the user may add web page previews to their messages, implies can_send_media_messages.\n\n can_send_polls (``bool``, *optional*):\n Pass True, if the user can send polls, implies can_send_media_messages.\n\n can_change_info (``bool``, *optional*):\n Pass True, if the user can change the chat title, photo and other settings.\n\n can_invite_users (``bool``, *optional*):\n Pass True, if the user can invite new users to the chat.\n\n can_pin_messages (``bool``, *optional*):\n Pass True, if the user can pin messages.\n\n Returns:\n On success, a :obj:`Chat ` object is returned.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n send_messages = True\n send_media = True\n send_stickers = True\n send_gifs = True\n send_games = True\n send_inline = True\n embed_links = True\n send_polls = True\n change_info = True\n invite_users = True\n pin_messages = True\n\n if can_send_messages:\n send_messages = None\n\n if can_send_media_messages:\n send_messages = None\n send_media = None\n\n if can_send_other_messages:\n send_messages = None\n send_media = None\n send_stickers = None\n send_gifs = None\n send_games = None\n send_inline = None\n\n if can_add_web_page_previews:\n send_messages = None\n send_media = None\n embed_links = None\n\n if can_send_polls:\n send_messages = None\n send_polls = None\n\n if can_change_info:\n change_info = None\n\n if can_invite_users:\n invite_users = None\n\n if can_pin_messages:\n pin_messages = None\n\n r = self.send(\n functions.messages.EditChatDefaultBannedRights(\n peer=self.resolve_peer(chat_id),\n banned_rights=types.ChatBannedRights(\n until_date=0,\n send_messages=send_messages,\n send_media=send_media,\n send_stickers=send_stickers,\n send_gifs=send_gifs,\n send_games=send_games,\n send_inline=send_inline,\n embed_links=embed_links,\n send_polls=send_polls,\n change_info=change_info,\n invite_users=invite_users,\n pin_messages=pin_messages\n )\n )\n )\n\n return Chat._parse_chat(self, r.chats[0])", "response": "Use this method to lift restrictions from a chat."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nuses this method to delete a chat photo.", "response": "def delete_chat_photo(\n self,\n chat_id: Union[int, str]\n ) -> bool:\n \"\"\"Use this method to delete a chat photo.\n Photos can't be changed for private chats.\n You must be an administrator in the chat for this to work and must have the appropriate admin rights.\n\n Note:\n In regular groups (non-supergroups), this method will only work if the \"All Members Are Admins\"\n setting is off.\n\n Args:\n chat_id (``int`` | ``str``):\n Unique identifier (int) or username (str) of the target chat.\n\n Returns:\n True on success.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n ``ValueError`` if a chat_id belongs to user.\n \"\"\"\n peer = self.resolve_peer(chat_id)\n\n if isinstance(peer, types.InputPeerChat):\n self.send(\n functions.messages.EditChatPhoto(\n chat_id=peer.chat_id,\n photo=types.InputChatPhotoEmpty()\n )\n )\n elif isinstance(peer, types.InputPeerChannel):\n self.send(\n functions.channels.EditPhoto(\n channel=peer,\n photo=types.InputChatPhotoEmpty()\n )\n )\n else:\n raise ValueError(\"The chat_id \\\"{}\\\" belongs to a user\".format(chat_id))\n\n return True"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nusing this method to add contacts to your Telegram address book.", "response": "def add_contacts(\n self,\n contacts: List[\"pyrogram.InputPhoneContact\"]\n ):\n \"\"\"Use this method to add contacts to your Telegram address book.\n\n Args:\n contacts (List of :obj:`InputPhoneContact `):\n The contact list to be added\n\n Returns:\n On success, the added contacts are returned.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n imported_contacts = self.send(\n functions.contacts.ImportContacts(\n contacts=contacts\n )\n )\n\n return imported_contacts"} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nsort the arguments in order to be used by the flags module.", "response": "def sort_args(args):\n \"\"\"Put flags at the end\"\"\"\n args = args.copy()\n flags = [i for i in args if FLAGS_RE.match(i[1])]\n\n for i in flags:\n args.remove(i)\n\n return args + flags"} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nuse this method to iterate through a user's dialogs sequentially. This convenience method does the same as repeatedly calling :meth:`get_dialogs` in a loop, thus saving you from the hassle of setting up boilerplate code. It is useful for getting the whole dialogs list with a single call. Args: offset_date (``int``): The offset date in Unix time taken from the top message of a :obj:`Dialog`. Defaults to 0 (most recent dialog). limit (``str``, *optional*): Limits the number of dialogs to be retrieved. By default, no limit is applied and all dialogs are returned. Returns: A generator yielding :obj:`Dialog ` objects. Raises: :class:`RPCError ` in case of a Telegram RPC error.", "response": "def iter_dialogs(\n self,\n offset_date: int = 0,\n limit: int = 0\n ) -> Generator[\"pyrogram.Dialog\", None, None]:\n \"\"\"Use this method to iterate through a user's dialogs sequentially.\n\n This convenience method does the same as repeatedly calling :meth:`get_dialogs` in a loop, thus saving you from\n the hassle of setting up boilerplate code. It is useful for getting the whole dialogs list with a single call.\n\n Args:\n offset_date (``int``):\n The offset date in Unix time taken from the top message of a :obj:`Dialog`.\n Defaults to 0 (most recent dialog).\n\n limit (``str``, *optional*):\n Limits the number of dialogs to be retrieved.\n By default, no limit is applied and all dialogs are returned.\n\n Returns:\n A generator yielding :obj:`Dialog ` objects.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n current = 0\n total = limit or (1 << 31) - 1\n limit = min(100, total)\n\n pinned_dialogs = self.get_dialogs(\n pinned_only=True\n ).dialogs\n\n for dialog in pinned_dialogs:\n yield dialog\n\n current += 1\n\n if current >= total:\n return\n\n while True:\n dialogs = self.get_dialogs(\n offset_date=offset_date,\n limit=limit\n ).dialogs\n\n if not dialogs:\n return\n\n offset_date = dialogs[-1].top_message.date\n\n for dialog in dialogs:\n yield dialog\n\n current += 1\n\n if current >= total:\n return"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef export_chat_invite_link(\n self,\n chat_id: Union[int, str]\n ) -> str:\n \"\"\"Use this method to generate a new invite link for a chat; any previously generated link is revoked.\n\n You must be an administrator in the chat for this to work and have the appropriate admin rights.\n\n Args:\n chat_id (``int`` | ``str``):\n Unique identifier for the target chat or username of the target channel/supergroup\n (in the format @username).\n\n Returns:\n On success, the exported invite link as string is returned.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n peer = self.resolve_peer(chat_id)\n\n if isinstance(peer, types.InputPeerChat):\n return self.send(\n functions.messages.ExportChatInvite(\n peer=peer.chat_id\n )\n ).link\n elif isinstance(peer, types.InputPeerChannel):\n return self.send(\n functions.channels.ExportInvite(\n channel=peer\n )\n ).link", "response": "Use this method to generate a new invite link for a chat."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef remove_cloud_password(\n self,\n password: str\n ) -> bool:\n \"\"\"Use this method to turn off the Two-Step Verification security feature (Cloud Password) on your account.\n\n Args:\n password (``str``):\n Your current password.\n\n Returns:\n True on success.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n ``ValueError`` in case there is no cloud password to remove.\n \"\"\"\n r = self.send(functions.account.GetPassword())\n\n if not r.has_password:\n raise ValueError(\"There is no cloud password to remove\")\n\n self.send(\n functions.account.UpdatePasswordSettings(\n password=compute_check(r, password),\n new_settings=types.account.PasswordInputSettings(\n new_algo=types.PasswordKdfAlgoUnknown(),\n new_password_hash=b\"\",\n hint=\"\"\n )\n )\n )\n\n return True", "response": "Remove a cloud password from the user s account."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef send_poll(\n self,\n chat_id: Union[int, str],\n question: str,\n options: List[str],\n disable_notification: bool = None,\n reply_to_message_id: int = None,\n reply_markup: Union[\n \"pyrogram.InlineKeyboardMarkup\",\n \"pyrogram.ReplyKeyboardMarkup\",\n \"pyrogram.ReplyKeyboardRemove\",\n \"pyrogram.ForceReply\"\n ] = None\n ) -> \"pyrogram.Message\":\n \"\"\"Use this method to send a new poll.\n\n Args:\n chat_id (``int`` | ``str``):\n Unique identifier (int) or username (str) of the target chat.\n For your personal cloud (Saved Messages) you can simply use \"me\" or \"self\".\n For a contact that exists in your Telegram address book you can use his phone number (str).\n\n question (``str``):\n The poll question, as string.\n\n options (List of ``str``):\n The poll options, as list of strings (2 to 10 options are allowed).\n\n disable_notification (``bool``, *optional*):\n Sends the message silently.\n Users will receive a notification with no sound.\n\n reply_to_message_id (``int``, *optional*):\n If the message is a reply, ID of the original message.\n\n reply_markup (:obj:`InlineKeyboardMarkup` | :obj:`ReplyKeyboardMarkup` | :obj:`ReplyKeyboardRemove` | :obj:`ForceReply`, *optional*):\n Additional interface options. An object for an inline keyboard, custom reply keyboard,\n instructions to remove reply keyboard or to force a reply from the user.\n\n Returns:\n On success, the sent :obj:`Message ` is returned.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n r = self.send(\n functions.messages.SendMedia(\n peer=self.resolve_peer(chat_id),\n media=types.InputMediaPoll(\n poll=types.Poll(\n id=0,\n question=question,\n answers=[\n types.PollAnswer(text=o, option=bytes([i]))\n for i, o in enumerate(options)\n ]\n )\n ),\n message=\"\",\n silent=disable_notification or None,\n reply_to_msg_id=reply_to_message_id,\n random_id=self.rnd_id(),\n reply_markup=reply_markup.write() if reply_markup else None\n )\n )\n\n for i in r.updates:\n if isinstance(i, (types.UpdateNewMessage, types.UpdateNewChannelMessage)):\n return pyrogram.Message._parse(\n self, i.message,\n {i.id: i for i in r.users},\n {i.id: i for i in r.chats}\n )", "response": "Send a new poll."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef delete_contacts(\n self,\n ids: List[int]\n ):\n \"\"\"Use this method to delete contacts from your Telegram address book.\n\n Args:\n ids (List of ``int``):\n A list of unique identifiers for the target users.\n Can be an ID (int), a username (string) or phone number (string).\n\n Returns:\n True on success.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n contacts = []\n\n for i in ids:\n try:\n input_user = self.resolve_peer(i)\n except PeerIdInvalid:\n continue\n else:\n if isinstance(input_user, types.InputPeerUser):\n contacts.append(input_user)\n\n return self.send(\n functions.contacts.DeleteContacts(\n id=contacts\n )\n )", "response": "Use this method to delete contacts from your Telegram address book."} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nuse this method to send a group of photos or videos to a chat.", "response": "def send_media_group(\n self,\n chat_id: Union[int, str],\n media: List[Union[\"pyrogram.InputMediaPhoto\", \"pyrogram.InputMediaVideo\"]],\n disable_notification: bool = None,\n reply_to_message_id: int = None\n ):\n \"\"\"Use this method to send a group of photos or videos as an album.\n\n Args:\n chat_id (``int`` | ``str``):\n Unique identifier (int) or username (str) of the target chat.\n For your personal cloud (Saved Messages) you can simply use \"me\" or \"self\".\n For a contact that exists in your Telegram address book you can use his phone number (str).\n\n media (List of :obj:`InputMediaPhoto` and :obj:`InputMediaVideo`):\n A list describing photos and videos to be sent, must include 2\u201310 items.\n\n disable_notification (``bool``, *optional*):\n Sends the message silently.\n Users will receive a notification with no sound.\n\n reply_to_message_id (``int``, *optional*):\n If the message is a reply, ID of the original message.\n\n Returns:\n On success, a :obj:`Messages ` object is returned containing all the\n single messages sent.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n multi_media = []\n\n for i in media:\n style = self.html if i.parse_mode.lower() == \"html\" else self.markdown\n\n if isinstance(i, pyrogram.InputMediaPhoto):\n if os.path.exists(i.media):\n while True:\n try:\n media = self.send(\n functions.messages.UploadMedia(\n peer=self.resolve_peer(chat_id),\n media=types.InputMediaUploadedPhoto(\n file=self.save_file(i.media)\n )\n )\n )\n except FloodWait as e:\n log.warning(\"Sleeping for {}s\".format(e.x))\n time.sleep(e.x)\n else:\n break\n\n media = types.InputMediaPhoto(\n id=types.InputPhoto(\n id=media.photo.id,\n access_hash=media.photo.access_hash,\n file_reference=b\"\"\n )\n )\n else:\n try:\n decoded = utils.decode(i.media)\n fmt = \" 24 else \" 24 else \"`,\n otherwise returns True.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n :class:`BotScoreNotModified` if the new score is not greater than the user's current score in the chat and force is False.\n \"\"\"\n r = self.send(\n functions.messages.SetGameScore(\n peer=self.resolve_peer(chat_id),\n score=score,\n id=message_id,\n user_id=self.resolve_peer(user_id),\n force=force or None,\n edit_message=not disable_edit_message or None\n )\n )\n\n for i in r.updates:\n if isinstance(i, (types.UpdateEditMessage, types.UpdateEditChannelMessage)):\n return pyrogram.Message._parse(\n self, i.message,\n {i.id: i for i in r.users},\n {i.id: i for i in r.chats}\n )\n\n return True", "response": "This method is used to set the score of a user in a game."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef pin_chat_message(\n self,\n chat_id: Union[int, str],\n message_id: int,\n disable_notification: bool = None\n ) -> bool:\n \"\"\"Use this method to pin a message in a group, channel or your own chat.\n You must be an administrator in the chat for this to work and must have the \"can_pin_messages\" admin right in\n the supergroup or \"can_edit_messages\" admin right in the channel.\n\n Args:\n chat_id (``int`` | ``str``):\n Unique identifier (int) or username (str) of the target chat.\n\n message_id (``int``):\n Identifier of a message to pin.\n\n disable_notification (``bool``):\n Pass True, if it is not necessary to send a notification to all chat members about the new pinned\n message. Notifications are always disabled in channels.\n\n Returns:\n True on success.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n self.send(\n functions.messages.UpdatePinnedMessage(\n peer=self.resolve_peer(chat_id),\n id=message_id,\n silent=disable_notification or None\n )\n )\n\n return True", "response": "Use this method to pin a message in a chat."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nuse this method to kick a user from a group supergroup or channel.", "response": "def kick_chat_member(\n self,\n chat_id: Union[int, str],\n user_id: Union[int, str],\n until_date: int = 0\n ) -> Union[\"pyrogram.Message\", bool]:\n \"\"\"Use this method to kick a user from a group, a supergroup or a channel.\n In the case of supergroups and channels, the user will not be able to return to the group on their own using\n invite links, etc., unless unbanned first. You must be an administrator in the chat for this to work and must\n have the appropriate admin rights.\n\n Note:\n In regular groups (non-supergroups), this method will only work if the \"All Members Are Admins\" setting is\n off in the target group. Otherwise members may only be removed by the group's creator or by the member\n that added them.\n\n Args:\n chat_id (``int`` | ``str``):\n Unique identifier (int) or username (str) of the target chat.\n\n user_id (``int`` | ``str``):\n Unique identifier (int) or username (str) of the target user.\n For a contact that exists in your Telegram address book you can use his phone number (str).\n\n until_date (``int``, *optional*):\n Date when the user will be unbanned, unix time.\n If user is banned for more than 366 days or less than 30 seconds from the current time they are\n considered to be banned forever. Defaults to 0 (ban forever).\n\n Returns:\n On success, either True or a service :obj:`Message ` will be returned (when applicable).\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n chat_peer = self.resolve_peer(chat_id)\n user_peer = self.resolve_peer(user_id)\n\n if isinstance(chat_peer, types.InputPeerChannel):\n r = self.send(\n functions.channels.EditBanned(\n channel=chat_peer,\n user_id=user_peer,\n banned_rights=types.ChatBannedRights(\n until_date=until_date,\n view_messages=True,\n send_messages=True,\n send_media=True,\n send_stickers=True,\n send_gifs=True,\n send_games=True,\n send_inline=True,\n embed_links=True\n )\n )\n )\n else:\n r = self.send(\n functions.messages.DeleteChatUser(\n chat_id=abs(chat_id),\n user_id=user_peer\n )\n )\n\n for i in r.updates:\n if isinstance(i, (types.UpdateNewMessage, types.UpdateNewChannelMessage)):\n return pyrogram.Message._parse(\n self, i.message,\n {i.id: i for i in r.users},\n {i.id: i for i in r.chats}\n )\n else:\n return True"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef forward_messages(\n self,\n chat_id: Union[int, str],\n from_chat_id: Union[int, str],\n message_ids: Iterable[int],\n disable_notification: bool = None,\n as_copy: bool = False,\n remove_caption: bool = False\n ) -> \"pyrogram.Messages\":\n \"\"\"Use this method to forward messages of any kind.\n\n Args:\n chat_id (``int`` | ``str``):\n Unique identifier (int) or username (str) of the target chat.\n For your personal cloud (Saved Messages) you can simply use \"me\" or \"self\".\n For a contact that exists in your Telegram address book you can use his phone number (str).\n\n from_chat_id (``int`` | ``str``):\n Unique identifier (int) or username (str) of the source chat where the original message was sent.\n For your personal cloud (Saved Messages) you can simply use \"me\" or \"self\".\n For a contact that exists in your Telegram address book you can use his phone number (str).\n\n message_ids (``iterable``):\n A list of Message identifiers in the chat specified in *from_chat_id* or a single message id.\n Iterators and Generators are also accepted.\n\n disable_notification (``bool``, *optional*):\n Sends the message silently.\n Users will receive a notification with no sound.\n\n as_copy (``bool``, *optional*):\n Pass True to forward messages without the forward header (i.e.: send a copy of the message content).\n Defaults to False.\n\n remove_caption (``bool``, *optional*):\n If set to True and *as_copy* is enabled as well, media captions are not preserved when copying the\n message. Has no effect if *as_copy* is not enabled.\n Defaults to False.\n\n Returns:\n On success and in case *message_ids* was an iterable, the returned value will be a list of the forwarded\n :obj:`Messages ` even if a list contains just one element, otherwise if\n *message_ids* was an integer, the single forwarded :obj:`Message `\n is returned.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n\n is_iterable = not isinstance(message_ids, int)\n message_ids = list(message_ids) if is_iterable else [message_ids]\n\n if as_copy:\n forwarded_messages = []\n\n for chunk in [message_ids[i:i + 200] for i in range(0, len(message_ids), 200)]:\n messages = self.get_messages(chat_id=from_chat_id, message_ids=chunk) # type: pyrogram.Messages\n\n for message in messages.messages:\n forwarded_messages.append(\n message.forward(\n chat_id,\n disable_notification=disable_notification,\n as_copy=True,\n remove_caption=remove_caption\n )\n )\n\n return pyrogram.Messages(\n client=self,\n total_count=len(forwarded_messages),\n messages=forwarded_messages\n ) if is_iterable else forwarded_messages[0]\n else:\n r = self.send(\n functions.messages.ForwardMessages(\n to_peer=self.resolve_peer(chat_id),\n from_peer=self.resolve_peer(from_chat_id),\n id=message_ids,\n silent=disable_notification or None,\n random_id=[self.rnd_id() for _ in message_ids]\n )\n )\n\n forwarded_messages = []\n\n users = {i.id: i for i in r.users}\n chats = {i.id: i for i in r.chats}\n\n for i in r.updates:\n if isinstance(i, (types.UpdateNewMessage, types.UpdateNewChannelMessage)):\n forwarded_messages.append(\n pyrogram.Message._parse(\n self, i.message,\n users, chats\n )\n )\n\n return pyrogram.Messages(\n client=self,\n total_count=len(forwarded_messages),\n messages=forwarded_messages\n ) if is_iterable else forwarded_messages[0]", "response": "This method is used to forward messages from one chat to another."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef download_media(\n self,\n message: Union[\"pyrogram.Message\", str],\n file_name: str = \"\",\n block: bool = True,\n progress: callable = None,\n progress_args: tuple = ()\n ) -> Union[str, None]:\n \"\"\"Use this method to download the media from a message.\n\n Args:\n message (:obj:`Message ` | ``str``):\n Pass a Message containing the media, the media itself (message.audio, message.video, ...) or\n the file id as string.\n\n file_name (``str``, *optional*):\n A custom *file_name* to be used instead of the one provided by Telegram.\n By default, all files are downloaded in the *downloads* folder in your working directory.\n You can also specify a path for downloading files in a custom location: paths that end with \"/\"\n are considered directories. All non-existent folders will be created automatically.\n\n block (``bool``, *optional*):\n Blocks the code execution until the file has been downloaded.\n Defaults to True.\n\n progress (``callable``):\n Pass a callback function to view the download progress.\n The function must take *(client, current, total, \\*args)* as positional arguments (look at the section\n below for a detailed description).\n\n progress_args (``tuple``):\n Extra custom arguments for the progress callback function. Useful, for example, if you want to pass\n a chat_id and a message_id in order to edit a message with the updated progress.\n\n Other Parameters:\n client (:obj:`Client `):\n The Client itself, useful when you want to call other API methods inside the callback function.\n\n current (``int``):\n The amount of bytes downloaded so far.\n\n total (``int``):\n The size of the file.\n\n *args (``tuple``, *optional*):\n Extra custom arguments as defined in the *progress_args* parameter.\n You can either keep *\\*args* or add every single extra argument in your function signature.\n\n Returns:\n On success, the absolute path of the downloaded file as string is returned, None otherwise.\n In case the download is deliberately stopped with :meth:`stop_transmission`, None is returned as well.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n ``ValueError`` if the message doesn't contain any downloadable media\n \"\"\"\n error_message = \"This message doesn't contain any downloadable media\"\n\n if isinstance(message, pyrogram.Message):\n if message.photo:\n media = pyrogram.Document(\n file_id=message.photo.sizes[-1].file_id,\n file_size=message.photo.sizes[-1].file_size,\n mime_type=\"\",\n date=message.photo.date,\n client=self\n )\n elif message.audio:\n media = message.audio\n elif message.document:\n media = message.document\n elif message.video:\n media = message.video\n elif message.voice:\n media = message.voice\n elif message.video_note:\n media = message.video_note\n elif message.sticker:\n media = message.sticker\n elif message.animation:\n media = message.animation\n else:\n raise ValueError(error_message)\n elif isinstance(message, (\n pyrogram.Photo,\n pyrogram.PhotoSize,\n pyrogram.Audio,\n pyrogram.Document,\n pyrogram.Video,\n pyrogram.Voice,\n pyrogram.VideoNote,\n pyrogram.Sticker,\n pyrogram.Animation\n )):\n if isinstance(message, pyrogram.Photo):\n media = pyrogram.Document(\n file_id=message.sizes[-1].file_id,\n file_size=message.sizes[-1].file_size,\n mime_type=\"\",\n date=message.date,\n client=self\n )\n else:\n media = message\n elif isinstance(message, str):\n media = pyrogram.Document(\n file_id=message,\n file_size=0,\n mime_type=\"\",\n client=self\n )\n else:\n raise ValueError(error_message)\n\n done = Event()\n path = [None]\n\n self.download_queue.put((media, file_name, done, progress, progress_args, path))\n\n if block:\n done.wait()\n\n return path[0]", "response": "Download a media from a message."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nuses this method to send a message to the Telegram address book.", "response": "def send_message(\n self,\n chat_id: Union[int, str],\n text: str,\n parse_mode: str = \"\",\n disable_web_page_preview: bool = None,\n disable_notification: bool = None,\n reply_to_message_id: int = None,\n reply_markup: Union[\n \"pyrogram.InlineKeyboardMarkup\",\n \"pyrogram.ReplyKeyboardMarkup\",\n \"pyrogram.ReplyKeyboardRemove\",\n \"pyrogram.ForceReply\"\n ] = None\n ) -> \"pyrogram.Message\":\n \"\"\"Use this method to send text messages.\n\n Args:\n chat_id (``int`` | ``str``):\n Unique identifier (int) or username (str) of the target chat.\n For your personal cloud (Saved Messages) you can simply use \"me\" or \"self\".\n For a contact that exists in your Telegram address book you can use his phone number (str).\n\n text (``str``):\n Text of the message to be sent.\n\n parse_mode (``str``, *optional*):\n Use :obj:`MARKDOWN ` or :obj:`HTML `\n if you want Telegram apps to show bold, italic, fixed-width text or inline URLs in your message.\n Defaults to Markdown.\n\n disable_web_page_preview (``bool``, *optional*):\n Disables link previews for links in this message.\n\n disable_notification (``bool``, *optional*):\n Sends the message silently.\n Users will receive a notification with no sound.\n\n reply_to_message_id (``int``, *optional*):\n If the message is a reply, ID of the original message.\n\n reply_markup (:obj:`InlineKeyboardMarkup` | :obj:`ReplyKeyboardMarkup` | :obj:`ReplyKeyboardRemove` | :obj:`ForceReply`, *optional*):\n Additional interface options. An object for an inline keyboard, custom reply keyboard,\n instructions to remove reply keyboard or to force a reply from the user.\n\n Returns:\n On success, the sent :obj:`Message` is returned.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n style = self.html if parse_mode.lower() == \"html\" else self.markdown\n message, entities = style.parse(text).values()\n\n r = self.send(\n functions.messages.SendMessage(\n peer=self.resolve_peer(chat_id),\n no_webpage=disable_web_page_preview or None,\n silent=disable_notification or None,\n reply_to_msg_id=reply_to_message_id,\n random_id=self.rnd_id(),\n reply_markup=reply_markup.write() if reply_markup else None,\n message=message,\n entities=entities\n )\n )\n\n if isinstance(r, types.UpdateShortSentMessage):\n peer = self.resolve_peer(chat_id)\n\n peer_id = (\n peer.user_id\n if isinstance(peer, types.InputPeerUser)\n else -peer.chat_id\n )\n\n return pyrogram.Message(\n message_id=r.id,\n chat=pyrogram.Chat(\n id=peer_id,\n type=\"private\",\n client=self\n ),\n text=message,\n date=r.date,\n outgoing=r.out,\n entities=entities,\n client=self\n )\n\n for i in r.updates:\n if isinstance(i, (types.UpdateNewMessage, types.UpdateNewChannelMessage)):\n return pyrogram.Message._parse(\n self, i.message,\n {i.id: i for i in r.users},\n {i.id: i for i in r.chats}\n )"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nuses this method to edit audio document photo or video messages.", "response": "def edit_message_media(\n self,\n chat_id: Union[int, str],\n message_id: int,\n media: InputMedia,\n reply_markup: \"pyrogram.InlineKeyboardMarkup\" = None\n ) -> \"pyrogram.Message\":\n \"\"\"Use this method to edit audio, document, photo, or video messages.\n\n If a message is a part of a message album, then it can be edited only to a photo or a video. Otherwise,\n message type can be changed arbitrarily. When inline message is edited, new file can't be uploaded.\n Use previously uploaded file via its file_id or specify a URL. On success, if the edited message was sent\n by the bot, the edited Message is returned, otherwise True is returned.\n\n Args:\n chat_id (``int`` | ``str``):\n Unique identifier (int) or username (str) of the target chat.\n For your personal cloud (Saved Messages) you can simply use \"me\" or \"self\".\n For a contact that exists in your Telegram address book you can use his phone number (str).\n\n message_id (``int``):\n Message identifier in the chat specified in chat_id.\n\n media (:obj:`InputMedia`)\n One of the InputMedia objects describing an animation, audio, document, photo or video.\n\n reply_markup (:obj:`InlineKeyboardMarkup`, *optional*):\n An InlineKeyboardMarkup object.\n\n Returns:\n On success, the edited :obj:`Message ` is returned.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n style = self.html if media.parse_mode.lower() == \"html\" else self.markdown\n caption = media.caption\n\n if isinstance(media, InputMediaPhoto):\n if os.path.exists(media.media):\n media = self.send(\n functions.messages.UploadMedia(\n peer=self.resolve_peer(chat_id),\n media=types.InputMediaUploadedPhoto(\n file=self.save_file(media.media)\n )\n )\n )\n\n media = types.InputMediaPhoto(\n id=types.InputPhoto(\n id=media.photo.id,\n access_hash=media.photo.access_hash,\n file_reference=b\"\"\n )\n )\n elif media.media.startswith(\"http\"):\n media = types.InputMediaPhotoExternal(\n url=media.media\n )\n else:\n try:\n decoded = utils.decode(media.media)\n fmt = \" 24 else \" 24 else \" 24 else \" 24 else \" 24 else \" bool:\n \"\"\"Use this method to enable the Two-Step Verification security feature (Cloud Password) on your account.\n\n This password will be asked when you log-in on a new device in addition to the SMS code.\n\n Args:\n password (``str``):\n Your password.\n\n hint (``str``, *optional*):\n A password hint.\n\n email (``str``, *optional*):\n Recovery e-mail.\n\n Returns:\n True on success.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n ``ValueError`` in case there is already a cloud password enabled.\n \"\"\"\n r = self.send(functions.account.GetPassword())\n\n if r.has_password:\n raise ValueError(\"There is already a cloud password enabled\")\n\n r.new_algo.salt1 += os.urandom(32)\n new_hash = btoi(compute_hash(r.new_algo, password))\n new_hash = itob(pow(r.new_algo.g, new_hash, btoi(r.new_algo.p)))\n\n self.send(\n functions.account.UpdatePasswordSettings(\n password=types.InputCheckPasswordEmpty(),\n new_settings=types.account.PasswordInputSettings(\n new_algo=r.new_algo,\n new_password_hash=new_hash,\n hint=hint,\n email=email\n )\n )\n )\n\n return True"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_chat_preview(\n self,\n invite_link: str\n ):\n \"\"\"Use this method to get the preview of a chat using the invite link.\n\n This method only returns a chat preview, if you want to join a chat use :meth:`join_chat`\n\n Args:\n invite_link (``str``):\n Unique identifier for the target chat in form of *t.me/joinchat/* links.\n\n Returns:\n Either :obj:`Chat` or :obj:`ChatPreview`, depending on whether you already joined the chat or not.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n ``ValueError`` in case of an invalid invite_link.\n \"\"\"\n match = self.INVITE_LINK_RE.match(invite_link)\n\n if match:\n r = self.send(\n functions.messages.CheckChatInvite(\n hash=match.group(1)\n )\n )\n\n if isinstance(r, types.ChatInvite):\n return pyrogram.ChatPreview._parse(self, r)\n\n if isinstance(r, types.ChatInviteAlready):\n chat = r.chat\n\n if isinstance(chat, types.Chat):\n return pyrogram.Chat._parse_chat_chat(self, chat)\n\n if isinstance(chat, types.Channel):\n return pyrogram.Chat._parse_channel_chat(self, chat)\n else:\n raise ValueError(\"The invite_link is invalid\")", "response": "Use this method to get the preview of a chat using the invite link."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nuse this method to change the title of a private chats.", "response": "def set_chat_title(\n self,\n chat_id: Union[int, str],\n title: str\n ) -> bool:\n \"\"\"Use this method to change the title of a chat.\n Titles can't be changed for private chats.\n You must be an administrator in the chat for this to work and must have the appropriate admin rights.\n\n Note:\n In regular groups (non-supergroups), this method will only work if the \"All Members Are Admins\"\n setting is off.\n\n Args:\n chat_id (``int`` | ``str``):\n Unique identifier (int) or username (str) of the target chat.\n\n title (``str``):\n New chat title, 1-255 characters.\n\n Returns:\n True on success.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n ``ValueError`` if a chat_id belongs to user.\n \"\"\"\n peer = self.resolve_peer(chat_id)\n\n if isinstance(peer, types.InputPeerChat):\n self.send(\n functions.messages.EditChatTitle(\n chat_id=peer.chat_id,\n title=title\n )\n )\n elif isinstance(peer, types.InputPeerChannel):\n self.send(\n functions.channels.EditTitle(\n channel=peer,\n title=title\n )\n )\n else:\n raise ValueError(\"The chat_id \\\"{}\\\" belongs to a user\".format(chat_id))\n\n return True"} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nuses this method to get data for high score tables.", "response": "def get_game_high_scores(\n self,\n user_id: Union[int, str],\n chat_id: Union[int, str],\n message_id: int = None\n ):\n \"\"\"Use this method to get data for high score tables.\n\n Args:\n user_id (``int`` | ``str``):\n Unique identifier (int) or username (str) of the target chat.\n For your personal cloud (Saved Messages) you can simply use \"me\" or \"self\".\n For a contact that exists in your Telegram address book you can use his phone number (str).\n\n chat_id (``int`` | ``str``, *optional*):\n Unique identifier (int) or username (str) of the target chat.\n For your personal cloud (Saved Messages) you can simply use \"me\" or \"self\".\n For a contact that exists in your Telegram address book you can use his phone number (str).\n Required if inline_message_id is not specified.\n\n message_id (``int``, *optional*):\n Identifier of the sent message.\n Required if inline_message_id is not specified.\n\n Returns:\n On success, a :obj:`GameHighScores ` object is returned.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n # TODO: inline_message_id\n\n return pyrogram.GameHighScores._parse(\n self,\n self.send(\n functions.messages.GetGameHighScores(\n peer=self.resolve_peer(chat_id),\n id=message_id,\n user_id=self.resolve_peer(user_id)\n )\n )\n )"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef unpin_chat_message(\n self,\n chat_id: Union[int, str]\n ) -> bool:\n \"\"\"Use this method to unpin a message in a group, channel or your own chat.\n You must be an administrator in the chat for this to work and must have the \"can_pin_messages\" admin\n right in the supergroup or \"can_edit_messages\" admin right in the channel.\n\n Args:\n chat_id (``int`` | ``str``):\n Unique identifier (int) or username (str) of the target chat.\n\n Returns:\n True on success.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n self.send(\n functions.messages.UpdatePinnedMessage(\n peer=self.resolve_peer(chat_id),\n id=0\n )\n )\n\n return True", "response": "Use this method to unpin a message in a chat."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef answer(self, text: str = None, show_alert: bool = None, url: str = None, cache_time: int = 0):\n return self._client.answer_callback_query(\n callback_query_id=self.id,\n text=text,\n show_alert=show_alert,\n url=url,\n cache_time=cache_time\n )", "response": "This method returns a response to the user."} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nusing this method to get up to date information about the chat.", "response": "def get_chat(\n self,\n chat_id: Union[int, str]\n ) -> \"pyrogram.Chat\":\n \"\"\"Use this method to get up to date information about the chat.\n Information include current name of the user for one-on-one conversations, current username of a user, group or\n channel, etc.\n\n Args:\n chat_id (``int`` | ``str``):\n Unique identifier (int) or username (str) of the target chat.\n Unique identifier for the target chat in form of a *t.me/joinchat/* link, identifier (int) or username\n of the target channel/supergroup (in the format @username).\n\n Returns:\n On success, a :obj:`Chat ` object is returned.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n ``ValueError`` in case the chat invite link refers to a chat you haven't joined yet.\n \"\"\"\n match = self.INVITE_LINK_RE.match(str(chat_id))\n\n if match:\n h = match.group(1)\n\n r = self.send(\n functions.messages.CheckChatInvite(\n hash=h\n )\n )\n\n if isinstance(r, types.ChatInvite):\n raise ValueError(\"You haven't joined \\\"t.me/joinchat/{}\\\" yet\".format(h))\n\n self.fetch_peers([r.chat])\n\n if isinstance(r.chat, types.Chat):\n chat_id = -r.chat.id\n\n if isinstance(r.chat, types.Channel):\n chat_id = int(\"-100\" + str(r.chat.id))\n\n peer = self.resolve_peer(chat_id)\n\n if isinstance(peer, types.InputPeerChannel):\n r = self.send(functions.channels.GetFullChannel(channel=peer))\n elif isinstance(peer, (types.InputPeerUser, types.InputPeerSelf)):\n r = self.send(functions.users.GetFullUser(id=peer))\n else:\n r = self.send(functions.messages.GetFullChat(chat_id=peer.chat_id))\n\n return pyrogram.Chat._parse_full(self, r)"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef answer_callback_query(\n self,\n callback_query_id: str,\n text: str = None,\n show_alert: bool = None,\n url: str = None,\n cache_time: int = 0\n ):\n \"\"\"Use this method to send answers to callback queries sent from inline keyboards.\n The answer will be displayed to the user as a notification at the top of the chat screen or as an alert.\n\n Args:\n callback_query_id (``str``):\n Unique identifier for the query to be answered.\n\n text (``str``):\n Text of the notification. If not specified, nothing will be shown to the user, 0-200 characters.\n\n show_alert (``bool``):\n If true, an alert will be shown by the client instead of a notification at the top of the chat screen.\n Defaults to False.\n\n url (``str``):\n URL that will be opened by the user's client.\n If you have created a Game and accepted the conditions via @Botfather, specify the URL that opens your\n game \u2013 note that this will only work if the query comes from a callback_game button.\n Otherwise, you may use links like t.me/your_bot?start=XXXX that open your bot with a parameter.\n\n cache_time (``int``):\n The maximum amount of time in seconds that the result of the callback query may be cached client-side.\n Telegram apps will support caching starting in version 3.14. Defaults to 0.\n\n Returns:\n True, on success.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n return self.send(\n functions.messages.SetBotCallbackAnswer(\n query_id=int(callback_query_id),\n cache_time=cache_time,\n alert=show_alert or None,\n message=text,\n url=url\n )\n )", "response": "Use this method to send answers to callback queries sent from inline keyboards."} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncreate a new filter.", "response": "def create(name: str, func: callable, **kwargs) -> type:\n \"\"\"Use this method to create a Filter.\n\n Custom filters give you extra control over which updates are allowed or not to be processed by your handlers.\n\n Args:\n name (``str``):\n Your filter's name. Can be anything you like.\n\n func (``callable``):\n A function that accepts two arguments *(filter, update)* and returns a Boolean: True if the update should be\n handled, False otherwise.\n The \"update\" argument type will vary depending on which `Handler `_ is coming from.\n For example, in a :obj:`MessageHandler ` the update type will be\n a :obj:`Message `; in a :obj:`CallbackQueryHandler ` the\n update type will be a :obj:`CallbackQuery `. Your function body can then access the\n incoming update and decide whether to allow it or not.\n\n **kwargs (``any``, *optional*):\n Any keyword argument you would like to pass. Useful for custom filters that accept parameters (e.g.:\n :meth:`Filters.command`, :meth:`Filters.regex`).\n \"\"\"\n # TODO: unpack kwargs using **kwargs into the dict itself. For Python 3.5+ only\n d = {\"__call__\": func}\n d.update(kwargs)\n\n return type(name, (Filter,), d)()"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nfilter commands in the log file.", "response": "def command(\n commands: str or list,\n prefix: str or list = \"/\",\n separator: str = \" \",\n case_sensitive: bool = False\n ):\n \"\"\"Filter commands, i.e.: text messages starting with \"/\" or any other custom prefix.\n\n Args:\n commands (``str`` | ``list``):\n The command or list of commands as string the filter should look for.\n Examples: \"start\", [\"start\", \"help\", \"settings\"]. When a message text containing\n a command arrives, the command itself and its arguments will be stored in the *command*\n field of the :class:`Message `.\n\n prefix (``str`` | ``list``, *optional*):\n A prefix or a list of prefixes as string the filter should look for.\n Defaults to \"/\" (slash). Examples: \".\", \"!\", [\"/\", \"!\", \".\"].\n Can be None or \"\" (empty string) to allow commands with no prefix at all.\n\n separator (``str``, *optional*):\n The command arguments separator. Defaults to \" \" (white space).\n Examples: /start first second, /start-first-second, /start.first.second.\n\n case_sensitive (``bool``, *optional*):\n Pass True if you want your command(s) to be case sensitive. Defaults to False.\n Examples: when True, command=\"Start\" would trigger /Start but not /start.\n \"\"\"\n\n def func(flt, message):\n text = message.text or message.caption\n\n if text:\n for p in flt.p:\n if text.startswith(p):\n s = text.split(flt.s)\n c, a = s[0][len(p):], s[1:]\n c = c if flt.cs else c.lower()\n message.command = ([c] + a) if c in flt.c else None\n break\n\n return bool(message.command)\n\n commands = commands if type(commands) is list else [commands]\n commands = {c if case_sensitive else c.lower() for c in commands}\n prefixes = set(prefix) if prefix else {\"\"}\n\n return create(\"Command\", func=func, c=commands, p=prefixes, s=separator, cs=case_sensitive)"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncreates a new instance of the class Message class that filters messages that match a given RegEx pattern.", "response": "def regex(pattern, flags: int = 0):\n \"\"\"Filter messages that match a given RegEx pattern.\n\n Args:\n pattern (``str``):\n The RegEx pattern as string, it will be applied to the text of a message. When a pattern matches,\n all the `Match Objects `_\n are stored in the *matches* field of the :class:`Message ` itself.\n\n flags (``int``, *optional*):\n RegEx flags.\n \"\"\"\n\n def f(_, m):\n m.matches = [i for i in _.p.finditer(m.text or m.caption or \"\")]\n return bool(m.matches)\n\n return create(\"Regex\", f, p=re.compile(pattern, flags))"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef change_cloud_password(\n self,\n current_password: str,\n new_password: str,\n new_hint: str = \"\"\n ) -> bool:\n \"\"\"Use this method to change your Two-Step Verification password (Cloud Password) with a new one.\n\n Args:\n current_password (``str``):\n Your current password.\n\n new_password (``str``):\n Your new password.\n\n new_hint (``str``, *optional*):\n A new password hint.\n\n Returns:\n True on success.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n ``ValueError`` in case there is no cloud password to change.\n \"\"\"\n r = self.send(functions.account.GetPassword())\n\n if not r.has_password:\n raise ValueError(\"There is no cloud password to change\")\n\n r.new_algo.salt1 += os.urandom(32)\n new_hash = btoi(compute_hash(r.new_algo, new_password))\n new_hash = itob(pow(r.new_algo.g, new_hash, btoi(r.new_algo.p)))\n\n self.send(\n functions.account.UpdatePasswordSettings(\n password=compute_check(r, current_password),\n new_settings=types.account.PasswordInputSettings(\n new_algo=r.new_algo,\n new_password_hash=new_hash,\n hint=new_hint\n )\n )\n )\n\n return True", "response": "Change your Cloud Password with a new one."} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nuses this method to get information about one member of a chat.", "response": "def get_chat_member(\n self,\n chat_id: Union[int, str],\n user_id: Union[int, str]\n ) -> \"pyrogram.ChatMember\":\n \"\"\"Use this method to get information about one member of a chat.\n\n Args:\n chat_id (``int`` | ``str``):\n Unique identifier (int) or username (str) of the target chat.\n\n user_id (``int`` | ``str``)::\n Unique identifier (int) or username (str) of the target chat.\n For your personal cloud (Saved Messages) you can simply use \"me\" or \"self\".\n For a contact that exists in your Telegram address book you can use his phone number (str).\n\n Returns:\n On success, a :obj:`ChatMember ` object is returned.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n chat_id = self.resolve_peer(chat_id)\n user_id = self.resolve_peer(user_id)\n\n if isinstance(chat_id, types.InputPeerChat):\n full_chat = self.send(\n functions.messages.GetFullChat(\n chat_id=chat_id.chat_id\n )\n )\n\n for member in pyrogram.ChatMembers._parse(self, full_chat).chat_members:\n if member.user.is_self:\n return member\n else:\n raise UserNotParticipant\n elif isinstance(chat_id, types.InputPeerChannel):\n r = self.send(\n functions.channels.GetParticipant(\n channel=chat_id,\n user_id=user_id\n )\n )\n\n users = {i.id: i for i in r.users}\n\n return pyrogram.ChatMember._parse(self, r.participant, users)\n else:\n raise ValueError(\"The chat_id \\\"{}\\\" belongs to a user\".format(chat_id))"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef iter_chat_members(\n self,\n chat_id: Union[int, str],\n limit: int = 0,\n query: str = \"\",\n filter: str = Filters.ALL\n ) -> Generator[\"pyrogram.ChatMember\", None, None]:\n \"\"\"Use this method to iterate through the members of a chat sequentially.\n\n This convenience method does the same as repeatedly calling :meth:`get_chat_members` in a loop, thus saving you\n from the hassle of setting up boilerplate code. It is useful for getting the whole members list of a chat with\n a single call.\n\n Args:\n chat_id (``int`` | ``str``):\n Unique identifier (int) or username (str) of the target chat.\n\n limit (``int``, *optional*):\n Limits the number of members to be retrieved.\n By default, no limit is applied and all members are returned.\n\n query (``str``, *optional*):\n Query string to filter members based on their display names and usernames.\n Defaults to \"\" (empty string).\n\n filter (``str``, *optional*):\n Filter used to select the kind of members you want to retrieve. Only applicable for supergroups\n and channels. It can be any of the followings:\n *\"all\"* - all kind of members,\n *\"kicked\"* - kicked (banned) members only,\n *\"restricted\"* - restricted members only,\n *\"bots\"* - bots only,\n *\"recent\"* - recent members only,\n *\"administrators\"* - chat administrators only.\n Defaults to *\"all\"*.\n\n Returns:\n A generator yielding :obj:`ChatMember ` objects.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n current = 0\n yielded = set()\n queries = [query] if query else QUERIES\n total = limit or (1 << 31) - 1\n limit = min(200, total)\n resolved_chat_id = self.resolve_peer(chat_id)\n\n filter = (\n Filters.RECENT\n if self.get_chat_members_count(chat_id) <= 10000 and filter == Filters.ALL\n else filter\n )\n\n if filter not in QUERYABLE_FILTERS:\n queries = [\"\"]\n\n for q in queries:\n offset = 0\n\n while True:\n chat_members = self.get_chat_members(\n chat_id=chat_id,\n offset=offset,\n limit=limit,\n query=q,\n filter=filter\n ).chat_members\n\n if not chat_members:\n break\n\n if isinstance(resolved_chat_id, types.InputPeerChat):\n total = len(chat_members)\n\n offset += len(chat_members)\n\n for chat_member in chat_members:\n user_id = chat_member.user.id\n\n if user_id in yielded:\n continue\n\n yield chat_member\n\n yielded.add(chat_member.user.id)\n\n current += 1\n\n if current >= total:\n return", "response": "This method returns an iterator over the members of a chat."} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncheck that the argument is a type and return it.", "response": "def _type_check(arg, msg):\n \"\"\"Check that the argument is a type, and return it (internal helper).\n\n As a special case, accept None and return type(None) instead.\n Also, _TypeAlias instances (e.g. Match, Pattern) are acceptable.\n\n The msg argument is a human-readable error message, e.g.\n\n \"Union[arg, ...]: arg should be a type.\"\n\n We append the repr() of the actual value (truncated to 100 chars).\n \"\"\"\n if arg is None:\n return type(None)\n if isinstance(arg, str):\n arg = _ForwardRef(arg)\n if (\n isinstance(arg, _TypingBase) and type(arg).__name__ == '_ClassVar' or\n not isinstance(arg, (type, _TypingBase)) and not callable(arg)\n ):\n raise TypeError(msg + \" Got %.100r.\" % (arg,))\n # Bare Union etc. are not valid as type arguments\n if (\n type(arg).__name__ in ('_Union', '_Optional') and\n not getattr(arg, '__origin__', None) or\n isinstance(arg, TypingMeta) and arg._gorg in (Generic, _Protocol)\n ):\n raise TypeError(\"Plain %s is not valid as type argument\" % arg)\n return arg"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn the repr of an object in a special - casing types.", "response": "def _type_repr(obj):\n \"\"\"Return the repr() of an object, special-casing types (internal helper).\n\n If obj is a type, we return a shorter version than the default\n type.__repr__, based on the module and qualified name, which is\n typically enough to uniquely identify a type. For everything\n else, we fall back on repr(obj).\n \"\"\"\n if isinstance(obj, type) and not isinstance(obj, TypingMeta):\n if obj.__module__ == 'builtins':\n return _qualname(obj)\n return '%s.%s' % (obj.__module__, _qualname(obj))\n if obj is ...:\n return ('...')\n if isinstance(obj, types.FunctionType):\n return obj.__name__\n return repr(obj)"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _make_subclasshook(cls):\n if isinstance(cls.__extra__, abc.ABCMeta):\n # The logic mirrors that of ABCMeta.__subclasscheck__.\n # Registered classes need not be checked here because\n # cls and its extra share the same _abc_registry.\n def __extrahook__(subclass):\n res = cls.__extra__.__subclasshook__(subclass)\n if res is not NotImplemented:\n return res\n if cls.__extra__ in subclass.__mro__:\n return True\n for scls in cls.__extra__.__subclasses__():\n if isinstance(scls, GenericMeta):\n continue\n if issubclass(subclass, scls):\n return True\n return NotImplemented\n else:\n # For non-ABC extras we'll just call issubclass().\n def __extrahook__(subclass):\n if cls.__extra__ and issubclass(subclass, cls.__extra__):\n return True\n return NotImplemented\n return __extrahook__", "response": "Construct a __subclasshook__ callable that incorporates\n the associated __extra__ class in subclass checks performed\n against cls.\nAttributeNames"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _no_slots_copy(dct):\n dict_copy = dict(dct)\n if '__slots__' in dict_copy:\n for slot in dict_copy['__slots__']:\n dict_copy.pop(slot, None)\n return dict_copy", "response": "Internal helper to copy class __dict__ and clean slots class variables."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _get_defaults(func):\n try:\n code = func.__code__\n except AttributeError:\n # Some built-in functions don't have __code__, __defaults__, etc.\n return {}\n pos_count = code.co_argcount\n arg_names = code.co_varnames\n arg_names = arg_names[:pos_count]\n defaults = func.__defaults__ or ()\n kwdefaults = func.__kwdefaults__\n res = dict(kwdefaults) if kwdefaults else {}\n pos_offset = pos_count - len(defaults)\n for name, value in zip(arg_names[pos_offset:], defaults):\n assert name not in res\n res[name] = value\n return res", "response": "Internal helper to extract the default arguments by name."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_type_hints(obj, globalns=None, localns=None):\n\n if getattr(obj, '__no_type_check__', None):\n return {}\n # Classes require a special treatment.\n if isinstance(obj, type):\n hints = {}\n for base in reversed(obj.__mro__):\n if globalns is None:\n base_globals = sys.modules[base.__module__].__dict__\n else:\n base_globals = globalns\n ann = base.__dict__.get('__annotations__', {})\n for name, value in ann.items():\n if value is None:\n value = type(None)\n if isinstance(value, str):\n value = _ForwardRef(value)\n value = _eval_type(value, base_globals, localns)\n hints[name] = value\n return hints\n\n if globalns is None:\n if isinstance(obj, types.ModuleType):\n globalns = obj.__dict__\n else:\n globalns = getattr(obj, '__globals__', {})\n if localns is None:\n localns = globalns\n elif localns is None:\n localns = globalns\n hints = getattr(obj, '__annotations__', None)\n if hints is None:\n # Return empty annotations for something that _could_ have them.\n if isinstance(obj, _allowed_types):\n return {}\n else:\n raise TypeError('{!r} is not a module, class, method, '\n 'or function.'.format(obj))\n defaults = _get_defaults(obj)\n hints = dict(hints)\n for name, value in hints.items():\n if value is None:\n value = type(None)\n if isinstance(value, str):\n value = _ForwardRef(value)\n value = _eval_type(value, globalns, localns)\n if name in defaults and defaults[name] is None:\n value = Optional[value]\n hints[name] = value\n return hints", "response": "Returns a dictionary of type hints for an object."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef no_type_check_decorator(decorator):\n\n @functools.wraps(decorator)\n def wrapped_decorator(*args, **kwds):\n func = decorator(*args, **kwds)\n func = no_type_check(func)\n return func\n\n return wrapped_decorator", "response": "Decorator to give another decorator the no_type_check effect."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef NewType(name, tp):\n\n def new_type(x):\n return x\n\n new_type.__name__ = name\n new_type.__supertype__ = tp\n return new_type", "response": "Create a new type that is a subtype of tp."} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nget all messages from the Telegram address book.", "response": "def get_messages(\n self,\n chat_id: Union[int, str],\n message_ids: Union[int, Iterable[int]] = None,\n reply_to_message_ids: Union[int, Iterable[int]] = None,\n replies: int = 1\n ) -> Union[\"pyrogram.Message\", \"pyrogram.Messages\"]:\n \"\"\"Use this method to get one or more messages that belong to a specific chat.\n You can retrieve up to 200 messages at once.\n\n Args:\n chat_id (``int`` | ``str``):\n Unique identifier (int) or username (str) of the target chat.\n For your personal cloud (Saved Messages) you can simply use \"me\" or \"self\".\n For a contact that exists in your Telegram address book you can use his phone number (str).\n\n message_ids (``iterable``, *optional*):\n Pass a single message identifier or a list of message ids (as integers) to get the content of the\n message themselves. Iterators and Generators are also accepted.\n\n reply_to_message_ids (``iterable``, *optional*):\n Pass a single message identifier or a list of message ids (as integers) to get the content of\n the previous message you replied to using this message. Iterators and Generators are also accepted.\n If *message_ids* is set, this argument will be ignored.\n\n replies (``int``, *optional*):\n The number of subsequent replies to get for each message. Defaults to 1.\n\n Returns:\n On success and in case *message_ids* or *reply_to_message_ids* was an iterable, the returned value will be a\n :obj:`Messages ` even if a list contains just one element. Otherwise, if *message_ids* or\n *reply_to_message_ids* was an integer, the single requested :obj:`Message ` is returned.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n ids, ids_type = (\n (message_ids, types.InputMessageID) if message_ids\n else (reply_to_message_ids, types.InputMessageReplyTo) if reply_to_message_ids\n else (None, None)\n )\n\n if ids is None:\n raise ValueError(\"No argument supplied\")\n\n peer = self.resolve_peer(chat_id)\n\n is_iterable = not isinstance(ids, int)\n ids = list(ids) if is_iterable else [ids]\n ids = [ids_type(id=i) for i in ids]\n\n if isinstance(peer, types.InputPeerChannel):\n rpc = functions.channels.GetMessages(channel=peer, id=ids)\n else:\n rpc = functions.messages.GetMessages(id=ids)\n\n while True:\n try:\n r = self.send(rpc)\n except FloodWait as e:\n log.warning(\"Sleeping for {}s\".format(e.x))\n time.sleep(e.x)\n else:\n break\n\n messages = pyrogram.Messages._parse(self, r, replies=replies)\n\n return messages if is_iterable else messages.messages[0]"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nuse this method to send information about a venue.", "response": "def send_venue(\n self,\n chat_id: Union[int, str],\n latitude: float,\n longitude: float,\n title: str,\n address: str,\n foursquare_id: str = \"\",\n foursquare_type: str = \"\",\n disable_notification: bool = None,\n reply_to_message_id: int = None,\n reply_markup: Union[\n \"pyrogram.InlineKeyboardMarkup\",\n \"pyrogram.ReplyKeyboardMarkup\",\n \"pyrogram.ReplyKeyboardRemove\",\n \"pyrogram.ForceReply\"\n ] = None\n ) -> \"pyrogram.Message\":\n \"\"\"Use this method to send information about a venue.\n\n Args:\n chat_id (``int`` | ``str``):\n Unique identifier (int) or username (str) of the target chat.\n For your personal cloud (Saved Messages) you can simply use \"me\" or \"self\".\n For a contact that exists in your Telegram address book you can use his phone number (str).\n\n latitude (``float``):\n Latitude of the venue.\n\n longitude (``float``):\n Longitude of the venue.\n\n title (``str``):\n Name of the venue.\n\n address (``str``):\n Address of the venue.\n\n foursquare_id (``str``, *optional*):\n Foursquare identifier of the venue.\n\n foursquare_type (``str``, *optional*):\n Foursquare type of the venue, if known.\n (For example, \"arts_entertainment/default\", \"arts_entertainment/aquarium\" or \"food/icecream\".)\n\n disable_notification (``bool``, *optional*):\n Sends the message silently.\n Users will receive a notification with no sound.\n\n reply_to_message_id (``int``, *optional*):\n If the message is a reply, ID of the original message\n\n reply_markup (:obj:`InlineKeyboardMarkup` | :obj:`ReplyKeyboardMarkup` | :obj:`ReplyKeyboardRemove` | :obj:`ForceReply`, *optional*):\n Additional interface options. An object for an inline keyboard, custom reply keyboard,\n instructions to remove reply keyboard or to force a reply from the user.\n\n Returns:\n On success, the sent :obj:`Message ` is returned.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n r = self.send(\n functions.messages.SendMedia(\n peer=self.resolve_peer(chat_id),\n media=types.InputMediaVenue(\n geo_point=types.InputGeoPoint(\n lat=latitude,\n long=longitude\n ),\n title=title,\n address=address,\n provider=\"\",\n venue_id=foursquare_id,\n venue_type=foursquare_type\n ),\n message=\"\",\n silent=disable_notification or None,\n reply_to_msg_id=reply_to_message_id,\n random_id=self.rnd_id(),\n reply_markup=reply_markup.write() if reply_markup else None\n )\n )\n\n for i in r.updates:\n if isinstance(i, (types.UpdateNewMessage, types.UpdateNewChannelMessage)):\n return pyrogram.Message._parse(\n self, i.message,\n {i.id: i for i in r.users},\n {i.id: i for i in r.chats}\n )"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_inline_bot_results(\n self,\n bot: Union[int, str],\n query: str,\n offset: str = \"\",\n latitude: float = None,\n longitude: float = None\n ):\n \"\"\"Use this method to get bot results via inline queries.\n You can then send a result using :obj:`send_inline_bot_result `\n\n Args:\n bot (``int`` | ``str``):\n Unique identifier of the inline bot you want to get results from. You can specify\n a @username (str) or a bot ID (int).\n\n query (``str``):\n Text of the query (up to 512 characters).\n\n offset (``str``, *optional*):\n Offset of the results to be returned.\n\n latitude (``float``, *optional*):\n Latitude of the location.\n Useful for location-based results only.\n\n longitude (``float``, *optional*):\n Longitude of the location.\n Useful for location-based results only.\n\n Returns:\n On Success, :obj:`BotResults ` is returned.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n ``TimeoutError`` if the bot fails to answer within 10 seconds\n \"\"\"\n # TODO: Don't return the raw type\n\n try:\n return self.send(\n functions.messages.GetInlineBotResults(\n bot=self.resolve_peer(bot),\n peer=types.InputPeerSelf(),\n query=query,\n offset=offset,\n geo_point=types.InputGeoPoint(\n lat=latitude,\n long=longitude\n ) if (latitude is not None and longitude is not None) else None\n )\n )\n except UnknownError as e:\n # TODO: Add this -503 Timeout error into the Error DB\n if e.x.error_code == -503 and e.x.error_message == \"Timeout\":\n raise TimeoutError(\"The inline bot didn't answer in time\") from None\n else:\n raise e", "response": "Use this method to get bot results via inline queries."} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nuses this method to join a group chat or channel.", "response": "def join_chat(\n self,\n chat_id: str\n ):\n \"\"\"Use this method to join a group chat or channel.\n\n Args:\n chat_id (``str``):\n Unique identifier for the target chat in form of a *t.me/joinchat/* link or username of the target\n channel/supergroup (in the format @username).\n\n Returns:\n On success, a :obj:`Chat ` object is returned.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n match = self.INVITE_LINK_RE.match(chat_id)\n\n if match:\n chat = self.send(\n functions.messages.ImportChatInvite(\n hash=match.group(1)\n )\n )\n if isinstance(chat.chats[0], types.Chat):\n return pyrogram.Chat._parse_chat_chat(self, chat.chats[0])\n elif isinstance(chat.chats[0], types.Channel):\n return pyrogram.Chat._parse_channel_chat(self, chat.chats[0])\n else:\n resolved_peer = self.send(\n functions.contacts.ResolveUsername(\n username=chat_id.lower().strip(\"@\")\n )\n )\n\n channel = types.InputPeerChannel(\n channel_id=resolved_peer.chats[0].id,\n access_hash=resolved_peer.chats[0].access_hash\n )\n\n chat = self.send(\n functions.channels.JoinChannel(\n channel=channel\n )\n )\n\n return pyrogram.Chat._parse_channel_chat(self, chat.chats[0])"} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nuses this method to get contacts from your address book.", "response": "def get_contacts(self):\n \"\"\"Use this method to get contacts from your Telegram address book.\n\n Returns:\n On success, a list of :obj:`User` objects is returned.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n while True:\n try:\n contacts = self.send(functions.contacts.GetContacts(hash=0))\n except FloodWait as e:\n log.warning(\"get_contacts flood: waiting {} seconds\".format(e.x))\n time.sleep(e.x)\n else:\n log.info(\"Total contacts: {}\".format(len(self.peers_by_phone)))\n return [pyrogram.User._parse(self, user) for user in contacts.users]"} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nuse this method to update the username of a user or a channel.", "response": "def update_chat_username(\n self,\n chat_id: Union[int, str],\n username: Union[str, None]\n ) -> bool:\n \"\"\"Use this method to update a channel or a supergroup username.\n \n To update your own username (for users only, not bots) you can use :meth:`update_username`.\n\n Args:\n chat_id (``int`` | ``str``)\n Unique identifier (int) or username (str) of the target chat.\n username (``str`` | ``None``):\n Username to set. Pass \"\" (empty string) or None to remove the username.\n\n Returns:\n True on success.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n ``ValueError`` if a chat_id belongs to a user or chat.\n \"\"\"\n\n peer = self.resolve_peer(chat_id)\n\n if isinstance(peer, types.InputPeerChannel):\n return bool(\n self.send(\n functions.channels.UpdateUsername(\n channel=peer,\n username=username or \"\"\n )\n )\n )\n else:\n raise ValueError(\"The chat_id \\\"{}\\\" belongs to a user or chat\".format(chat_id))"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef send_game(\n self,\n chat_id: Union[int, str],\n game_short_name: str,\n disable_notification: bool = None,\n reply_to_message_id: int = None,\n reply_markup: Union[\n \"pyrogram.InlineKeyboardMarkup\",\n \"pyrogram.ReplyKeyboardMarkup\",\n \"pyrogram.ReplyKeyboardRemove\",\n \"pyrogram.ForceReply\"\n ] = None\n ) -> \"pyrogram.Message\":\n \"\"\"Use this method to send a game.\n\n Args:\n chat_id (``int`` | ``str``):\n Unique identifier (int) or username (str) of the target chat.\n For your personal cloud (Saved Messages) you can simply use \"me\" or \"self\".\n For a contact that exists in your Telegram address book you can use his phone number (str).\n\n game_short_name (``str``):\n Short name of the game, serves as the unique identifier for the game. Set up your games via Botfather.\n\n disable_notification (``bool``, *optional*):\n Sends the message silently.\n Users will receive a notification with no sound.\n\n reply_to_message_id (``int``, *optional*):\n If the message is a reply, ID of the original message.\n\n reply_markup (:obj:`InlineKeyboardMarkup`, *optional*):\n An object for an inline keyboard. If empty, one \u2018Play game_title\u2019 button will be shown automatically.\n If not empty, the first button must launch the game.\n\n Returns:\n On success, the sent :obj:`Message` is returned.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n r = self.send(\n functions.messages.SendMedia(\n peer=self.resolve_peer(chat_id),\n media=types.InputMediaGame(\n id=types.InputGameShortName(\n bot_id=types.InputUserSelf(),\n short_name=game_short_name\n ),\n ),\n message=\"\",\n silent=disable_notification or None,\n reply_to_msg_id=reply_to_message_id,\n random_id=self.rnd_id(),\n reply_markup=reply_markup.write() if reply_markup else None\n )\n )\n\n for i in r.updates:\n if isinstance(i, (types.UpdateNewMessage, types.UpdateNewChannelMessage)):\n return pyrogram.Message._parse(\n self, i.message,\n {i.id: i for i in r.users},\n {i.id: i for i in r.chats}\n )", "response": "Send a game to the user."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef edit_message_reply_markup(\n self,\n chat_id: Union[int, str],\n message_id: int,\n reply_markup: \"pyrogram.InlineKeyboardMarkup\" = None\n ) -> \"pyrogram.Message\":\n \"\"\"Use this method to edit only the reply markup of messages sent by the bot or via the bot (for inline bots).\n\n Args:\n chat_id (``int`` | ``str``):\n Unique identifier (int) or username (str) of the target chat.\n For your personal cloud (Saved Messages) you can simply use \"me\" or \"self\".\n For a contact that exists in your Telegram address book you can use his phone number (str).\n\n message_id (``int``):\n Message identifier in the chat specified in chat_id.\n\n reply_markup (:obj:`InlineKeyboardMarkup`, *optional*):\n An InlineKeyboardMarkup object.\n\n Returns:\n On success, if edited message is sent by the bot, the edited\n :obj:`Message ` is returned, otherwise True is returned.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n\n r = self.send(\n functions.messages.EditMessage(\n peer=self.resolve_peer(chat_id),\n id=message_id,\n reply_markup=reply_markup.write() if reply_markup else None\n )\n )\n\n for i in r.updates:\n if isinstance(i, (types.UpdateEditMessage, types.UpdateEditChannelMessage)):\n return pyrogram.Message._parse(\n self, i.message,\n {i.id: i for i in r.users},\n {i.id: i for i in r.chats}\n )", "response": "Use this method to edit the reply markup of a message in a chat."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nusing this method to delete your own profile photos.", "response": "def delete_user_profile_photos(\n self,\n id: Union[str, List[str]]\n ) -> bool:\n \"\"\"Use this method to delete your own profile photos.\n\n Args:\n id (``str`` | ``list``):\n A single :obj:`Photo ` id as string or multiple ids as list of strings for deleting\n more than one photos at once.\n\n Returns:\n True on success.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n id = id if isinstance(id, list) else [id]\n input_photos = []\n\n for i in id:\n s = unpack(\"`.\n\n Use this method as a shortcut for:\n\n .. code-block:: python\n\n client.answer_inline_query(\n inline_query.id,\n results=[...]\n )\n\n Example:\n .. code-block:: python\n\n inline_query.answer([...])\n\n Args:\n results (List of :obj:`InlineQueryResult `):\n A list of results for the inline query.\n\n cache_time (``int``, *optional*):\n The maximum amount of time in seconds that the result of the inline query may be cached on the server.\n Defaults to 300.\n\n is_personal (``bool``, *optional*):\n Pass True, if results may be cached on the server side only for the user that sent the query.\n By default, results may be returned to any user who sends the same query.\n\n next_offset (``str``, *optional*):\n Pass the offset that a client should send in the next query with the same text to receive more results.\n Pass an empty string if there are no more results or if you don\u2018t support pagination.\n Offset length can\u2019t exceed 64 bytes.\n\n switch_pm_text (``str``, *optional*):\n If passed, clients will display a button with specified text that switches the user to a private chat\n with the bot and sends the bot a start message with the parameter switch_pm_parameter\n\n switch_pm_parameter (``str``, *optional*):\n `Deep-linking `_ parameter for the /start message sent to\n the bot when user presses the switch button. 1-64 characters, only A-Z, a-z, 0-9, _ and - are allowed.\n\n Example: An inline bot that sends YouTube videos can ask the user to connect the bot to their YouTube\n account to adapt search results accordingly. To do this, it displays a \"Connect your YouTube account\"\n button above the results, or even before showing any. The user presses the button, switches to a private\n chat with the bot and, in doing so, passes a start parameter that instructs the bot to return an oauth\n link. Once done, the bot can offer a switch_inline button so that the user can easily return to the chat\n where they wanted to use the bot's inline capabilities.\n \"\"\"\n\n return self._client.answer_inline_query(\n inline_query_id=self.id,\n results=results,\n cache_time=cache_time,\n is_personal=is_personal,\n next_offset=next_offset,\n switch_pm_text=switch_pm_text,\n switch_pm_parameter=switch_pm_parameter\n )", "response": "This method returns the answer of the inline query."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nusing this method to set a new profile photo for a private chat.", "response": "def set_chat_photo(\n self,\n chat_id: Union[int, str],\n photo: str\n ) -> bool:\n \"\"\"Use this method to set a new profile photo for the chat.\n Photos can't be changed for private chats.\n You must be an administrator in the chat for this to work and must have the appropriate admin rights.\n\n Note:\n In regular groups (non-supergroups), this method will only work if the \"All Members Are Admins\"\n setting is off.\n\n Args:\n chat_id (``int`` | ``str``):\n Unique identifier (int) or username (str) of the target chat.\n\n photo (``str``):\n New chat photo. You can pass a :class:`Photo` id or a file path to upload a new photo.\n\n Returns:\n True on success.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n ``ValueError`` if a chat_id belongs to user.\n \"\"\"\n peer = self.resolve_peer(chat_id)\n\n if os.path.exists(photo):\n photo = types.InputChatUploadedPhoto(file=self.save_file(photo))\n else:\n s = unpack(\"` in case of a Telegram RPC error. ``ValueError`` if you used an invalid filter or a chat_id that belongs to a user.", "response": "def get_chat_members(\n self,\n chat_id: Union[int, str],\n offset: int = 0,\n limit: int = 200,\n query: str = \"\",\n filter: str = Filters.ALL\n ) -> \"pyrogram.ChatMembers\":\n \"\"\"Use this method to get a chunk of the members list of a chat.\n\n You can get up to 200 chat members at once.\n A chat can be either a basic group, a supergroup or a channel.\n You must be admin to retrieve the members list of a channel (also known as \"subscribers\").\n For a more convenient way of getting chat members see :meth:`iter_chat_members`.\n\n Args:\n chat_id (``int`` | ``str``):\n Unique identifier (int) or username (str) of the target chat.\n\n offset (``int``, *optional*):\n Sequential number of the first member to be returned.\n Defaults to 0 [1]_.\n\n limit (``int``, *optional*):\n Limits the number of members to be retrieved.\n Defaults to 200, which is also the maximum server limit allowed per method call.\n\n query (``str``, *optional*):\n Query string to filter members based on their display names and usernames.\n Defaults to \"\" (empty string) [2]_.\n\n filter (``str``, *optional*):\n Filter used to select the kind of members you want to retrieve. Only applicable for supergroups\n and channels. It can be any of the followings:\n *\"all\"* - all kind of members,\n *\"kicked\"* - kicked (banned) members only,\n *\"restricted\"* - restricted members only,\n *\"bots\"* - bots only,\n *\"recent\"* - recent members only,\n *\"administrators\"* - chat administrators only.\n Defaults to *\"all\"*.\n\n .. [1] Server limit: on supergroups, you can get up to 10,000 members for a single query and up to 200 members\n on channels.\n\n .. [2] A query string is applicable only for *\"all\"*, *\"kicked\"* and *\"restricted\"* filters only.\n\n Returns:\n On success, a :obj:`ChatMembers` object is returned.\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n ``ValueError`` if you used an invalid filter or a chat_id that belongs to a user.\n \"\"\"\n peer = self.resolve_peer(chat_id)\n\n if isinstance(peer, types.InputPeerChat):\n return pyrogram.ChatMembers._parse(\n self,\n self.send(\n functions.messages.GetFullChat(\n chat_id=peer.chat_id\n )\n )\n )\n elif isinstance(peer, types.InputPeerChannel):\n filter = filter.lower()\n\n if filter == Filters.ALL:\n filter = types.ChannelParticipantsSearch(q=query)\n elif filter == Filters.KICKED:\n filter = types.ChannelParticipantsKicked(q=query)\n elif filter == Filters.RESTRICTED:\n filter = types.ChannelParticipantsBanned(q=query)\n elif filter == Filters.BOTS:\n filter = types.ChannelParticipantsBots()\n elif filter == Filters.RECENT:\n filter = types.ChannelParticipantsRecent()\n elif filter == Filters.ADMINISTRATORS:\n filter = types.ChannelParticipantsAdmins()\n else:\n raise ValueError(\"Invalid filter \\\"{}\\\"\".format(filter))\n\n while True:\n try:\n return pyrogram.ChatMembers._parse(\n self,\n self.send(\n functions.channels.GetParticipants(\n channel=peer,\n filter=filter,\n offset=offset,\n limit=limit,\n hash=0\n )\n )\n )\n except FloodWait as e:\n log.warning(\"Sleeping for {}s\".format(e.x))\n time.sleep(e.x)\n else:\n raise ValueError(\"The chat_id \\\"{}\\\" belongs to a user\".format(chat_id))"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef leave_chat(\n self,\n chat_id: Union[int, str],\n delete: bool = False\n ):\n \"\"\"Use this method to leave a group chat or channel.\n\n Args:\n chat_id (``int`` | ``str``):\n Unique identifier for the target chat or username of the target channel/supergroup\n (in the format @username).\n\n delete (``bool``, *optional*):\n Deletes the group chat dialog after leaving (for simple group chats, not supergroups).\n\n Raises:\n :class:`RPCError ` in case of a Telegram RPC error.\n \"\"\"\n peer = self.resolve_peer(chat_id)\n\n if isinstance(peer, types.InputPeerChannel):\n return self.send(\n functions.channels.LeaveChannel(\n channel=self.resolve_peer(chat_id)\n )\n )\n elif isinstance(peer, types.InputPeerChat):\n r = self.send(\n functions.messages.DeleteChatUser(\n chat_id=peer.chat_id,\n user_id=types.InputPeerSelf()\n )\n )\n\n if delete:\n self.send(\n functions.messages.DeleteHistory(\n peer=peer,\n max_id=0\n )\n )\n\n return r", "response": "Use this method to leave a group chat or channel or supergroup."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef grouper(n, iterable, padvalue=None):\n \"grouper(3, 'abcdefg', 'x') --> ('a','b','c'), ('d','e','f'), ('g','x','x')\"\n return zip_longest(*[iter(iterable)]*n, fillvalue=padvalue)", "response": "Group n elements into a list of lists."} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning a truncated random walk.", "response": "def random_walk(self, path_length, alpha=0, rand=random.Random(), start=None):\n \"\"\" Returns a truncated random walk.\n\n path_length: Length of the random walk.\n alpha: probability of restarts.\n start: the start node of the random walk.\n \"\"\"\n G = self\n if start:\n path = [start]\n else:\n # Sampling is uniform w.r.t V, and not w.r.t E\n path = [rand.choice(list(G.keys()))]\n\n while len(path) < path_length:\n cur = path[-1]\n if len(G[cur]) > 0:\n if rand.random() >= alpha:\n path.append(rand.choice(G[cur]))\n else:\n path.append(path[0])\n else:\n break\n return [str(node) for node in path]"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncounting the word frequences in a list of sentences.", "response": "def count_words(file):\n \"\"\" Counts the word frequences in a list of sentences.\n\n Note:\n This is a helper function for parallel execution of `Vocabulary.from_text`\n method.\n \"\"\"\n c = Counter()\n with open(file, 'r') as f:\n for l in f:\n words = l.strip().split()\n c.update(words)\n return c"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef parallel_generate_walks(d_graph, global_walk_length, num_walks, cpu_num, sampling_strategy=None,\n num_walks_key=None, walk_length_key=None, neighbors_key=None, probabilities_key=None,\n first_travel_key=None, quiet=False):\n \"\"\"\n Generates the random walks which will be used as the skip-gram input.\n :return: List of walks. Each walk is a list of nodes.\n \"\"\"\n\n walks = list()\n\n\n\n if not quiet:\n pbar = tqdm(total=num_walks, desc='Generating walks (CPU: {})'.format(cpu_num))\n\n for n_walk in range(num_walks):\n\n # Update progress bar\n if not quiet:\n pbar.update(1)\n\n # Shuffle the nodes\n shuffled_nodes = list(d_graph.keys())\n random.shuffle(shuffled_nodes)\n\n # Start a random walk from every node\n for source in shuffled_nodes:\n\n # Skip nodes with specific num_walks\n if source in sampling_strategy and \\\n num_walks_key in sampling_strategy[source] and \\\n sampling_strategy[source][num_walks_key] <= n_walk:\n continue\n\n # Start walk\n walk = [source]\n\n # Calculate walk length\n if source in sampling_strategy:\n walk_length = sampling_strategy[source].get(walk_length_key, global_walk_length)\n else:\n walk_length = global_walk_length\n\n # Perform walk\n while len(walk) < walk_length:\n\n walk_options = d_graph[walk[-1]].get(neighbors_key, None)\n\n # Skip dead end nodes\n if not walk_options:\n break\n\n if len(walk) == 1: # For the first step\n probabilities = d_graph[walk[-1]][first_travel_key]\n walk_to = np.random.choice(walk_options, size=1, p=probabilities)[0]\n else:\n probabilities = d_graph[walk[-1]][probabilities_key][walk[-2]]\n walk_to = np.random.choice(walk_options, size=1, p=probabilities)[0]\n\n walk.append(walk_to)\n\n walk = list(map(str, walk)) # Convert all to strings\n\n walks.append(walk)\n\n if not quiet:\n pbar.close()\n\n return walks", "response": "Generates the random walks which will be used as the skip - gram input."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ngenerating a KeyedVectors instance with all the entries in the dictionary.", "response": "def as_keyed_vectors(self):\n \"\"\"\n Generated a KeyedVectors instance with all\n :return:\n \"\"\"\n\n edge_generator = combinations_with_replacement(self.kv.index2word, r=2)\n\n if not self.quiet:\n vocab_size = len(self.kv.vocab)\n total_size = reduce(lambda x, y: x * y, range(1, vocab_size + 2)) / \\\n (2 * reduce(lambda x, y: x * y, range(1, vocab_size)))\n\n edge_generator = tqdm(edge_generator, desc='Generating edge features', total=total_size)\n\n # Generate features\n tokens = []\n features = []\n for edge in edge_generator:\n token = str(tuple(sorted(edge)))\n embedding = self._embed(edge)\n\n tokens.append(token)\n features.append(embedding)\n\n # Build KV instance\n edge_kv = KeyedVectors(vector_size=self.kv.vector_size)\n edge_kv.add(\n entities=tokens,\n weights=features)\n\n return edge_kv"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _precompute_probabilities(self):\n\n d_graph = self.d_graph\n first_travel_done = set()\n\n nodes_generator = self.graph.nodes() if self.quiet \\\n else tqdm(self.graph.nodes(), desc='Computing transition probabilities')\n\n for source in nodes_generator:\n\n # Init probabilities dict for first travel\n if self.PROBABILITIES_KEY not in d_graph[source]:\n d_graph[source][self.PROBABILITIES_KEY] = dict()\n\n for current_node in self.graph.neighbors(source):\n\n # Init probabilities dict\n if self.PROBABILITIES_KEY not in d_graph[current_node]:\n d_graph[current_node][self.PROBABILITIES_KEY] = dict()\n\n unnormalized_weights = list()\n first_travel_weights = list()\n d_neighbors = list()\n\n # Calculate unnormalized weights\n for destination in self.graph.neighbors(current_node):\n\n p = self.sampling_strategy[current_node].get(self.P_KEY,\n self.p) if current_node in self.sampling_strategy else self.p\n q = self.sampling_strategy[current_node].get(self.Q_KEY,\n self.q) if current_node in self.sampling_strategy else self.q\n\n if destination == source: # Backwards probability\n ss_weight = self.graph[current_node][destination].get(self.weight_key, 1) * 1 / p\n elif destination in self.graph[source]: # If the neighbor is connected to the source\n ss_weight = self.graph[current_node][destination].get(self.weight_key, 1)\n else:\n ss_weight = self.graph[current_node][destination].get(self.weight_key, 1) * 1 / q\n\n # Assign the unnormalized sampling strategy weight, normalize during random walk\n unnormalized_weights.append(ss_weight)\n if current_node not in first_travel_done:\n first_travel_weights.append(self.graph[current_node][destination].get(self.weight_key, 1))\n d_neighbors.append(destination)\n\n # Normalize\n unnormalized_weights = np.array(unnormalized_weights)\n d_graph[current_node][self.PROBABILITIES_KEY][\n source] = unnormalized_weights / unnormalized_weights.sum()\n\n if current_node not in first_travel_done:\n unnormalized_weights = np.array(first_travel_weights)\n d_graph[current_node][self.FIRST_TRAVEL_KEY] = unnormalized_weights / unnormalized_weights.sum()\n first_travel_done.add(current_node)\n\n # Save neighbors\n d_graph[current_node][self.NEIGHBORS_KEY] = d_neighbors", "response": "Precomputes transition probabilities for each node in the graph."} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ngenerate the random walks which will be used as the skip - gram input.", "response": "def _generate_walks(self):\n \"\"\"\n Generates the random walks which will be used as the skip-gram input.\n :return: List of walks. Each walk is a list of nodes.\n \"\"\"\n\n flatten = lambda l: [item for sublist in l for item in sublist]\n\n # Split num_walks for each worker\n num_walks_lists = np.array_split(range(self.num_walks), self.workers)\n\n walk_results = Parallel(n_jobs=self.workers, temp_folder=self.temp_folder, require=self.require)(\n delayed(parallel_generate_walks)(self.d_graph,\n self.walk_length,\n len(num_walks),\n idx,\n self.sampling_strategy,\n self.NUM_WALKS_KEY,\n self.WALK_LENGTH_KEY,\n self.NEIGHBORS_KEY,\n self.PROBABILITIES_KEY,\n self.FIRST_TRAVEL_KEY,\n self.quiet) for\n idx, num_walks\n in enumerate(num_walks_lists, 1))\n\n walks = flatten(walk_results)\n\n return walks"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nfit the Word2Vec model using the nodes in self. walks.", "response": "def fit(self, **skip_gram_params):\n \"\"\"\n Creates the embeddings using gensim's Word2Vec.\n :param skip_gram_params: Parameteres for gensim.models.Word2Vec - do not supply 'size' it is taken from the Node2Vec 'dimensions' parameter\n :type skip_gram_params: dict\n :return: A gensim word2vec model\n \"\"\"\n\n if 'workers' not in skip_gram_params:\n skip_gram_params['workers'] = self.workers\n\n if 'size' not in skip_gram_params:\n skip_gram_params['size'] = self.dimensions\n\n return gensim.models.Word2Vec(self.walks, **skip_gram_params)"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_user_info(twitter_user):\n user_info = query_user_info(user=twitter_user)\n twitter_user_data = {}\n twitter_user_data[\"user\"] = user_info.user\n twitter_user_data[\"fullname\"] = user_info.full_name\n twitter_user_data[\"location\"] = user_info.location\n twitter_user_data[\"blog\"] = user_info.blog\n twitter_user_data[\"date_joined\"] = user_info.date_joined\n twitter_user_data[\"id\"] = user_info.id\n twitter_user_data[\"num_tweets\"] = user_info.tweets\n twitter_user_data[\"following\"] = user_info.following\n twitter_user_data[\"followers\"] = user_info.followers\n twitter_user_data[\"likes\"] = user_info.likes\n twitter_user_data[\"lists\"] = user_info.lists\n\n return twitter_user_data", "response": "Returns a dictionary of user data from the twitter user"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef query_single_page(query, lang, pos, retry=50, from_user=False):\n url = get_query_url(query, lang, pos, from_user)\n\n try:\n response = requests.get(url, headers=HEADER)\n if pos is None: # html response\n html = response.text or ''\n json_resp = None\n else:\n html = ''\n try:\n json_resp = json.loads(response.text)\n html = json_resp['items_html'] or ''\n except ValueError as e:\n logger.exception('Failed to parse JSON \"{}\" while requesting \"{}\"'.format(e, url))\n\n tweets = list(Tweet.from_html(html))\n\n if not tweets:\n if json_resp:\n pos = json_resp['min_position']\n else:\n pos = None\n if retry > 0:\n return query_single_page(query, lang, pos, retry - 1, from_user)\n else:\n return [], pos\n\n if json_resp:\n return tweets, urllib.parse.quote(json_resp['min_position'])\n if from_user:\n return tweets, tweets[-1].id\n return tweets, \"TWEET-{}-{}\".format(tweets[-1].id, tweets[0].id)\n\n except requests.exceptions.HTTPError as e:\n logger.exception('HTTPError {} while requesting \"{}\"'.format(\n e, url))\n except requests.exceptions.ConnectionError as e:\n logger.exception('ConnectionError {} while requesting \"{}\"'.format(\n e, url))\n except requests.exceptions.Timeout as e:\n logger.exception('TimeOut {} while requesting \"{}\"'.format(\n e, url))\n except json.decoder.JSONDecodeError as e:\n logger.exception('Failed to parse JSON \"{}\" while requesting \"{}\".'.format(\n e, url))\n\n if retry > 0:\n logger.info('Retrying... (Attempts left: {})'.format(retry))\n return query_single_page(query, lang, pos, retry - 1)\n\n logger.error('Giving up.')\n return [], None", "response": "Query the TWEET API for tweets from the given URL."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef query_tweets_once_generator(query, limit=None, lang='', pos=None):\n logger.info('Querying {}'.format(query))\n query = query.replace(' ', '%20').replace('#', '%23').replace(':', '%3A')\n num_tweets = 0\n try:\n while True:\n new_tweets, new_pos = query_single_page(query, lang, pos)\n if len(new_tweets) == 0:\n logger.info('Got {} tweets for {}.'.format(\n num_tweets, query))\n return\n\n for t in new_tweets:\n yield t, pos\n\n # use new_pos only once you have iterated through all old tweets\n pos = new_pos\n\n num_tweets += len(new_tweets)\n\n if limit and num_tweets >= limit:\n logger.info('Got {} tweets for {}.'.format(\n num_tweets, query))\n return\n\n except KeyboardInterrupt:\n logger.info('Program interrupted by user. Returning tweets gathered '\n 'so far...')\n except BaseException:\n logger.exception('An unknown error occurred! Returning tweets '\n 'gathered so far.')\n logger.info('Got {} tweets for {}.'.format(\n num_tweets, query))", "response": "Query twitter for all tweets in a single page."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef query_user_page(url, retry=10):\n\n try:\n response = requests.get(url, headers=HEADER)\n html = response.text or ''\n\n user = User()\n user_info = user.from_html(html)\n if not user_info:\n return None\n\n return user_info\n\n except requests.exceptions.HTTPError as e:\n logger.exception('HTTPError {} while requesting \"{}\"'.format(\n e, url))\n except requests.exceptions.ConnectionError as e:\n logger.exception('ConnectionError {} while requesting \"{}\"'.format(\n e, url))\n except requests.exceptions.Timeout as e:\n logger.exception('TimeOut {} while requesting \"{}\"'.format(\n e, url))\n\n if retry > 0:\n logger.info('Retrying... (Attempts left: {})'.format(retry))\n return query_user_page(url, retry-1)\n\n logger.error('Giving up.')\n return None", "response": "Query a twitter user page."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef query_user_info(user):\n\n\n try:\n user_info = query_user_page(INIT_URL_USER.format(u=user))\n if user_info:\n logger.info(f\"Got user information from username {user}\")\n return user_info\n\n except KeyboardInterrupt:\n logger.info(\"Program interrupted by user. Returning user information gathered so far...\")\n except BaseException:\n logger.exception(\"An unknown error occurred! Returning user information gathered so far...\")\n\n logger.info(f\"Got user information from username {user}\")\n return user_info", "response": "Query the user info page for a given user."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a User object with captured data from a twitter user page.", "response": "def from_soup(self, tag_prof_header, tag_prof_nav):\n \"\"\"\n Returns the scraped user data from a twitter user page.\n\n :param tag_prof_header: captures the left hand part of user info\n :param tag_prof_nav: captures the upper part of user info\n :return: Returns a User object with captured data via beautifulsoup\n \"\"\"\n\n self.user= tag_prof_header.find('a', {'class':'ProfileHeaderCard-nameLink u-textInheritColor js-nav'})['href'].strip(\"/\") \n self.full_name = tag_prof_header.find('a', {'class':'ProfileHeaderCard-nameLink u-textInheritColor js-nav'}).text\n \n location = tag_prof_header.find('span', {'class':'ProfileHeaderCard-locationText u-dir'}) \n if location is None:\n self.location = \"None\"\n else: \n self.location = location.text.strip()\n\n blog = tag_prof_header.find('span', {'class':\"ProfileHeaderCard-urlText u-dir\"})\n if blog is None:\n blog = \"None\"\n else:\n self.blog = blog.text.strip() \n\n date_joined = tag_prof_header.find('div', {'class':\"ProfileHeaderCard-joinDate\"}).find('span', {'class':'ProfileHeaderCard-joinDateText js-tooltip u-dir'})['title']\n if date_joined is None:\n self.data_joined = \"Unknown\"\n else: \n self.date_joined = date_joined.strip()\n\n self.id = tag_prof_nav.find('div',{'class':'ProfileNav'})['data-user-id']\n tweets = tag_prof_nav.find('span', {'class':\"ProfileNav-value\"})['data-count']\n if tweets is None:\n self.tweets = 0\n else:\n self.tweets = int(tweets)\n\n following = tag_prof_nav.find('li', {'class':\"ProfileNav-item ProfileNav-item--following\"}).\\\n find('span', {'class':\"ProfileNav-value\"})['data-count']\n if following is None:\n following = 0\n else:\n self.following = int(following)\n\n followers = tag_prof_nav.find('li', {'class':\"ProfileNav-item ProfileNav-item--followers\"}).\\\n find('span', {'class':\"ProfileNav-value\"})['data-count']\n if followers is None:\n self.followers = 0\n else:\n self.followers = int(followers) \n \n likes = tag_prof_nav.find('li', {'class':\"ProfileNav-item ProfileNav-item--favorites\"}).\\\n find('span', {'class':\"ProfileNav-value\"})['data-count']\n if likes is None:\n self.likes = 0\n else:\n self.likes = int(likes) \n \n lists = tag_prof_nav.find('li', {'class':\"ProfileNav-item ProfileNav-item--lists\"})\n if lists is None:\n self.lists = 0\n elif lists.find('span', {'class':\"ProfileNav-value\"}) is None: \n self.lists = 0\n else: \n lists = lists.find('span', {'class':\"ProfileNav-value\"}).text \n self.lists = int(lists)\n return(self)"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nevaluate the likelihood of a latent variable link", "response": "def pdf_link(self, link_f, y, Y_metadata=None):\n \"\"\"\n :param link_f: latent variables link(f)\n :type link_f: Nx1 array\n :param y: data\n :type y: Nx1 array\n :param Y_metadata: includes censoring information in dictionary key 'censored'\n :returns: likelihood evaluated for this point\n :rtype: float\n \"\"\"\n return np.exp(self.logpdf_link(link_f, y, Y_metadata=Y_metadata))"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn the logpdf of a link with the given latent variables", "response": "def logpdf_link(self, link_f, y, Y_metadata=None):\n \"\"\"\n :param link_f: latent variables (link(f))\n :type link_f: Nx1 array\n :param y: data\n :type y: Nx1 array\n :param Y_metadata: includes censoring information in dictionary key 'censored'\n :returns: likelihood evaluated for this point\n :rtype: float\n \"\"\"\n assert np.atleast_1d(link_f).shape == np.atleast_1d(y).shape\n c = np.zeros_like(y)\n if Y_metadata is not None and 'censored' in Y_metadata.keys():\n c = Y_metadata['censored']\n\n uncensored = (1-c)* (-0.5*np.log(2*np.pi*self.variance) - np.log(y) - (np.log(y)-link_f)**2 /(2*self.variance) )\n censored = c*np.log( 1 - stats.norm.cdf((np.log(y) - link_f)/np.sqrt(self.variance)) )\n logpdf = uncensored + censored\n return logpdf"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef dlogpdf_dlink(self, link_f, y, Y_metadata=None):\n assert np.atleast_1d(link_f).shape == np.atleast_1d(y).shape\n c = np.zeros_like(y)\n if Y_metadata is not None and 'censored' in Y_metadata.keys():\n c = Y_metadata['censored']\n\n val = np.log(y) - link_f\n val_scaled = val/np.sqrt(self.variance)\n val_scaled2 = val/self.variance\n uncensored = (1-c)*(val_scaled2)\n a = (1- stats.norm.cdf(val_scaled))\n # llg(z) = 1. / (1 - norm_cdf(r / sqrt(s2))). * (1 / sqrt(2 * pi * s2). * exp(-1 / (2. * s2). * r. ^ 2));\n censored = c*( 1./a) * (np.exp(-1.* val**2 /(2*self.variance)) / np.sqrt(2*np.pi*self.variance))\n # censored = c * (1. / (1 - stats.norm.cdf(val_scaled))) * (stats.norm.pdf(val_scaled))\n gradient = uncensored + censored\n return gradient", "response": "Function that computes the derivative of logpdf wrt link_f wrt y"} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nfunctions to compute the derivative of log likelihood evaluated at points link ( f ) w. r. t variance parameter", "response": "def dlogpdf_dlink_dvar(self, link_f, y, Y_metadata=None):\n \"\"\"\n :param link_f: latent variables link(f)\n :type link_f: Nx1 array\n :param y: data\n :type y: Nx1 array\n :param Y_metadata: Y_metadata not used in gaussian\n :returns: derivative of log likelihood evaluated at points link(f) w.r.t variance parameter\n :rtype: Nx1 array\n \"\"\"\n c = np.zeros_like(y)\n if Y_metadata is not None and 'censored' in Y_metadata.keys():\n c = Y_metadata['censored']\n\n val = np.log(y) - link_f\n val_scaled = val/np.sqrt(self.variance)\n val_scaled2 = val/self.variance\n a = (1 - stats.norm.cdf(val_scaled))\n uncensored = (1-c)*(-val/(self.variance**2))\n censored = c * (-val*np.exp(-val**2/self.variance)/( 4*np.pi*(self.variance**2)*(a**2)) +\n (-1 + (val**2)/self.variance)*np.exp(-val**2/(2*self.variance) ) /\n ( a*(np.sqrt(2.*np.pi)*2*self.variance**1.5)) )\n dlik_grad_dsigma = uncensored + censored\n # dlik_grad_dsigma = dlik_grad_dsigma*self.variance\n return dlik_grad_dsigma"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef d2logpdf_dlink2_dvar(self, link_f, y, Y_metadata=None):\n c = np.zeros_like(y)\n if Y_metadata is not None and 'censored' in Y_metadata.keys():\n c = Y_metadata['censored']\n val = np.log(y) - link_f\n val_scaled = val/np.sqrt(self.variance)\n val_scaled2 = val/self.variance\n a = (1 - stats.norm.cdf(val_scaled))\n uncensored = (1-c)*( 1./(self.variance**2) )\n censored = c*( val*np.exp(-3*(val**2)/(2*self.variance) )/ ((a**3)*np.sqrt(8*np.pi**3)*self.variance**(5/2.))\n + np.exp(-val**2/self.variance)/((a**2)*4*np.pi*self.variance**2)\n - np.exp(-val**2/self.variance)*val**2 / ((a**2)*2*np.pi*self.variance**3)\n + np.exp(-val**2/self.variance)/ ( (a**2)*4*np.pi*self.variance**2)\n - np.exp(-val**2/ (2*self.variance))*val / ( a*np.sqrt(2*np.pi)*2*self.variance**(5/2.))\n - np.exp(-val**2/self.variance)*(val**2) / ((a**2)*4*np.pi*self.variance**3)\n - np.exp(-val**2/ (2*self.variance))*val/ (a*np.sqrt(2*np.pi)*self.variance**(5/2.))\n + np.exp(-val**2/ (2*self.variance))*(val**3) / (a*np.sqrt(2*np.pi)*2*self.variance**(7/2.)) )\n dlik_hess_dsigma = uncensored + censored\n return dlik_hess_dsigma", "response": "Function d2logpdf_dlink2_dvar - Function to compute the derivative of log likelihood evaluated at points link ( f y"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning a set of samples based on a given value of the latent variable.", "response": "def samples(self, gp, Y_metadata=None):\n \"\"\"\n Returns a set of samples of observations based on a given value of the latent variable.\n\n :param gp: latent variable\n \"\"\"\n orig_shape = gp.shape\n gp = gp.flatten()"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _preprocess_values(self, Y):\n Y_prep = Y.copy()\n Y1 = Y[Y.flatten()==1].size\n Y2 = Y[Y.flatten()==0].size\n assert Y1 + Y2 == Y.size, 'Bernoulli likelihood is meant to be used only with outputs in {0, 1}.'\n Y_prep[Y.flatten() == 0] = -1\n return Y_prep", "response": "Preprocess the values of the observations in Y to be used by the likelihood function."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef pdf_link(self, inv_link_f, y, Y_metadata=None):\n #objective = (inv_link_f**y) * ((1.-inv_link_f)**(1.-y))\n return np.where(y==1, inv_link_f, 1.-inv_link_f)", "response": "Evaluate the pdf of a latent variable inverse link of f."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nlogs - likelihood function given inverse link of f and data y.", "response": "def logpdf_link(self, inv_link_f, y, Y_metadata=None):\n \"\"\"\n Log Likelihood function given inverse link of f.\n\n .. math::\n \\\\ln p(y_{i}|\\\\lambda(f_{i})) = y_{i}\\\\log\\\\lambda(f_{i}) + (1-y_{i})\\\\log (1-f_{i})\n\n :param inv_link_f: latent variables inverse link of f.\n :type inv_link_f: Nx1 array\n :param y: data\n :type y: Nx1 array\n :param Y_metadata: Y_metadata not used in bernoulli\n :returns: log likelihood evaluated at points inverse link of f.\n :rtype: float\n \"\"\"\n #objective = y*np.log(inv_link_f) + (1.-y)*np.log(inv_link_f)\n p = np.where(y==1, inv_link_f, 1.-inv_link_f)\n return np.log(np.clip(p, 1e-9 ,np.inf))"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef dlogpdf_dlink(self, inv_link_f, y, Y_metadata=None):\n #grad = (y/inv_link_f) - (1.-y)/(1-inv_link_f)\n #grad = np.where(y, 1./inv_link_f, -1./(1-inv_link_f))\n ff = np.clip(inv_link_f, 1e-9, 1-1e-9)\n denom = np.where(y==1, ff, -(1-ff))\n return 1./denom", "response": "Gradient of the pdf at y given inverse link of f."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef d2logpdf_dlink2(self, inv_link_f, y, Y_metadata=None):\n #d2logpdf_dlink2 = -y/(inv_link_f**2) - (1-y)/((1-inv_link_f)**2)\n #d2logpdf_dlink2 = np.where(y, -1./np.square(inv_link_f), -1./np.square(1.-inv_link_f))\n arg = np.where(y==1, inv_link_f, 1.-inv_link_f)\n ret = -1./np.square(np.clip(arg, 1e-9, 1e9))\n if np.any(np.isinf(ret)):\n stop\n return ret", "response": "This function computes the second derivative of logpdf at y given the inverse link of f and the inverse link of f_i and inverse link of f_j."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nfunctions to compute the third order derivative log - likelihood function at y given inverse link of f w. r. t inverse link of f w. r. t inverse link of f w. r. t inverse link of f w. r. t inverse link of f w. r. t inverse link of f w. r. t inverse link of f w. r. t inverse link of f w. r. t inverse link of f w. r. t inverse link of f.", "response": "def d3logpdf_dlink3(self, inv_link_f, y, Y_metadata=None):\n \"\"\"\n Third order derivative log-likelihood function at y given inverse link of f w.r.t inverse link of f\n\n .. math::\n \\\\frac{d^{3} \\\\ln p(y_{i}|\\\\lambda(f_{i}))}{d^{3}\\\\lambda(f)} = \\\\frac{2y_{i}}{\\\\lambda(f)^{3}} - \\\\frac{2(1-y_{i}}{(1-\\\\lambda(f))^{3}}\n\n :param inv_link_f: latent variables passed through inverse link of f.\n :type inv_link_f: Nx1 array\n :param y: data\n :type y: Nx1 array\n :param Y_metadata: Y_metadata not used in bernoulli\n :returns: third derivative of log likelihood evaluated at points inverse_link(f)\n :rtype: Nx1 array\n \"\"\"\n assert np.atleast_1d(inv_link_f).shape == np.atleast_1d(y).shape\n #d3logpdf_dlink3 = 2*(y/(inv_link_f**3) - (1-y)/((1-inv_link_f)**3))\n state = np.seterr(divide='ignore')\n # TODO check y \\in {0, 1} or {-1, 1}\n d3logpdf_dlink3 = np.where(y==1, 2./(inv_link_f**3), -2./((1.-inv_link_f)**3))\n np.seterr(**state)\n return d3logpdf_dlink3"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ngetting the \"quantiles\" of the binary labels (Bernoulli draws). all the quantiles must be either 0 or 1, since those are the only values the draw can take!", "response": "def predictive_quantiles(self, mu, var, quantiles, Y_metadata=None):\n \"\"\"\n Get the \"quantiles\" of the binary labels (Bernoulli draws). all the\n quantiles must be either 0 or 1, since those are the only values the\n draw can take!\n \"\"\"\n p = self.predictive_mean(mu, var)\n return [np.asarray(p>(q/100.), dtype=np.int32) for q in quantiles]"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef samples(self, gp, Y_metadata=None):\n orig_shape = gp.shape\n gp = gp.flatten()\n ns = np.ones_like(gp, dtype=int)\n Ysim = np.random.binomial(ns, self.gp_link.transf(gp))\n return Ysim.reshape(orig_shape)", "response": "Returns a set of samples based on a given value of the latent variable."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef align_subplots(N,M,xlim=None, ylim=None):\n #find sensible xlim,ylim\n if xlim is None:\n xlim = [np.inf,-np.inf]\n for i in range(N*M):\n plt.subplot(N,M,i+1)\n xlim[0] = min(xlim[0],plt.xlim()[0])\n xlim[1] = max(xlim[1],plt.xlim()[1])\n if ylim is None:\n ylim = [np.inf,-np.inf]\n for i in range(N*M):\n plt.subplot(N,M,i+1)\n ylim[0] = min(ylim[0],plt.ylim()[0])\n ylim[1] = max(ylim[1],plt.ylim()[1])\n\n for i in range(N*M):\n plt.subplot(N,M,i+1)\n plt.xlim(xlim)\n plt.ylim(ylim)\n if (i)%M:\n plt.yticks([])\n else:\n removeRightTicks()\n if i<(M*(N-1)):\n plt.xticks([])\n else:\n removeUpperTicks()", "response": "aligns all of the subplots with the same limits turn off unnecessary ticks"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef align_subplot_array(axes,xlim=None, ylim=None):\n #find sensible xlim,ylim\n if xlim is None:\n xlim = [np.inf,-np.inf]\n for ax in axes.flatten():\n xlim[0] = min(xlim[0],ax.get_xlim()[0])\n xlim[1] = max(xlim[1],ax.get_xlim()[1])\n if ylim is None:\n ylim = [np.inf,-np.inf]\n for ax in axes.flatten():\n ylim[0] = min(ylim[0],ax.get_ylim()[0])\n ylim[1] = max(ylim[1],ax.get_ylim()[1])\n\n N,M = axes.shape\n for i,ax in enumerate(axes.flatten()):\n ax.set_xlim(xlim)\n ax.set_ylim(ylim)\n if (i)%M:\n ax.set_yticks([])\n else:\n removeRightTicks(ax)\n if i<(M*(N-1)):\n ax.set_xticks([])\n else:\n removeUpperTicks(ax)", "response": "Align axes in the array with the given limits."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef x_frame2D(X,plot_limits=None,resolution=None):\n assert X.shape[1] ==2, \"x_frame2D is defined for two-dimensional inputs\"\n if plot_limits is None:\n xmin,xmax = X.min(0),X.max(0)\n xmin, xmax = xmin-0.2*(xmax-xmin), xmax+0.2*(xmax-xmin)\n elif len(plot_limits)==2:\n xmin, xmax = plot_limits\n else:\n raise ValueError(\"Bad limits for plotting\")\n\n resolution = resolution or 50\n xx,yy = np.mgrid[xmin[0]:xmax[0]:1j*resolution,xmin[1]:xmax[1]:1j*resolution]\n Xnew = np.vstack((xx.flatten(),yy.flatten())).T\n return Xnew, xx, yy, xmin, xmax", "response": "Internal helper function for making plots on a 2D dataset."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef from_EV(E, V):\n a = np.square(E) / V\n b = E / V\n return Gamma(a, b)", "response": "Creates an instance of Gamma Prior from an expected value and variance."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef samples(self, gp, Y_metadata):\n N1, N2 = gp.shape\n Ysim = np.zeros((N1,N2))\n ind = Y_metadata['output_index'].flatten()\n for j in np.unique(ind):\n flt = ind==j\n gp_filtered = gp[flt,:]\n n1 = gp_filtered.shape[0]\n lik = self.likelihoods_list[j]\n _ysim = np.array([np.random.normal(lik.gp_link.transf(gpj), scale=np.sqrt(lik.variance), size=1) for gpj in gp_filtered.flatten()])\n Ysim[flt,:] = _ysim.reshape(n1,N2)\n return Ysim", "response": "Returns a set of samples based on a given value of the latent variable."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncompute psi - statistics for ss - linear kernels.", "response": "def psicomputations(variance, Z, variational_posterior, return_psi2_n=False):\n \"\"\"\n Compute psi-statistics for ss-linear kernel\n \"\"\"\n # here are the \"statistics\" for psi0, psi1 and psi2\n # Produced intermediate results:\n # psi0 N\n # psi1 NxM\n # psi2 MxM\n mu = variational_posterior.mean\n S = variational_posterior.variance\n\n psi0 = (variance*(np.square(mu)+S)).sum(axis=1)\n Zv = variance * Z\n psi1 = np.dot(mu,Zv.T)\n if return_psi2_n:\n psi2 = psi1[:,:,None] * psi1[:,None,:] + np.dot(S[:,None,:] * Zv[None,:,:], Zv.T)\n else:\n psi2 = np.dot(S.sum(axis=0) * Zv, Zv.T) + tdot(psi1.T)\n\n return psi0, psi1, psi2"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _psi2computations(dL_dpsi2, variance, Z, mu, S):\n # here are the \"statistics\" for psi1 and psi2\n # Produced intermediate results:\n # _psi2_dvariance Q\n # _psi2_dZ MxQ\n # _psi2_dmu NxQ\n # _psi2_dS NxQ\n\n variance2 = np.square(variance)\n common_sum = np.dot(mu,(variance*Z).T)\n if len(dL_dpsi2.shape)==2:\n Z_expect = (np.dot(dL_dpsi2,Z)*Z).sum(axis=0)\n dL_dpsi2T = dL_dpsi2+dL_dpsi2.T\n common_expect = np.dot(common_sum,np.dot(dL_dpsi2T,Z))\n Z2_expect = np.inner(common_sum,dL_dpsi2T)\n Z1_expect = np.dot(dL_dpsi2T,Z)\n \n dL_dvar = 2.*S.sum(axis=0)*variance*Z_expect+(common_expect*mu).sum(axis=0)\n \n dL_dmu = common_expect*variance\n \n dL_dS = np.empty(S.shape)\n dL_dS[:] = Z_expect*variance2\n \n dL_dZ = variance2*S.sum(axis=0)*Z1_expect+np.dot(Z2_expect.T,variance*mu)\n else:\n N,M,Q = mu.shape[0],Z.shape[0],mu.shape[1]\n dL_dpsi2_ = dL_dpsi2.sum(axis=0)\n Z_expect = (np.dot(dL_dpsi2.reshape(N*M,M),Z).reshape(N,M,Q)*Z[None,:,:]).sum(axis=1)\n dL_dpsi2T = dL_dpsi2_+dL_dpsi2_.T\n dL_dpsi2T_ = dL_dpsi2+np.swapaxes(dL_dpsi2, 1, 2)\n common_expect = np.dot(common_sum,np.dot(dL_dpsi2T,Z))\n common_expect_ = (common_sum[:,:,None]*np.dot(dL_dpsi2T_.reshape(N*M,M),Z).reshape(N,M,Q)).sum(axis=1)\n Z2_expect = (common_sum[:,:,None]*dL_dpsi2T_).sum(axis=1)\n Z1_expect = np.dot(dL_dpsi2T_.reshape(N*M,M),Z).reshape(N,M,Q)\n \n dL_dvar = 2.*variance*(S*Z_expect).sum(axis=0)+(common_expect_*mu).sum(axis=0)\n \n dL_dmu = common_expect_*variance\n \n dL_dS = np.empty(S.shape)\n dL_dS[:] = variance2* Z_expect\n \n dL_dZ = variance2*(S[:,None,:]*Z1_expect).sum(axis=0)+np.dot(Z2_expect.T,variance*mu)\n\n return dL_dvar, dL_dmu, dL_dS, dL_dZ", "response": "Compute the statistics for psi2 and psi1"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef concatenate_offset(self, X):\n return np.c_[np.ones((X.shape[0], 1)), X]", "response": "Concatenate the offset column X into a single column."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef posterior_inf(self, X=None, posterior=None):\n if X is None:\n try:\n X = self._highest_parent_.X\n except NameError:\n raise RuntimeError(\"This kernel is not part of a model and cannot be used for posterior inference\")\n if posterior is None:\n try:\n posterior = self._highest_parent_.posterior\n except NameError:\n raise RuntimeError(\"This kernel is not part of a model and cannot be used for posterior inference\")\n phi_alpha = self.phi(X) * self.variance\n return (phi_alpha).T.dot(posterior.woodbury_vector), (np.eye(phi_alpha.shape[1])*self.variance - mdot(phi_alpha.T, posterior.woodbury_inv, phi_alpha))", "response": "Do the posterior inference on the parameters given this kernels functions functions\n and the model posterior."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef plot_data(self, which_data_rows='all',\n which_data_ycols='all', visible_dims=None,\n projection='2d', label=None, **plot_kwargs):\n \"\"\"\n Plot the training data\n - For higher dimensions than two, use fixed_inputs to plot the data points with some of the inputs fixed.\n\n Can plot only part of the data\n using which_data_rows and which_data_ycols.\n\n :param which_data_rows: which of the training data to plot (default all)\n :type which_data_rows: 'all' or a slice object to slice self.X, self.Y\n :param which_data_ycols: when the data has several columns (independant outputs), only plot these\n :type which_data_ycols: 'all' or a list of integers\n :param visible_dims: an array specifying the input dimensions to plot (maximum two)\n :type visible_dims: a numpy array\n :param {'2d','3d'} projection: whether to plot in 2d or 3d. This only applies when plotting two dimensional inputs!\n :param str label: the label for the plot\n :param kwargs plot_kwargs: kwargs for the data plot for the plotting library you are using\n\n :returns list: of plots created.\n \"\"\"\n canvas, plot_kwargs = pl().new_canvas(projection=projection, **plot_kwargs)\n plots = _plot_data(self, canvas, which_data_rows, which_data_ycols, visible_dims, projection, label, **plot_kwargs)\n return pl().add_to_canvas(canvas, plots)", "response": "Plot the training data for the current object."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nplot the training data input error for the current object.", "response": "def plot_data_error(self, which_data_rows='all',\n which_data_ycols='all', visible_dims=None,\n projection='2d', label=None, **error_kwargs):\n \"\"\"\n Plot the training data input error.\n\n For higher dimensions than two, use fixed_inputs to plot the data points with some of the inputs fixed.\n\n Can plot only part of the data\n using which_data_rows and which_data_ycols.\n\n :param which_data_rows: which of the training data to plot (default all)\n :type which_data_rows: 'all' or a slice object to slice self.X, self.Y\n :param which_data_ycols: when the data has several columns (independant outputs), only plot these\n :type which_data_ycols: 'all' or a list of integers\n :param visible_dims: an array specifying the input dimensions to plot (maximum two)\n :type visible_dims: a numpy array\n :param {'2d','3d'} projection: whether to plot in 2d or 3d. This only applies when plotting two dimensional inputs!\n :param dict error_kwargs: kwargs for the error plot for the plotting library you are using\n :param str label: the label for the plot\n :param kwargs plot_kwargs: kwargs for the data plot for the plotting library you are using\n\n :returns list: of plots created.\n \"\"\"\n canvas, error_kwargs = pl().new_canvas(projection=projection, **error_kwargs)\n plots = _plot_data_error(self, canvas, which_data_rows, which_data_ycols, visible_dims, projection, label, **error_kwargs)\n return pl().add_to_canvas(canvas, plots)"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef plot_inducing(self, visible_dims=None, projection='2d', label='inducing', legend=True, **plot_kwargs):\n canvas, kwargs = pl().new_canvas(projection=projection, **plot_kwargs)\n plots = _plot_inducing(self, canvas, visible_dims, projection, label, **kwargs)\n return pl().add_to_canvas(canvas, plots, legend=legend)", "response": "Plot the inducing inputs of a sparse GP model."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef plot_errorbars_trainset(self, which_data_rows='all',\n which_data_ycols='all', fixed_inputs=None,\n plot_raw=False, apply_link=False, label=None, projection='2d',\n predict_kw=None, **plot_kwargs):\n \"\"\"\n Plot the errorbars of the GP likelihood on the training data.\n These are the errorbars after the appropriate\n approximations according to the likelihood are done.\n\n This also works for heteroscedastic likelihoods.\n\n Give the Y_metadata in the predict_kw if you need it.\n\n :param which_data_rows: which of the training data to plot (default all)\n :type which_data_rows: 'all' or a slice object to slice self.X, self.Y\n :param which_data_ycols: when the data has several columns (independant outputs), only plot these\n :param fixed_inputs: a list of tuple [(i,v), (i,v)...], specifying that input dimension i should be set to value v.\n :type fixed_inputs: a list of tuples\n :param dict predict_kwargs: kwargs for the prediction used to predict the right quantiles.\n :param kwargs plot_kwargs: kwargs for the data plot for the plotting library you are using\n \"\"\"\n canvas, kwargs = pl().new_canvas(projection=projection, **plot_kwargs)\n plots = _plot_errorbars_trainset(self, canvas, which_data_rows, which_data_ycols,\n fixed_inputs, plot_raw, apply_link, label, projection, predict_kw, **kwargs)\n return pl().add_to_canvas(canvas, plots)", "response": "Plots the errorbars of the GP likelihood on the training data."} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nget a matrix L which satisfies LLT YYT", "response": "def _get_YYTfactor(self, Y):\n \"\"\"\n find a matrix L which satisfies LLT = YYT.\n\n Note that L may have fewer columns than Y.\n \"\"\"\n N, D = Y.shape\n if (N>=D):\n return Y.view(np.ndarray)\n else:\n return jitchol(tdot(Y))"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef parameters_changed(self):\n # Get the model matrices from the kernel\n (F,L,Qc,H,Pinf,dF,dQc,dPinf) = self.kern.sde()\n\n # Use the Kalman filter to evaluate the likelihood\n self._log_marginal_likelihood = self.kf_likelihood(F,L,Qc,H,self.sigma2,Pinf,self.X.T,self.Y.T)\n gradients = self.compute_gradients()\n self.sigma2.gradient_full[:] = gradients[-1]\n self.kern.gradient_full[:] = gradients[:-1]", "response": "This method is called when the parameters of the current object have changed."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef student_t_approx(optimize=True, plot=True):\n real_std = 0.1\n #Start a function, any function\n X = np.linspace(0.0, np.pi*2, 100)[:, None]\n Y = np.sin(X) + np.random.randn(*X.shape)*real_std\n Y = Y/Y.max()\n Yc = Y.copy()\n\n X_full = np.linspace(0.0, np.pi*2, 500)[:, None]\n Y_full = np.sin(X_full)\n Y_full = Y_full/Y_full.max()\n\n #Slightly noisy data\n Yc[75:80] += 1\n\n #Very noisy data\n #Yc[10] += 100\n #Yc[25] += 10\n #Yc[23] += 10\n #Yc[26] += 1000\n #Yc[24] += 10\n #Yc = Yc/Yc.max()\n\n #Add student t random noise to datapoints\n deg_free = 1\n print(\"Real noise: \", real_std)\n initial_var_guess = 0.5\n edited_real_sd = initial_var_guess\n\n # Kernel object\n kernel1 = GPy.kern.RBF(X.shape[1]) + GPy.kern.White(X.shape[1])\n kernel2 = GPy.kern.RBF(X.shape[1]) + GPy.kern.White(X.shape[1])\n kernel3 = GPy.kern.RBF(X.shape[1]) + GPy.kern.White(X.shape[1])\n kernel4 = GPy.kern.RBF(X.shape[1]) + GPy.kern.White(X.shape[1])\n\n #Gaussian GP model on clean data\n m1 = GPy.models.GPRegression(X, Y.copy(), kernel=kernel1)\n # optimize\n m1['.*white'].constrain_fixed(1e-5)\n m1.randomize()\n\n #Gaussian GP model on corrupt data\n m2 = GPy.models.GPRegression(X, Yc.copy(), kernel=kernel2)\n m2['.*white'].constrain_fixed(1e-5)\n m2.randomize()\n\n #Student t GP model on clean data\n t_distribution = GPy.likelihoods.StudentT(deg_free=deg_free, sigma2=edited_real_sd)\n laplace_inf = GPy.inference.latent_function_inference.Laplace()\n m3 = GPy.core.GP(X, Y.copy(), kernel3, likelihood=t_distribution, inference_method=laplace_inf)\n m3['.*t_scale2'].constrain_bounded(1e-6, 10.)\n m3['.*white'].constrain_fixed(1e-5)\n m3.randomize()\n\n #Student t GP model on corrupt data\n t_distribution = GPy.likelihoods.StudentT(deg_free=deg_free, sigma2=edited_real_sd)\n laplace_inf = GPy.inference.latent_function_inference.Laplace()\n m4 = GPy.core.GP(X, Yc.copy(), kernel4, likelihood=t_distribution, inference_method=laplace_inf)\n m4['.*t_scale2'].constrain_bounded(1e-6, 10.)\n m4['.*white'].constrain_fixed(1e-5)\n m4.randomize()\n print(m4)\n debug=True\n if debug:\n m4.optimize(messages=1)\n from matplotlib import pyplot as pb\n pb.plot(m4.X, m4.inference_method.f_hat)\n pb.plot(m4.X, m4.Y, 'rx')\n m4.plot()\n print(m4)\n return m4\n\n if optimize:\n optimizer='scg'\n print(\"Clean Gaussian\")\n m1.optimize(optimizer, messages=1)\n print(\"Corrupt Gaussian\")\n m2.optimize(optimizer, messages=1)\n print(\"Clean student t\")\n m3.optimize(optimizer, messages=1)\n print(\"Corrupt student t\")\n m4.optimize(optimizer, messages=1)\n\n if plot:\n plt.figure(1)\n plt.suptitle('Gaussian likelihood')\n ax = plt.subplot(211)\n m1.plot(ax=ax)\n plt.plot(X_full, Y_full)\n plt.ylim(-1.5, 1.5)\n plt.title('Gaussian clean')\n\n ax = plt.subplot(212)\n m2.plot(ax=ax)\n plt.plot(X_full, Y_full)\n plt.ylim(-1.5, 1.5)\n plt.title('Gaussian corrupt')\n\n plt.figure(2)\n plt.suptitle('Student-t likelihood')\n ax = plt.subplot(211)\n m3.plot(ax=ax)\n plt.plot(X_full, Y_full)\n plt.ylim(-1.5, 1.5)\n plt.title('Student-t rasm clean')\n\n ax = plt.subplot(212)\n m4.plot(ax=ax)\n plt.plot(X_full, Y_full)\n plt.ylim(-1.5, 1.5)\n plt.title('Student-t rasm corrupt')\n\n return m1, m2, m3, m4", "response": "Example of regressing with a student t likelihood using Laplace\n "} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning covariance between X and X2.", "response": "def K(self, X, X2, target):\n \"\"\"Return covariance between X and X2.\"\"\"\n self._K_computations(X, X2)\n target += self.variance*self._K_dvar"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncompute the diagonal of the covariance matrix for X.", "response": "def Kdiag(self, X, target):\n \"\"\"Compute the diagonal of the covariance matrix for X.\"\"\"\n self._K_diag_computations(X)\n target+= self.variance*self._K_diag_dvar"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef dKdiag_dX(self, dL_dKdiag, X, target):\n self._K_diag_computations(X)\n arg = self._K_diag_poly_arg\n target += 2.*self.weight_variance*self.degree*self.variance*X*dL_dKdiag[:, None]*(arg**(self.degree-1))[:, None]", "response": "Gradient of diagonal of covariance with respect to X"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef plot_ARD(kernel, filtering=None, legend=False, canvas=None, **kwargs):\n Tango.reset()\n\n ard_params = np.atleast_2d(kernel.input_sensitivity(summarize=False))\n bottom = 0\n last_bottom = bottom\n\n x = np.arange(kernel._effective_input_dim)\n\n parts = []\n def visit(x):\n if (not isinstance(x, CombinationKernel)) and isinstance(x, Kern):\n parts.append(x)\n kernel.traverse(visit)\n\n if filtering is None:\n filtering = [k.name for k in parts]\n\n bars = []\n kwargs = update_not_existing_kwargs(kwargs, pl().defaults.ard)\n\n\n if canvas is None:\n canvas, kwargs = pl().new_canvas(xlim=(-.5, kernel._effective_input_dim-.5), xlabel='input dimension', ylabel='ard contribution', **kwargs)\n\n for i in range(ard_params.shape[0]):\n if parts[i].name in filtering:\n c = Tango.nextMedium()\n bars.append(pl().barplot(canvas, x,\n ard_params[i,:], color=c,\n label=parts[i].name,\n bottom=bottom, **kwargs))\n last_bottom = ard_params[i,:]\n bottom += last_bottom\n else:\n print(\"filtering out {}\".format(parts[i].name))\n\n #add_bar_labels(fig, ax, [bars[-1]], bottom=bottom-last_bottom)\n\n return pl().add_to_canvas(canvas, bars, legend=legend)", "response": "Plots the ARD kernels using matplotlib."} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nplots a kernel covariance w. r. t. another x.", "response": "def plot_covariance(kernel, x=None, label=None,\n plot_limits=None, visible_dims=None, resolution=None,\n projection='2d', levels=20, **kwargs):\n \"\"\"\n Plot a kernel covariance w.r.t. another x.\n\n :param array-like x: the value to use for the other kernel argument (kernels are a function of two variables!)\n :param plot_limits: the range over which to plot the kernel\n :type plot_limits: Either (xmin, xmax) for 1D or (xmin, xmax, ymin, ymax) / ((xmin, xmax), (ymin, ymax)) for 2D\n :param array-like visible_dims: input dimensions (!) to use for x. Make sure to select 2 or less dimensions to plot.\n :resolution: the resolution of the lines used in plotting. for 2D this defines the grid for kernel evaluation.\n :param {2d|3d} projection: What projection shall we use to plot the kernel?\n :param int levels: for 2D projection, how many levels for the contour plot to use?\n :param kwargs: valid kwargs for your specific plotting library\n \"\"\"\n X = np.ones((2, kernel._effective_input_dim)) * [[-3], [3]]\n _, free_dims, Xgrid, xx, yy, _, _, resolution = helper_for_plot_data(kernel, X, plot_limits, visible_dims, None, resolution)\n\n from numbers import Number\n if x is None:\n from ...kern.src.stationary import Stationary\n x = np.ones((1, kernel._effective_input_dim)) * (not isinstance(kernel, Stationary))\n elif isinstance(x, Number):\n x = np.ones((1, kernel._effective_input_dim))*x\n K = kernel.K(Xgrid, x)\n\n if projection == '3d':\n xlabel = 'X[:,0]'\n ylabel = 'X[:,1]'\n zlabel = \"k(X, {!s})\".format(np.asanyarray(x).tolist())\n else:\n xlabel = 'X'\n ylabel = \"k(X, {!s})\".format(np.asanyarray(x).tolist())\n zlabel = None\n\n canvas, kwargs = pl().new_canvas(projection=projection, xlabel=xlabel, ylabel=ylabel, zlabel=zlabel, **kwargs)\n\n if len(free_dims)<=2:\n if len(free_dims)==1:\n # 1D plotting:\n update_not_existing_kwargs(kwargs, pl().defaults.meanplot_1d) # @UndefinedVariable\n plots = dict(covariance=[pl().plot(canvas, Xgrid[:, free_dims], K, label=label, **kwargs)])\n else:\n if projection == '2d':\n update_not_existing_kwargs(kwargs, pl().defaults.meanplot_2d) # @UndefinedVariable\n plots = dict(covariance=[pl().contour(canvas, xx[:, 0], yy[0, :],\n K.reshape(resolution, resolution),\n levels=levels, label=label, **kwargs)])\n elif projection == '3d':\n update_not_existing_kwargs(kwargs, pl().defaults.meanplot_3d) # @UndefinedVariable\n plots = dict(covariance=[pl().surface(canvas, xx, yy,\n K.reshape(resolution, resolution),\n label=label,\n **kwargs)])\n return pl().add_to_canvas(canvas, plots)\n\n else:\n raise NotImplementedError(\"Cannot plot a kernel with more than two input dimensions\")"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef olympic_marathon_men(optimize=True, plot=True):\n try:import pods\n except ImportError:\n print('pods unavailable, see https://github.com/sods/ods for example datasets')\n return\n data = pods.datasets.olympic_marathon_men()\n\n # create simple GP Model\n m = GPy.models.GPRegression(data['X'], data['Y'])\n\n # set the lengthscale to be something sensible (defaults to 1)\n m.kern.lengthscale = 10.\n\n if optimize:\n m.optimize('bfgs', max_iters=200)\n if plot:\n m.plot(plot_limits=(1850, 2050))\n\n return m", "response": "Run a standard Gaussian process regression on the Olympic marathon data."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef coregionalization_toy(optimize=True, plot=True):\n #build a design matrix with a column of integers indicating the output\n X1 = np.random.rand(50, 1) * 8\n X2 = np.random.rand(30, 1) * 5\n\n #build a suitable set of observed variables\n Y1 = np.sin(X1) + np.random.randn(*X1.shape) * 0.05\n Y2 = np.sin(X2) + np.random.randn(*X2.shape) * 0.05 + 2.\n\n m = GPy.models.GPCoregionalizedRegression(X_list=[X1,X2], Y_list=[Y1,Y2])\n\n if optimize:\n m.optimize('bfgs', max_iters=100)\n\n if plot:\n slices = GPy.util.multioutput.get_slices([X1,X2])\n m.plot(fixed_inputs=[(1,0)],which_data_rows=slices[0],Y_metadata={'output_index':0})\n m.plot(fixed_inputs=[(1,1)],which_data_rows=slices[1],Y_metadata={'output_index':1},ax=pb.gca())\n return m", "response": "A simple demonstration of coregionalization on two sinusoidal functions."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nperform Gaussian process regression on the latitude and longitude of the Mount Epomeo runs.", "response": "def epomeo_gpx(max_iters=200, optimize=True, plot=True):\n \"\"\"\n Perform Gaussian process regression on the latitude and longitude data\n from the Mount Epomeo runs. Requires gpxpy to be installed on your system\n to load in the data.\n \"\"\"\n try:import pods\n except ImportError:\n print('pods unavailable, see https://github.com/sods/ods for example datasets')\n return\n data = pods.datasets.epomeo_gpx()\n num_data_list = []\n for Xpart in data['X']:\n num_data_list.append(Xpart.shape[0])\n\n num_data_array = np.array(num_data_list)\n num_data = num_data_array.sum()\n Y = np.zeros((num_data, 2))\n t = np.zeros((num_data, 2))\n start = 0\n for Xpart, index in zip(data['X'], range(len(data['X']))):\n end = start+Xpart.shape[0]\n t[start:end, :] = np.hstack((Xpart[:, 0:1],\n index*np.ones((Xpart.shape[0], 1))))\n Y[start:end, :] = Xpart[:, 1:3]\n\n num_inducing = 200\n Z = np.hstack((np.linspace(t[:,0].min(), t[:, 0].max(), num_inducing)[:, None],\n np.random.randint(0, 4, num_inducing)[:, None]))\n\n k1 = GPy.kern.RBF(1)\n k2 = GPy.kern.Coregionalize(output_dim=5, rank=5)\n k = k1**k2\n\n m = GPy.models.SparseGPRegression(t, Y, kernel=k, Z=Z, normalize_Y=True)\n m.constrain_fixed('.*variance', 1.)\n m.inducing_inputs.constrain_fixed()\n m.Gaussian_noise.variance.constrain_bounded(1e-3, 1e-1)\n m.optimize(max_iters=max_iters,messages=True)\n\n return m"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nrunning a multimodal error surface for Gaussian process and Gaussian process with a variety of optimizations.", "response": "def multiple_optima(gene_number=937, resolution=80, model_restarts=10, seed=10000, max_iters=300, optimize=True, plot=True):\n \"\"\"\n Show an example of a multimodal error surface for Gaussian process\n regression. Gene 939 has bimodal behaviour where the noisy mode is\n higher.\n \"\"\"\n\n # Contour over a range of length scales and signal/noise ratios.\n length_scales = np.linspace(0.1, 60., resolution)\n log_SNRs = np.linspace(-3., 4., resolution)\n\n try:import pods\n except ImportError:\n print('pods unavailable, see https://github.com/sods/ods for example datasets')\n return\n data = pods.datasets.della_gatta_TRP63_gene_expression(data_set='della_gatta',gene_number=gene_number)\n # data['Y'] = data['Y'][0::2, :]\n # data['X'] = data['X'][0::2, :]\n\n data['Y'] = data['Y'] - np.mean(data['Y'])\n\n lls = GPy.examples.regression._contour_data(data, length_scales, log_SNRs, GPy.kern.RBF)\n if plot:\n pb.contour(length_scales, log_SNRs, np.exp(lls), 20, cmap=pb.cm.jet)\n ax = pb.gca()\n pb.xlabel('length scale')\n pb.ylabel('log_10 SNR')\n\n xlim = ax.get_xlim()\n ylim = ax.get_ylim()\n\n # Now run a few optimizations\n models = []\n optim_point_x = np.empty(2)\n optim_point_y = np.empty(2)\n np.random.seed(seed=seed)\n for i in range(0, model_restarts):\n # kern = GPy.kern.RBF(1, variance=np.random.exponential(1.), lengthscale=np.random.exponential(50.))\n kern = GPy.kern.RBF(1, variance=np.random.uniform(1e-3, 1), lengthscale=np.random.uniform(5, 50))\n\n m = GPy.models.GPRegression(data['X'], data['Y'], kernel=kern)\n m.likelihood.variance = np.random.uniform(1e-3, 1)\n optim_point_x[0] = m.rbf.lengthscale\n optim_point_y[0] = np.log10(m.rbf.variance) - np.log10(m.likelihood.variance);\n\n # optimize\n if optimize:\n m.optimize('scg', xtol=1e-6, ftol=1e-6, max_iters=max_iters)\n\n optim_point_x[1] = m.rbf.lengthscale\n optim_point_y[1] = np.log10(m.rbf.variance) - np.log10(m.likelihood.variance);\n\n if plot:\n pb.arrow(optim_point_x[0], optim_point_y[0], optim_point_x[1] - optim_point_x[0], optim_point_y[1] - optim_point_y[0], label=str(i), head_length=1, head_width=0.5, fc='k', ec='k')\n models.append(m)\n\n if plot:\n ax.set_xlim(xlim)\n ax.set_ylim(ylim)\n return m"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _contour_data(data, length_scales, log_SNRs, kernel_call=GPy.kern.RBF):\n\n lls = []\n total_var = np.var(data['Y'])\n kernel = kernel_call(1, variance=1., lengthscale=1.)\n model = GPy.models.GPRegression(data['X'], data['Y'], kernel=kernel)\n for log_SNR in log_SNRs:\n SNR = 10.**log_SNR\n noise_var = total_var / (1. + SNR)\n signal_var = total_var - noise_var\n model.kern['.*variance'] = signal_var\n model.likelihood.variance = noise_var\n length_scale_lls = []\n\n for length_scale in length_scales:\n model['.*lengthscale'] = length_scale\n length_scale_lls.append(model.log_likelihood())\n\n lls.append(length_scale_lls)\n\n return np.array(lls)", "response": "Evaluate the GP objective function for a given data set for a given length scales and a base 10 logarithm signal portion of the data set."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef olympic_100m_men(optimize=True, plot=True):\n try:import pods\n except ImportError:\n print('pods unavailable, see https://github.com/sods/ods for example datasets')\n return\n data = pods.datasets.olympic_100m_men()\n\n # create simple GP Model\n m = GPy.models.GPRegression(data['X'], data['Y'])\n\n # set the lengthscale to be something sensible (defaults to 1)\n m.rbf.lengthscale = 10\n\n if optimize:\n m.optimize('bfgs', max_iters=200)\n\n if plot:\n m.plot(plot_limits=(1850, 2050))\n return m", "response": "Run a standard Gaussian process regression on the Rogers and Girolami olympics data."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nrunning a simple demonstration of a standard Gaussian process fitting it to data sampled from an RBF covariance.", "response": "def toy_rbf_1d(optimize=True, plot=True):\n \"\"\"Run a simple demonstration of a standard Gaussian process fitting it to data sampled from an RBF covariance.\"\"\"\n try:import pods\n except ImportError:\n print('pods unavailable, see https://github.com/sods/ods for example datasets')\n return\n data = pods.datasets.toy_rbf_1d()\n\n # create simple GP Model\n m = GPy.models.GPRegression(data['X'], data['Y'])\n\n if optimize:\n m.optimize('bfgs')\n if plot:\n m.plot()\n\n return m"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef toy_poisson_rbf_1d_laplace(optimize=True, plot=True):\n optimizer='scg'\n x_len = 100\n X = np.linspace(0, 10, x_len)[:, None]\n f_true = np.random.multivariate_normal(np.zeros(x_len), GPy.kern.RBF(1).K(X))\n Y = np.array([np.random.poisson(np.exp(f)) for f in f_true])[:,None]\n\n kern = GPy.kern.RBF(1)\n poisson_lik = GPy.likelihoods.Poisson()\n laplace_inf = GPy.inference.latent_function_inference.Laplace()\n\n # create simple GP Model\n m = GPy.core.GP(X, Y, kernel=kern, likelihood=poisson_lik, inference_method=laplace_inf)\n\n if optimize:\n m.optimize(optimizer)\n if plot:\n m.plot()\n # plot the real underlying rate function\n pb.plot(X, np.exp(f_true), '--k', linewidth=2)\n\n return m", "response": "Run a simple demonstration of a standard Gaussian process fitting it to data sampled from an RBF covariance."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef robot_wireless(max_iters=100, kernel=None, optimize=True, plot=True):\n try:import pods\n except ImportError:\n print('pods unavailable, see https://github.com/sods/ods for example datasets')\n return\n data = pods.datasets.robot_wireless()\n\n # create simple GP Model\n m = GPy.models.GPRegression(data['Y'], data['X'], kernel=kernel)\n\n # optimize\n if optimize:\n m.optimize(max_iters=max_iters)\n\n Xpredict = m.predict(data['Ytest'])[0]\n if plot:\n pb.plot(data['Xtest'][:, 0], data['Xtest'][:, 1], 'r-')\n pb.plot(Xpredict[:, 0], Xpredict[:, 1], 'b-')\n pb.axis('equal')\n pb.title('WiFi Localization with Gaussian Processes')\n pb.legend(('True Location', 'Predicted Location'))\n\n sse = ((data['Xtest'] - Xpredict)**2).sum()\n\n print(('Sum of squares error on test data: ' + str(sse)))\n return m", "response": "Predict the location of a robot given wirelss signal strength readings."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef silhouette(max_iters=100, optimize=True, plot=True):\n try:import pods\n except ImportError:\n print('pods unavailable, see https://github.com/sods/ods for example datasets')\n return\n data = pods.datasets.silhouette()\n\n # create simple GP Model\n m = GPy.models.GPRegression(data['X'], data['Y'])\n\n # optimize\n if optimize:\n m.optimize(messages=True, max_iters=max_iters)\n\n print(m)\n return m", "response": "Predict the pose of a figure given a silhouette."} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nrun a 1D example of a sparse GP regression.", "response": "def sparse_GP_regression_1D(num_samples=400, num_inducing=5, max_iters=100, optimize=True, plot=True, checkgrad=False):\n \"\"\"Run a 1D example of a sparse GP regression.\"\"\"\n # sample inputs and outputs\n X = np.random.uniform(-3., 3., (num_samples, 1))\n Y = np.sin(X) + np.random.randn(num_samples, 1) * 0.05\n # construct kernel\n rbf = GPy.kern.RBF(1)\n # create simple GP Model\n m = GPy.models.SparseGPRegression(X, Y, kernel=rbf, num_inducing=num_inducing)\n\n if checkgrad:\n m.checkgrad()\n\n if optimize:\n m.optimize('tnc', max_iters=max_iters)\n\n if plot:\n m.plot()\n\n return m"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef sparse_GP_regression_2D(num_samples=400, num_inducing=50, max_iters=100, optimize=True, plot=True, nan=False):\n np.random.seed(1234)\n X = np.random.uniform(-3., 3., (num_samples, 2))\n Y = np.sin(X[:, 0:1]) * np.sin(X[:, 1:2]) + np.random.randn(num_samples, 1) * 0.05\n if nan:\n inan = np.random.binomial(1,.2,size=Y.shape)\n Y[inan] = np.nan\n\n # construct kernel\n rbf = GPy.kern.RBF(2)\n\n # create simple GP Model\n m = GPy.models.SparseGPRegression(X, Y, kernel=rbf, num_inducing=num_inducing)\n\n # contrain all parameters to be positive (but not inducing inputs)\n m['.*len'] = 2.\n\n m.checkgrad()\n\n # optimize\n if optimize:\n m.optimize('tnc', messages=1, max_iters=max_iters)\n\n # plot\n if plot:\n m.plot()\n\n print(m)\n return m", "response": "Run a 2D example of a sparse GP regression."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef uncertain_inputs_sparse_regression(max_iters=200, optimize=True, plot=True):\n fig, axes = pb.subplots(1, 2, figsize=(12, 5), sharex=True, sharey=True)\n\n # sample inputs and outputs\n S = np.ones((20, 1))\n X = np.random.uniform(-3., 3., (20, 1))\n Y = np.sin(X) + np.random.randn(20, 1) * 0.05\n # likelihood = GPy.likelihoods.Gaussian(Y)\n Z = np.random.uniform(-3., 3., (7, 1))\n\n k = GPy.kern.RBF(1)\n # create simple GP Model - no input uncertainty on this one\n m = GPy.models.SparseGPRegression(X, Y, kernel=k, Z=Z)\n\n if optimize:\n m.optimize('scg', messages=1, max_iters=max_iters)\n\n if plot:\n m.plot(ax=axes[0])\n axes[0].set_title('no input uncertainty')\n print(m)\n\n # the same Model with uncertainty\n m = GPy.models.SparseGPRegression(X, Y, kernel=GPy.kern.RBF(1), Z=Z, X_variance=S)\n if optimize:\n m.optimize('scg', messages=1, max_iters=max_iters)\n if plot:\n m.plot(ax=axes[1])\n axes[1].set_title('with input uncertainty')\n fig.canvas.draw()\n\n print(m)\n return m", "response": "Run a 1D example of a sparse GP regression with uncertain inputs."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef simple_mean_function(max_iters=100, optimize=True, plot=True):\n #create simple mean function\n mf = GPy.core.Mapping(1,1)\n mf.f = np.sin\n mf.update_gradients = lambda a,b: None\n\n X = np.linspace(0,10,50).reshape(-1,1)\n Y = np.sin(X) + 0.5*np.cos(3*X) + 0.1*np.random.randn(*X.shape)\n\n k =GPy.kern.RBF(1)\n lik = GPy.likelihoods.Gaussian()\n m = GPy.core.GP(X, Y, kernel=k, likelihood=lik, mean_function=mf)\n if optimize:\n m.optimize(max_iters=max_iters)\n if plot:\n m.plot(plot_limits=(-10,15))\n return m", "response": "The simplest possible mean function. No parameters, just a simple Sinusoid."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef parametric_mean_function(max_iters=100, optimize=True, plot=True):\n #create simple mean function\n mf = GPy.core.Mapping(1,1)\n mf.f = np.sin\n\n X = np.linspace(0,10,50).reshape(-1,1)\n Y = np.sin(X) + 0.5*np.cos(3*X) + 0.1*np.random.randn(*X.shape) + 3*X\n\n mf = GPy.mappings.Linear(1,1)\n\n k =GPy.kern.RBF(1)\n lik = GPy.likelihoods.Gaussian()\n m = GPy.core.GP(X, Y, kernel=k, likelihood=lik, mean_function=mf)\n if optimize:\n m.optimize(max_iters=max_iters)\n if plot:\n m.plot()\n return m", "response": "Create a linear mean function with parameters that we ll learn alongside the kernel"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\npredicts the mean and variance at a series of new points X1new X2new", "response": "def predict(self, X1new, X2new):\n \"\"\"\n Return the predictive mean and variance at a series of new points X1new, X2new\n Only returns the diagonal of the predictive variance, for now.\n\n :param X1new: The points at which to make a prediction\n :type X1new: np.ndarray, Nnew x self.input_dim1\n :param X2new: The points at which to make a prediction\n :type X2new: np.ndarray, Nnew x self.input_dim2\n\n \"\"\"\n k1xf = self.kern1.K(X1new, self.X1)\n k2xf = self.kern2.K(X2new, self.X2)\n A = k1xf.dot(self.U1)\n B = k2xf.dot(self.U2)\n mu = A.dot(self.Ytilde.reshape(self.num_data1, self.num_data2, order='F')).dot(B.T).flatten(order='F')\n k1xx = self.kern1.Kdiag(X1new)\n k2xx = self.kern2.Kdiag(X2new)\n BA = np.kron(B, A)\n var = np.kron(k2xx, k1xx) - np.sum(BA**2*self.Wi, 1) + self.likelihood.variance\n\n return mu[:, None], var[:, None]"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nmaking the right decisions for prediction with a model based on the standard arguments of plotting. This is quite complex and will take a while to understand, so do not change anything in here lightly!!!", "response": "def helper_predict_with_model(self, Xgrid, plot_raw, apply_link, percentiles, which_data_ycols, predict_kw, samples=0):\n \"\"\"\n Make the right decisions for prediction with a model\n based on the standard arguments of plotting.\n\n This is quite complex and will take a while to understand,\n so do not change anything in here lightly!!!\n \"\"\"\n # Put some standards into the predict_kw so that prediction is done automatically:\n if predict_kw is None:\n predict_kw = {}\n if 'likelihood' not in predict_kw:\n if plot_raw:\n from ...likelihoods import Gaussian\n from ...likelihoods.link_functions import Identity\n lik = Gaussian(Identity(), 1e-9) # Make the likelihood not add any noise\n else:\n lik = None\n predict_kw['likelihood'] = lik\n if 'Y_metadata' not in predict_kw:\n predict_kw['Y_metadata'] = {}\n if 'output_index' not in predict_kw['Y_metadata']:\n predict_kw['Y_metadata']['output_index'] = Xgrid[:,-1:].astype(np.int)\n\n mu, _ = self.predict(Xgrid, **predict_kw)\n\n if percentiles is not None:\n percentiles = self.predict_quantiles(Xgrid, quantiles=percentiles, **predict_kw)\n else: percentiles = []\n\n if samples > 0:\n fsamples = self.posterior_samples(Xgrid, size=samples, **predict_kw)\n fsamples = fsamples[:, which_data_ycols, :]\n else:\n fsamples = None\n\n # Filter out the ycolums which we want to plot:\n retmu = mu[:, which_data_ycols]\n percs = [p[:, which_data_ycols] for p in percentiles]\n\n if plot_raw and apply_link:\n for i in range(len(which_data_ycols)):\n retmu[:, [i]] = self.likelihood.gp_link.transf(mu[:, [i]])\n for perc in percs:\n perc[:, [i]] = self.likelihood.gp_link.transf(perc[:, [i]])\n if fsamples is not None:\n for s in range(fsamples.shape[-1]):\n fsamples[:, i, s] = self.likelihood.gp_link.transf(fsamples[:, i, s])\n return retmu, percs, fsamples"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef helper_for_plot_data(self, X, plot_limits, visible_dims, fixed_inputs, resolution):\n #work out what the inputs are for plotting (1D or 2D)\n if fixed_inputs is None:\n fixed_inputs = []\n fixed_dims = get_fixed_dims(fixed_inputs)\n free_dims = get_free_dims(self, visible_dims, fixed_dims)\n\n if len(free_dims) == 1:\n #define the frame on which to plot\n resolution = resolution or 200\n Xnew, xmin, xmax = x_frame1D(X[:,free_dims], plot_limits=plot_limits, resolution=resolution)\n Xgrid = np.zeros((Xnew.shape[0],self.input_dim))\n Xgrid[:,free_dims] = Xnew\n for i,v in fixed_inputs:\n Xgrid[:,i] = v\n x = Xgrid\n y = None\n elif len(free_dims) == 2:\n #define the frame for plotting on\n resolution = resolution or 35\n Xnew, x, y, xmin, xmax = x_frame2D(X[:,free_dims], plot_limits, resolution)\n Xgrid = np.zeros((Xnew.shape[0], self.input_dim))\n Xgrid[:,free_dims] = Xnew\n #xmin = Xgrid.min(0)[free_dims]\n #xmax = Xgrid.max(0)[free_dims]\n for i,v in fixed_inputs:\n Xgrid[:,i] = v\n else:\n raise TypeError(\"calculated free_dims {} from visible_dims {} and fixed_dims {} is neither 1D nor 2D\".format(free_dims, visible_dims, fixed_dims))\n return fixed_dims, free_dims, Xgrid, x, y, xmin, xmax, resolution", "response": "This function creates a grid of data for the current language."} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nstratifies subsampling if labels are given. This means due to rounding errors you might get a little differences between the num_samples and the returned subsampled X.", "response": "def subsample_X(X, labels, num_samples=1000):\n \"\"\"\n Stratified subsampling if labels are given.\n This means due to rounding errors you might get a little differences between the\n num_samples and the returned subsampled X.\n \"\"\"\n if X.shape[0] > num_samples:\n print(\"Warning: subsampling X, as it has more samples then {}. X.shape={!s}\".format(int(num_samples), X.shape))\n if labels is not None:\n subsample = []\n for _, _, _, _, index, _ in scatter_label_generator(labels, X, (0, None, None)):\n subsample.append(np.random.choice(index, size=max(2, int(index.size*(float(num_samples)/X.shape[0]))), replace=False))\n subsample = np.hstack(subsample)\n else:\n subsample = np.random.choice(X.shape[0], size=1000, replace=False)\n X = X[subsample]\n labels = labels[subsample]\n #=======================================================================\n # <<>>\n # <<>>\n # plt.close('all')\n # fig, ax = plt.subplots(1,1)\n # from GPy.plotting.matplot_dep.dim_reduction_plots import most_significant_input_dimensions\n # import matplotlib.patches as mpatches\n # i1, i2 = most_significant_input_dimensions(m, None)\n # xmin, xmax = 100, -100\n # ymin, ymax = 100, -100\n # legend_handles = []\n #\n # X = m.X.mean[:, [i1, i2]]\n # X = m.X.variance[:, [i1, i2]]\n #\n # xmin = X[:,0].min(); xmax = X[:,0].max()\n # ymin = X[:,1].min(); ymax = X[:,1].max()\n # range_ = [[xmin, xmax], [ymin, ymax]]\n # ul = np.unique(labels)\n #\n # for i, l in enumerate(ul):\n # #cdict = dict(red =[(0., colors[i][0], colors[i][0]), (1., colors[i][0], colors[i][0])],\n # # green=[(0., colors[i][0], colors[i][1]), (1., colors[i][1], colors[i][1])],\n # # blue =[(0., colors[i][0], colors[i][2]), (1., colors[i][2], colors[i][2])],\n # # alpha=[(0., 0., .0), (.5, .5, .5), (1., .5, .5)])\n # #cmap = LinearSegmentedColormap('{}'.format(l), cdict)\n # cmap = LinearSegmentedColormap.from_list('cmap_{}'.format(str(l)), [colors[i], colors[i]], 255)\n # cmap._init()\n # #alphas = .5*(1+scipy.special.erf(np.linspace(-2,2, cmap.N+3)))#np.log(np.linspace(np.exp(0), np.exp(1.), cmap.N+3))\n # alphas = (scipy.special.erf(np.linspace(0,2.4, cmap.N+3)))#np.log(np.linspace(np.exp(0), np.exp(1.), cmap.N+3))\n # cmap._lut[:, -1] = alphas\n # print l\n # x, y = X[labels==l].T\n #\n # heatmap, xedges, yedges = np.histogram2d(x, y, bins=300, range=range_)\n # #heatmap, xedges, yedges = np.histogram2d(x, y, bins=100)\n #\n # im = ax.imshow(heatmap, extent=[xedges[0], xedges[-1], yedges[0], yedges[-1]], cmap=cmap, aspect='auto', interpolation='nearest', label=str(l))\n # legend_handles.append(mpatches.Patch(color=colors[i], label=l))\n # ax.set_xlim(xmin, xmax)\n # ax.set_ylim(ymin, ymax)\n # plt.legend(legend_handles, [l.get_label() for l in legend_handles])\n # plt.draw()\n # plt.show()\n #=======================================================================\n return X, labels"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef update_not_existing_kwargs(to_update, update_from):\n if to_update is None:\n to_update = {}\n to_update.update({k:v for k,v in update_from.items() if k not in to_update})\n return to_update", "response": "This function updates the keyword aguments from update_from in\n to_update only if the keys are not set in to_update."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_x_y_var(model):\n # model given\n if hasattr(model, 'has_uncertain_inputs') and model.has_uncertain_inputs():\n X = model.X.mean.values\n X_variance = model.X.variance.values\n else:\n try:\n X = model.X.values\n except AttributeError:\n X = model.X\n X_variance = None\n try:\n Y = model.Y.values\n except AttributeError:\n Y = model.Y\n\n if isinstance(model, WarpedGP) and not model.predict_in_warped_space:\n Y = model.Y_normalized\n \n if sparse.issparse(Y): Y = Y.todense().view(np.ndarray)\n return X, X_variance, Y", "response": "Get the data from a model as\n X X_variance Y the variance of the inputs X_variance and Y the outputs of the inputs X_variance and Y the outputs of the outputs of the model as\n "} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nwork out what the inputs are for plotting", "response": "def get_free_dims(model, visible_dims, fixed_dims):\n \"\"\"\n work out what the inputs are for plotting (1D or 2D)\n\n The visible dimensions are the dimensions, which are visible.\n the fixed_dims are the fixed dimensions for this.\n\n The free_dims are then the visible dims without the fixed dims.\n \"\"\"\n if visible_dims is None:\n visible_dims = np.arange(model.input_dim)\n dims = np.asanyarray(visible_dims)\n if fixed_dims is not None:\n dims = [dim for dim in dims if dim not in fixed_dims]\n return np.asanyarray([dim for dim in dims if dim is not None])"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef x_frame1D(X,plot_limits=None,resolution=None):\n assert X.shape[1] ==1, \"x_frame1D is defined for one-dimensional inputs\"\n if plot_limits is None:\n from GPy.core.parameterization.variational import VariationalPosterior\n if isinstance(X, VariationalPosterior):\n xmin,xmax = X.mean.min(0),X.mean.max(0)\n else:\n xmin,xmax = X.min(0),X.max(0)\n xmin, xmax = xmin-0.25*(xmax-xmin), xmax+0.25*(xmax-xmin)\n elif len(plot_limits) == 2:\n xmin, xmax = map(np.atleast_1d, plot_limits)\n else:\n raise ValueError(\"Bad limits for plotting\")\n\n Xnew = np.linspace(float(xmin),float(xmax),int(resolution) or 200)[:,None]\n return Xnew, xmin, xmax", "response": "Internal helper function for making plots for one - dimensional inputs."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef x_frame2D(X,plot_limits=None,resolution=None):\n assert X.shape[1]==2, \"x_frame2D is defined for two-dimensional inputs\"\n if plot_limits is None:\n xmin, xmax = X.min(0), X.max(0)\n xmin, xmax = xmin-0.075*(xmax-xmin), xmax+0.075*(xmax-xmin)\n elif len(plot_limits) == 2:\n xmin, xmax = plot_limits\n try:\n xmin = xmin[0], xmin[1]\n except:\n # only one limit given, copy over to other lim\n xmin = [plot_limits[0], plot_limits[0]]\n xmax = [plot_limits[1], plot_limits[1]]\n elif len(plot_limits) == 4:\n xmin, xmax = (plot_limits[0], plot_limits[2]), (plot_limits[1], plot_limits[3])\n else:\n raise ValueError(\"Bad limits for plotting\")\n\n resolution = resolution or 50\n xx, yy = np.mgrid[xmin[0]:xmax[0]:1j*resolution,xmin[1]:xmax[1]:1j*resolution]\n Xnew = np.c_[xx.flat, yy.flat]\n return Xnew, xx, yy, xmin, xmax", "response": "Internal helper function for making plots on a 2D dataset."} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncenter X in PCA space.", "response": "def center(self, X):\n \"\"\"\n Center `X` in PCA space.\n \"\"\"\n X = X.copy()\n inan = numpy.isnan(X)\n if self.mu is None:\n X_ = numpy.ma.masked_array(X, inan)\n self.mu = X_.mean(0).base\n self.sigma = X_.std(0).base\n reduce(lambda y,x: setitem(x[0], x[1], x[2]), zip(X.T, inan.T, self.mu), None)\n X = X - self.mu\n X = X / numpy.where(self.sigma == 0, 1e-30, self.sigma)\n return X"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef project(self, X, Q=None):\n if Q is None:\n Q = self.Q\n if Q > X.shape[1]:\n raise IndexError(\"requested dimension larger then input dimension\")\n X = self.center(X)\n return X.dot(self.eigvectors[:, :Q])", "response": "Project X into PCA space defined by the highest eigenvalues."} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nplot fractions of Eigenvalues sorted in descending order.", "response": "def plot_fracs(self, Q=None, ax=None, fignum=None):\n \"\"\"\n Plot fractions of Eigenvalues sorted in descending order.\n \"\"\"\n from ..plotting import Tango\n Tango.reset()\n col = Tango.nextMedium()\n if ax is None:\n fig = pylab.figure(fignum)\n ax = fig.add_subplot(111)\n if Q is None:\n Q = self.Q\n ticks = numpy.arange(Q)\n bar = ax.bar(ticks - .4, self.fracs[:Q], color=col)\n ax.set_xticks(ticks, map(lambda x: r\"${}$\".format(x), ticks + 1))\n ax.set_ylabel(\"Eigenvalue fraction\")\n ax.set_xlabel(\"PC\")\n ax.set_ylim(0, ax.get_ylim()[1])\n ax.set_xlim(ticks.min() - .5, ticks.max() + .5)\n try:\n pylab.tight_layout()\n except:\n pass\n return bar"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef plot_2d(self, X, labels=None, s=20, marker='o',\n dimensions=(0, 1), ax=None, colors=None,\n fignum=None, cmap=None, # @UndefinedVariable\n ** kwargs):\n \"\"\"\n Plot dimensions `dimensions` with given labels against each other in\n PC space. Labels can be any sequence of labels of dimensions X.shape[0].\n Labels can be drawn with a subsequent call to legend()\n \"\"\"\n if cmap is None:\n cmap = matplotlib.cm.jet\n if ax is None:\n fig = pylab.figure(fignum)\n ax = fig.add_subplot(111)\n if labels is None:\n labels = numpy.zeros(X.shape[0])\n ulabels = []\n for lab in labels:\n if not lab in ulabels:\n ulabels.append(lab)\n nlabels = len(ulabels)\n if colors is None:\n colors = iter([cmap(float(i) / nlabels) for i in range(nlabels)])\n else:\n colors = iter(colors)\n X_ = self.project(X, self.Q)[:,dimensions]\n kwargs.update(dict(s=s))\n plots = list()\n for i, l in enumerate(ulabels):\n kwargs.update(dict(color=next(colors), marker=marker[i % len(marker)]))\n plots.append(ax.scatter(*X_[labels == l, :].T, label=str(l), **kwargs))\n ax.set_xlabel(r\"PC$_1$\")\n ax.set_ylabel(r\"PC$_2$\")\n try:\n pylab.tight_layout()\n except:\n pass\n return plots", "response": "Plot the PC space with given labels against each other in\n PC space."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef plot_latent_scatter(self, labels=None,\n which_indices=None,\n legend=True,\n plot_limits=None,\n marker='<>^vsd',\n num_samples=1000,\n projection='2d',\n **kwargs):\n \"\"\"\n Plot a scatter plot of the latent space.\n\n :param array-like labels: a label for each data point (row) of the inputs\n :param (int, int) which_indices: which input dimensions to plot against each other\n :param bool legend: whether to plot the legend on the figure\n :param plot_limits: the plot limits for the plot\n :type plot_limits: (xmin, xmax, ymin, ymax) or ((xmin, xmax), (ymin, ymax))\n :param str marker: markers to use - cycle if more labels then markers are given\n :param kwargs: the kwargs for the scatter plots\n \"\"\"\n canvas, projection, kwargs, sig_dims = _new_canvas(self, projection, kwargs, which_indices)\n\n X, _, _ = get_x_y_var(self)\n if labels is None:\n labels = np.ones(self.num_data)\n legend = False\n else:\n legend = find_best_layout_for_subplots(len(np.unique(labels)))[1]\n scatters = _plot_latent_scatter(canvas, X, sig_dims, labels, marker, num_samples, projection=projection, **kwargs)\n return pl().add_to_canvas(canvas, dict(scatter=scatters), legend=legend)", "response": "Plot a scatter plot of the latent space."} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nplots the inducing inputs for a single node.", "response": "def plot_latent_inducing(self,\n which_indices=None,\n legend=False,\n plot_limits=None,\n marker=None,\n projection='2d',\n **kwargs):\n \"\"\"\n Plot a scatter plot of the inducing inputs.\n\n :param [int] which_indices: which input dimensions to plot against each other\n :param bool legend: whether to plot the legend on the figure\n :param plot_limits: the plot limits for the plot\n :type plot_limits: (xmin, xmax, ymin, ymax) or ((xmin, xmax), (ymin, ymax))\n :param str marker: marker to use [default is custom arrow like]\n :param kwargs: the kwargs for the scatter plots\n :param str projection: for now 2d or 3d projection (other projections can be implemented, see developer documentation)\n \"\"\"\n canvas, projection, kwargs, sig_dims = _new_canvas(self, projection, kwargs, which_indices)\n\n if legend: label = 'inducing'\n else: label = None\n if marker is not None:\n kwargs['marker'] = marker\n update_not_existing_kwargs(kwargs, pl().defaults.inducing_2d) # @UndefinedVariable\n from .data_plots import _plot_inducing\n scatters = _plot_inducing(self, canvas, sig_dims[:2], projection, label, **kwargs)\n return pl().add_to_canvas(canvas, dict(scatter=scatters), legend=legend)"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef plot_magnification(self, labels=None, which_indices=None,\n resolution=60, marker='<>^vsd', legend=True,\n plot_limits=None,\n updates=False,\n mean=True, covariance=True,\n kern=None, num_samples=1000,\n scatter_kwargs=None, plot_scatter=True,\n **imshow_kwargs):\n \"\"\"\n Plot the magnification factor of the GP on the inputs. This is the\n density of the GP as a gray scale.\n\n :param array-like labels: a label for each data point (row) of the inputs\n :param (int, int) which_indices: which input dimensions to plot against each other\n :param int resolution: the resolution at which we predict the magnification factor\n :param str marker: markers to use - cycle if more labels then markers are given\n :param bool legend: whether to plot the legend on the figure\n :param plot_limits: the plot limits for the plot\n :type plot_limits: (xmin, xmax, ymin, ymax) or ((xmin, xmax), (ymin, ymax))\n :param bool updates: if possible, make interactive updates using the specific library you are using\n :param bool mean: use the mean of the Wishart embedding for the magnification factor\n :param bool covariance: use the covariance of the Wishart embedding for the magnification factor\n :param :py:class:`~GPy.kern.Kern` kern: the kernel to use for prediction\n :param int num_samples: the number of samples to plot maximally. We do a stratified subsample from the labels, if the number of samples (in X) is higher then num_samples.\n :param imshow_kwargs: the kwargs for the imshow (magnification factor)\n :param kwargs: the kwargs for the scatter plots\n \"\"\"\n input_1, input_2 = which_indices = self.get_most_significant_input_dimensions(which_indices)[:2]\n X = get_x_y_var(self)[0]\n _, _, Xgrid, _, _, xmin, xmax, resolution = helper_for_plot_data(self, X, plot_limits, which_indices, None, resolution)\n canvas, imshow_kwargs = pl().new_canvas(xlim=(xmin[0], xmax[0]), ylim=(xmin[1], xmax[1]),\n xlabel='latent dimension %i' % input_1, ylabel='latent dimension %i' % input_2, **imshow_kwargs)\n plots = {}\n if legend and plot_scatter:\n if (labels is not None):\n legend = find_best_layout_for_subplots(len(np.unique(labels)))[1]\n else:\n labels = np.ones(self.num_data)\n legend = False\n if plot_scatter:\n plots['scatters'] = _plot_latent_scatter(canvas, X, which_indices, labels, marker, num_samples, projection='2d', **scatter_kwargs or {})\n plots['view'] = _plot_magnification(self, canvas, which_indices, Xgrid, xmin, xmax, resolution, updates, mean, covariance, kern, **imshow_kwargs)\n retval = pl().add_to_canvas(canvas, plots,\n legend=legend,\n )\n _wait_for_updates(plots['view'], updates)\n return retval", "response": "Plot the magnification factor of the GP on the inputs."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef plot_latent(self, labels=None, which_indices=None,\n resolution=60, legend=True,\n plot_limits=None,\n updates=False,\n kern=None, marker='<>^vsd',\n num_samples=1000, projection='2d',\n scatter_kwargs=None, **imshow_kwargs):\n \"\"\"\n Plot the latent space of the GP on the inputs. This is the\n density of the GP posterior as a grey scale and the\n scatter plot of the input dimemsions selected by which_indices.\n\n :param array-like labels: a label for each data point (row) of the inputs\n :param (int, int) which_indices: which input dimensions to plot against each other\n :param int resolution: the resolution at which we predict the magnification factor\n :param bool legend: whether to plot the legend on the figure\n :param plot_limits: the plot limits for the plot\n :type plot_limits: (xmin, xmax, ymin, ymax) or ((xmin, xmax), (ymin, ymax))\n :param bool updates: if possible, make interactive updates using the specific library you are using\n :param :py:class:`~GPy.kern.Kern` kern: the kernel to use for prediction\n :param str marker: markers to use - cycle if more labels then markers are given\n :param int num_samples: the number of samples to plot maximally. We do a stratified subsample from the labels, if the number of samples (in X) is higher then num_samples.\n :param imshow_kwargs: the kwargs for the imshow (magnification factor)\n :param scatter_kwargs: the kwargs for the scatter plots\n \"\"\"\n if projection != '2d':\n raise ValueError('Cannot plot latent in other then 2 dimensions, consider plot_scatter')\n input_1, input_2 = which_indices = self.get_most_significant_input_dimensions(which_indices)[:2]\n X = get_x_y_var(self)[0]\n _, _, Xgrid, _, _, xmin, xmax, resolution = helper_for_plot_data(self, X, plot_limits, which_indices, None, resolution)\n canvas, imshow_kwargs = pl().new_canvas(xlim=(xmin[0], xmax[0]), ylim=(xmin[1], xmax[1]),\n xlabel='latent dimension %i' % input_1, ylabel='latent dimension %i' % input_2, **imshow_kwargs)\n if legend:\n if (labels is not None):\n legend = find_best_layout_for_subplots(len(np.unique(labels)))[1]\n else:\n labels = np.ones(self.num_data)\n legend = False\n scatters = _plot_latent_scatter(canvas, X, which_indices, labels, marker, num_samples, projection='2d', **scatter_kwargs or {})\n view = _plot_latent(self, canvas, which_indices, Xgrid, xmin, xmax, resolution, updates, kern, **imshow_kwargs)\n retval = pl().add_to_canvas(canvas, dict(scatter=scatters, imshow=view), legend=legend)\n _wait_for_updates(view, updates)\n return retval", "response": "Plot the latent space of the GP on the inputs."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef plot_steepest_gradient_map(self, output_labels=None, data_labels=None, which_indices=None,\n resolution=15, legend=True,\n plot_limits=None,\n updates=False,\n kern=None, marker='<>^vsd',\n num_samples=1000,\n annotation_kwargs=None, scatter_kwargs=None, **imshow_kwargs):\n\n \"\"\"\n Plot the latent space of the GP on the inputs. This is the\n density of the GP posterior as a grey scale and the\n scatter plot of the input dimemsions selected by which_indices.\n\n :param array-like labels: a label for each data point (row) of the inputs\n :param (int, int) which_indices: which input dimensions to plot against each other\n :param int resolution: the resolution at which we predict the magnification factor\n :param bool legend: whether to plot the legend on the figure, if int plot legend columns on legend\n :param plot_limits: the plot limits for the plot\n :type plot_limits: (xmin, xmax, ymin, ymax) or ((xmin, xmax), (ymin, ymax))\n :param bool updates: if possible, make interactive updates using the specific library you are using\n :param :py:class:`~GPy.kern.Kern` kern: the kernel to use for prediction\n :param str marker: markers to use - cycle if more labels then markers are given\n :param int num_samples: the number of samples to plot maximally. We do a stratified subsample from the labels, if the number of samples (in X) is higher then num_samples.\n :param imshow_kwargs: the kwargs for the imshow (magnification factor)\n :param annotation_kwargs: the kwargs for the annotation plot\n :param scatter_kwargs: the kwargs for the scatter plots\n \"\"\"\n input_1, input_2 = which_indices = self.get_most_significant_input_dimensions(which_indices)[:2]\n X = get_x_y_var(self)[0]\n _, _, Xgrid, _, _, xmin, xmax, resolution = helper_for_plot_data(self, X, plot_limits, which_indices, None, resolution)\n canvas, imshow_kwargs = pl().new_canvas(xlim=(xmin[0], xmax[0]), ylim=(xmin[1], xmax[1]),\n xlabel='latent dimension %i' % input_1, ylabel='latent dimension %i' % input_2, **imshow_kwargs)\n if (data_labels is not None):\n legend = find_best_layout_for_subplots(len(np.unique(data_labels)))[1]\n else:\n data_labels = np.ones(self.num_data)\n legend = False\n plots = dict(scatter=_plot_latent_scatter(canvas, X, which_indices, data_labels, marker, num_samples, **scatter_kwargs or {}))\n plots.update(_plot_steepest_gradient_map(self, canvas, which_indices, Xgrid, xmin, xmax, resolution, output_labels, updates, kern, annotation_kwargs=annotation_kwargs, **imshow_kwargs))\n retval = pl().add_to_canvas(canvas, plots, legend=legend)\n _wait_for_updates(plots['annotation'], updates)\n return retval", "response": "Plot the latent space of the GP on the inputs."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _set_expressions(self, expressions):\n Symbolic_core._set_expressions(self, expressions)\n Symbolic_core._set_variables(self, self.cacheable)\n # Substitute z with x to obtain kdiag.\n for x, z in zip(self.variables['X'], self.variables['Z']):\n expressions['kdiag'] = expressions['kdiag'].subs(z, x)\n Symbolic_core._set_expressions(self, expressions)", "response": "This method is overwritten because we need to set expressions to obtain kdiag."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef balance_matrix(A):\n\n if len(A.shape) != 2 or (A.shape[0] != A.shape[1]):\n raise ValueError('balance_matrix: Expecting square matrix')\n\n N = A.shape[0] # matrix size\n\n gebal = sp.linalg.lapack.get_lapack_funcs('gebal',(A,))\n bA, lo, hi, pivscale, info = gebal(A, permute=True, scale=True,overwrite_a=False)\n if info < 0:\n raise ValueError('balance_matrix: Illegal value in %d-th argument of internal gebal ' % -info)\n # calculating the similarity transforamtion:\n def perm_matr(D, c1,c2):\n \"\"\"\n Function creates the permutation matrix which swaps columns c1 and c2.\n\n Input:\n --------------\n D: int\n Size of the permutation matrix\n c1: int\n Column 1. Numeration starts from 1...D\n c2: int\n Column 2. Numeration starts from 1...D\n \"\"\"\n i1 = c1-1; i2 = c2-1 # indices\n P = np.eye(D);\n P[i1,i1] = 0.0; P[i2,i2] = 0.0; # nullify diagonal elements\n P[i1,i2] = 1.0; P[i2,i1] = 1.0\n\n return P\n\n P = np.eye(N) # permutation matrix\n if (hi != N-1): # there are row permutations\n for k in range(N-1,hi,-1):\n new_perm = perm_matr(N, k+1, pivscale[k])\n P = np.dot(P,new_perm)\n if (lo != 0):\n for k in range(0,lo,1):\n new_perm = perm_matr(N, k+1, pivscale[k])\n P = np.dot(P,new_perm)\n D = pivscale.copy()\n D[0:lo] = 1.0; D[hi+1:N] = 1.0 # thesee scaling factors must be set to one.\n #D = np.diag(D) # make a diagonal matrix\n\n T = np.dot(P,np.diag(D)) # similarity transformation in question\n T_inv = np.dot(np.diag(D**(-1)),P.T)\n\n #print( np.max(A - np.dot(T, np.dot(bA, T_inv) )) )\n return bA.copy(), T, T_inv", "response": "This function balancing_matrix is used to compute the similarity transformation of the original matrix A."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef balance_ss_model(F,L,Qc,H,Pinf,P0,dF=None,dQc=None,dPinf=None,dP0=None):\n\n bF,T,T_inv = balance_matrix(F)\n\n bL = np.dot( T_inv, L)\n bQc = Qc # not affected\n\n bH = np.dot(H, T)\n\n bPinf = np.dot(T_inv, np.dot(Pinf, T_inv.T))\n\n #import pdb; pdb.set_trace()\n# LL,islower = linalg.cho_factor(Pinf)\n# inds = np.triu_indices(Pinf.shape[0],k=1)\n# LL[inds] = 0.0\n# bLL = np.dot(T_inv, LL)\n# bPinf = np.dot( bLL, bLL.T)\n\n bP0 = np.dot(T_inv, np.dot(P0, T_inv.T))\n\n if dF is not None:\n bdF = np.zeros(dF.shape)\n for i in range(dF.shape[2]):\n bdF[:,:,i] = np.dot( T_inv, np.dot( dF[:,:,i], T))\n\n else:\n bdF = None\n\n if dPinf is not None:\n bdPinf = np.zeros(dPinf.shape)\n for i in range(dPinf.shape[2]):\n bdPinf[:,:,i] = np.dot( T_inv, np.dot( dPinf[:,:,i], T_inv.T))\n\n# LL,islower = linalg.cho_factor(dPinf[:,:,i])\n# inds = np.triu_indices(dPinf[:,:,i].shape[0],k=1)\n# LL[inds] = 0.0\n# bLL = np.dot(T_inv, LL)\n# bdPinf[:,:,i] = np.dot( bLL, bLL.T)\n\n\n else:\n bdPinf = None\n\n if dP0 is not None:\n bdP0 = np.zeros(dP0.shape)\n for i in range(dP0.shape[2]):\n bdP0[:,:,i] = np.dot( T_inv, np.dot( dP0[:,:,i], T_inv.T))\n else:\n bdP0 = None\n\n\n bdQc = dQc # not affected\n\n # (F,L,Qc,H,Pinf,P0,dF,dQc,dPinf,dP0)\n\n return bF, bL, bQc, bH, bPinf, bP0, bdF, bdQc, bdPinf, bdP0", "response": "This function calculates the state - space balance of a single state - space model."} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nfunctioning returns the inverse square root of R matrix on step k.", "response": "def R_isrk(self, k):\n \"\"\"\n Function returns the inverse square root of R matrix on step k.\n \"\"\"\n ind = int(self.index[self.R_time_var_index, k])\n R = self.R[:, :, ind]\n\n if (R.shape[0] == 1): # 1-D case handle simplier. No storage\n # of the result, just compute it each time.\n inv_square_root = np.sqrt(1.0/R)\n else:\n if self.svd_each_time:\n\n (U, S, Vh) = sp.linalg.svd(R, full_matrices=False,\n compute_uv=True, overwrite_a=False,\n check_finite=True)\n\n inv_square_root = U * 1.0/np.sqrt(S)\n else:\n if ind in self.R_square_root:\n inv_square_root = self.R_square_root[ind]\n else:\n (U, S, Vh) = sp.linalg.svd(R, full_matrices=False,\n compute_uv=True,\n overwrite_a=False,\n check_finite=True)\n\n inv_square_root = U * 1.0/np.sqrt(S)\n\n self.R_square_root[ind] = inv_square_root\n\n return inv_square_root"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef Hk(self, k, m_pred, P_pred): # returns state iteration matrix\n\n return self.H[:, :, int(self.index[self.H_time_var_index, k])]", "response": "returns the covariance matrix of the is\n function"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef Qk(self, k):\n return self.Q[:, :, self.index[self.Q_time_var_index, k]]", "response": "Returns noise matrix of dynamic model on iteration k."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef Ak(self, k, m_pred, P_pred): # returns state iteration matrix\n\n return self.A[:, :, self.index[self.A_time_var_index, k]]", "response": "returns the Jacobian of measurement function"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef extended_kalman_filter(cls,p_state_dim, p_a, p_f_A, p_f_Q, p_h, p_f_H, p_f_R, Y, m_init=None,\n P_init=None,calc_log_likelihood=False):\n\n \"\"\"\n Extended Kalman Filter\n\n Input:\n -----------------\n\n p_state_dim: integer\n\n p_a: if None - the function from the linear model is assumed. No non-\n linearity in the dynamic is assumed.\n\n function (k, x_{k-1}, A_{k}). Dynamic function.\n k: (iteration number),\n x_{k-1}: (previous state)\n x_{k}: Jacobian matrices of f_a. In the linear case it is exactly A_{k}.\n\n p_f_A: matrix - in this case function which returns this matrix is assumed.\n Look at this parameter description in kalman_filter function.\n\n function (k, m, P) return Jacobian of dynamic function, it is\n passed into p_a.\n\n k: (iteration number),\n m: point where Jacobian is evaluated\n P: parameter for Jacobian, usually covariance matrix.\n\n p_f_Q: matrix. In this case function which returns this matrix is asumed.\n Look at this parameter description in kalman_filter function.\n\n function (k). Returns noise matrix of dynamic model on iteration k.\n k: (iteration number).\n\n p_h: if None - the function from the linear measurement model is assumed.\n No nonlinearity in the measurement is assumed.\n\n function (k, x_{k}, H_{k}). Measurement function.\n k: (iteration number),\n x_{k}: (current state)\n H_{k}: Jacobian matrices of f_h. In the linear case it is exactly H_{k}.\n\n p_f_H: matrix - in this case function which returns this matrix is assumed.\n function (k, m, P) return Jacobian of dynamic function, it is\n passed into p_h.\n k: (iteration number),\n m: point where Jacobian is evaluated\n P: parameter for Jacobian, usually covariance matrix.\n\n p_f_R: matrix. In this case function which returns this matrix is asumed.\n function (k). Returns noise matrix of measurement equation\n on iteration k.\n k: (iteration number).\n\n Y: matrix or vector\n Data. If Y is matrix then samples are along 0-th dimension and\n features along the 1-st. May have missing values.\n\n p_mean: vector\n Initial distribution mean. If None it is assumed to be zero\n\n P_init: square symmetric matrix or scalar\n Initial covariance of the states. If the parameter is scalar\n then it is assumed that initial covariance matrix is unit matrix\n multiplied by this scalar. If None the unit matrix is used instead.\n\n calc_log_likelihood: boolean\n Whether to calculate marginal likelihood of the state-space model.\n \"\"\"\n\n # Y\n Y.shape, old_Y_shape = cls._reshape_input_data(Y.shape)\n\n # m_init\n if m_init is None:\n m_init = np.zeros((p_state_dim,1))\n else:\n m_init = np.atleast_2d(m_init).T\n\n # P_init\n if P_init is None:\n P_init = np.eye(p_state_dim)\n elif not isinstance(P_init, collections.Iterable): #scalar\n P_init = P_init*np.eye(p_state_dim)\n\n if p_a is None:\n p_a = lambda k,m,A: np.dot(A, m)\n\n old_A_shape = None\n if not isinstance(p_f_A, types.FunctionType): # not a function but array\n p_f_A = np.atleast_1d(p_f_A)\n (p_A, old_A_shape) = cls._check_A_matrix(p_f_A)\n\n p_f_A = lambda k, m, P: p_A[:,:, 0] # make function\n else:\n if p_f_A(1, m_init, P_init).shape[0] != m_init.shape[0]:\n raise ValueError(\"p_f_A function returns matrix of wrong size\")\n\n old_Q_shape = None\n if not isinstance(p_f_Q, types.FunctionType): # not a function but array\n p_f_Q = np.atleast_1d(p_f_Q)\n (p_Q, old_Q_shape) = cls._check_Q_matrix(p_f_Q)\n\n p_f_Q = lambda k: p_Q[:,:, 0] # make function\n else:\n if p_f_Q(1).shape[0] != m_init.shape[0]:\n raise ValueError(\"p_f_Q function returns matrix of wrong size\")\n\n if p_h is None:\n lambda k,m,H: np.dot(H, m)\n\n old_H_shape = None\n if not isinstance(p_f_H, types.FunctionType): # not a function but array\n p_f_H = np.atleast_1d(p_f_H)\n (p_H, old_H_shape) = cls._check_H_matrix(p_f_H)\n\n p_f_H = lambda k, m, P: p_H # make function\n else:\n if p_f_H(1, m_init, P_init).shape[0] != Y.shape[1]:\n raise ValueError(\"p_f_H function returns matrix of wrong size\")\n\n old_R_shape = None\n if not isinstance(p_f_R, types.FunctionType): # not a function but array\n p_f_R = np.atleast_1d(p_f_R)\n (p_R, old_R_shape) = cls._check_H_matrix(p_f_R)\n\n p_f_R = lambda k: p_R # make function\n else:\n if p_f_R(1).shape[0] != m_init.shape[0]:\n raise ValueError(\"p_f_R function returns matrix of wrong size\")\n\n# class dynamic_callables_class(Dynamic_Model_Callables):\n#\n# Ak =\n# Qk =\n\n\n class measurement_callables_class(R_handling_Class):\n def __init__(self,R, index, R_time_var_index, unique_R_number):\n super(measurement_callables_class,self).__init__(R, index, R_time_var_index, unique_R_number)\n\n Hk = AddMethodToClass(f_H)\n f_h = AddMethodToClass(f_hl)\n\n\n (M, P,log_likelihood, grad_log_likelihood) = cls._kalman_algorithm_raw(p_state_dim, p_a, p_f_A, p_f_Q, p_h, p_f_H, p_f_R, Y, m_init,\n P_init, calc_log_likelihood,\n calc_grad_log_likelihood=False, grad_calc_params=None)\n\n if old_Y_shape is not None:\n Y.shape = old_Y_shape\n\n if old_A_shape is not None:\n p_A.shape = old_A_shape\n\n if old_Q_shape is not None:\n p_Q.shape = old_Q_shape\n\n if old_H_shape is not None:\n p_H.shape = old_H_shape\n\n if old_R_shape is not None:\n p_R.shape = old_R_shape\n\n return (M, P)", "response": "extended kalman filter class"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _kalman_prediction_step(k, p_m , p_P, p_dyn_model_callable, calc_grad_log_likelihood=False,\n p_dm = None, p_dP = None):\n \"\"\"\n Desctrete prediction function\n\n Input:\n k:int\n Iteration No. Starts at 0. Total number of iterations equal to the\n number of measurements.\n\n p_m: matrix of size (state_dim, time_series_no)\n Mean value from the previous step. For \"multiple time series mode\"\n it is matrix, second dimension of which correspond to different\n time series.\n\n p_P:\n Covariance matrix from the previous step.\n\n p_dyn_model_callable: class\n\n\n calc_grad_log_likelihood: boolean\n Whether to calculate gradient of the marginal likelihood\n of the state-space model. If true then the next parameter must\n provide the extra parameters for gradient calculation.\n\n p_dm: 3D array (state_dim, time_series_no, parameters_no)\n Mean derivatives from the previous step. For \"multiple time series mode\"\n it is 3D array, second dimension of which correspond to different\n time series.\n\n p_dP: 3D array (state_dim, state_dim, parameters_no)\n Mean derivatives from the previous step\n\n Output:\n ----------------------------\n m_pred, P_pred, dm_pred, dP_pred: metrices, 3D objects\n Results of the prediction steps.\n\n \"\"\"\n\n # index correspond to values from previous iteration.\n A = p_dyn_model_callable.Ak(k,p_m,p_P) # state transition matrix (or Jacobian)\n Q = p_dyn_model_callable.Qk(k) # state noise matrix\n\n # Prediction step ->\n m_pred = p_dyn_model_callable.f_a(k, p_m, A) # predicted mean\n P_pred = A.dot(p_P).dot(A.T) + Q # predicted variance\n # Prediction step <-\n\n if calc_grad_log_likelihood:\n dA_all_params = p_dyn_model_callable.dAk(k) # derivatives of A wrt parameters\n dQ_all_params = p_dyn_model_callable.dQk(k) # derivatives of Q wrt parameters\n\n param_number = p_dP.shape[2]\n\n # p_dm, p_dP - derivatives form the previoius step\n dm_pred = np.empty(p_dm.shape)\n dP_pred = np.empty(p_dP.shape)\n\n for j in range(param_number):\n dA = dA_all_params[:,:,j]\n dQ = dQ_all_params[:,:,j]\n\n dP = p_dP[:,:,j]\n dm = p_dm[:,:,j]\n dm_pred[:,:,j] = np.dot(dA, p_m) + np.dot(A, dm)\n # prediction step derivatives for current parameter:\n\n dP_pred[:,:,j] = np.dot( dA ,np.dot(p_P, A.T))\n dP_pred[:,:,j] += dP_pred[:,:,j].T\n dP_pred[:,:,j] += np.dot( A ,np.dot(dP, A.T)) + dQ\n\n dP_pred[:,:,j] = 0.5*(dP_pred[:,:,j] + dP_pred[:,:,j].T) #symmetrize\n else:\n dm_pred = None\n dP_pred = None\n\n return m_pred, P_pred, dm_pred, dP_pred", "response": "Desctrete prediction function\n\n Input:\n k:int\n Iteration No. Starts at 0. Total number of iterations equal to the\n number of measurements.\n\n p_m: matrix of size (state_dim, time_series_no)\n Mean value from the previous step. For \"multiple time series mode\"\n it is matrix, second dimension of which correspond to different\n time series.\n\n p_P:\n Covariance matrix from the previous step.\n\n p_dyn_model_callable: class\n\n\n calc_grad_log_likelihood: boolean\n Whether to calculate gradient of the marginal likelihood\n of the state-space model. If true then the next parameter must\n provide the extra parameters for gradient calculation.\n\n p_dm: 3D array (state_dim, time_series_no, parameters_no)\n Mean derivatives from the previous step. For \"multiple time series mode\"\n it is 3D array, second dimension of which correspond to different\n time series.\n\n p_dP: 3D array (state_dim, state_dim, parameters_no)\n Mean derivatives from the previous step\n\n Output:\n ----------------------------\n m_pred, P_pred, dm_pred, dP_pred: metrices, 3D objects\n Results of the prediction steps."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _kalman_prediction_step_SVD(k, p_m , p_P, p_dyn_model_callable, calc_grad_log_likelihood=False,\n p_dm = None, p_dP = None):\n \"\"\"\n Desctrete prediction function\n\n Input:\n k:int\n Iteration No. Starts at 0. Total number of iterations equal to the\n number of measurements.\n\n p_m: matrix of size (state_dim, time_series_no)\n Mean value from the previous step. For \"multiple time series mode\"\n it is matrix, second dimension of which correspond to different\n time series.\n\n p_P: tuple (Prev_cov, S, V)\n Covariance matrix from the previous step and its SVD decomposition.\n Prev_cov = V * S * V.T The tuple is (Prev_cov, S, V)\n\n p_dyn_model_callable: object\n\n calc_grad_log_likelihood: boolean\n Whether to calculate gradient of the marginal likelihood\n of the state-space model. If true then the next parameter must\n provide the extra parameters for gradient calculation.\n\n p_dm: 3D array (state_dim, time_series_no, parameters_no)\n Mean derivatives from the previous step. For \"multiple time series mode\"\n it is 3D array, second dimension of which correspond to different\n time series.\n\n p_dP: 3D array (state_dim, state_dim, parameters_no)\n Mean derivatives from the previous step\n\n Output:\n ----------------------------\n m_pred, P_pred, dm_pred, dP_pred: metrices, 3D objects\n Results of the prediction steps.\n\n \"\"\"\n\n # covariance from the previous step and its SVD decomposition\n # p_prev_cov = v * S * V.T\n Prev_cov, S_old, V_old = p_P\n #p_prev_cov_tst = np.dot(p_V, (p_S * p_V).T) # reconstructed covariance from the previous step\n\n # index correspond to values from previous iteration.\n A = p_dyn_model_callable.Ak(k,p_m,Prev_cov) # state transition matrix (or Jacobian)\n Q = p_dyn_model_callable.Qk(k) # state noise matrx. This is necessary for the square root calculation (next step)\n Q_sr = p_dyn_model_callable.Q_srk(k)\n # Prediction step ->\n m_pred = p_dyn_model_callable.f_a(k, p_m, A) # predicted mean\n\n # coavariance prediction have changed:\n svd_1_matr = np.vstack( ( (np.sqrt(S_old)* np.dot(A,V_old)).T , Q_sr.T) )\n (U,S,Vh) = sp.linalg.svd( svd_1_matr,full_matrices=False, compute_uv=True,\n overwrite_a=False,check_finite=True)\n\n # predicted variance computed by the regular method. For testing\n #P_pred_tst = A.dot(Prev_cov).dot(A.T) + Q\n V_new = Vh.T\n S_new = S**2\n\n P_pred = np.dot(V_new * S_new, V_new.T) # prediction covariance\n P_pred = (P_pred, S_new, Vh.T)\n # Prediction step <-\n\n # derivatives\n if calc_grad_log_likelihood:\n dA_all_params = p_dyn_model_callable.dAk(k) # derivatives of A wrt parameters\n dQ_all_params = p_dyn_model_callable.dQk(k) # derivatives of Q wrt parameters\n\n param_number = p_dP.shape[2]\n\n # p_dm, p_dP - derivatives form the previoius step\n dm_pred = np.empty(p_dm.shape)\n dP_pred = np.empty(p_dP.shape)\n\n for j in range(param_number):\n dA = dA_all_params[:,:,j]\n dQ = dQ_all_params[:,:,j]\n\n #dP = p_dP[:,:,j]\n #dm = p_dm[:,:,j]\n dm_pred[:,:,j] = np.dot(dA, p_m) + np.dot(A, p_dm[:,:,j])\n # prediction step derivatives for current parameter:\n\n\n dP_pred[:,:,j] = np.dot( dA ,np.dot(Prev_cov, A.T))\n dP_pred[:,:,j] += dP_pred[:,:,j].T\n dP_pred[:,:,j] += np.dot( A ,np.dot(p_dP[:,:,j], A.T)) + dQ\n\n dP_pred[:,:,j] = 0.5*(dP_pred[:,:,j] + dP_pred[:,:,j].T) #symmetrize\n else:\n dm_pred = None\n dP_pred = None\n\n return m_pred, P_pred, dm_pred, dP_pred", "response": "Kalman prediction function for the Kalman model."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _kalman_update_step(k, p_m , p_P, p_meas_model_callable, measurement, calc_log_likelihood= False,\n calc_grad_log_likelihood=False, p_dm = None, p_dP = None):\n \"\"\"\n Input:\n\n k: int\n Iteration No. Starts at 0. Total number of iterations equal to the\n number of measurements.\n\n m_P: matrix of size (state_dim, time_series_no)\n Mean value from the previous step. For \"multiple time series mode\"\n it is matrix, second dimension of which correspond to different\n time series.\n\n p_P:\n Covariance matrix from the prediction step.\n\n p_meas_model_callable: object\n\n measurement: (measurement_dim, time_series_no) matrix\n One measurement used on the current update step. For\n \"multiple time series mode\" it is matrix, second dimension of\n which correspond to different time series.\n\n calc_log_likelihood: boolean\n Whether to calculate marginal likelihood of the state-space model.\n\n calc_grad_log_likelihood: boolean\n Whether to calculate gradient of the marginal likelihood\n of the state-space model. If true then the next parameter must\n provide the extra parameters for gradient calculation.\n\n p_dm: 3D array (state_dim, time_series_no, parameters_no)\n Mean derivatives from the prediction step. For \"multiple time series mode\"\n it is 3D array, second dimension of which correspond to different\n time series.\n\n p_dP: array\n Covariance derivatives from the prediction step.\n\n Output:\n ----------------------------\n m_upd, P_upd, dm_upd, dP_upd: metrices, 3D objects\n Results of the prediction steps.\n\n log_likelihood_update: double or 1D array\n Update to the log_likelihood from this step\n\n d_log_likelihood_update: (grad_params_no, time_series_no) matrix\n Update to the gradient of log_likelihood, \"multiple time series mode\"\n adds extra columns to the gradient.\n\n \"\"\"\n #import pdb; pdb.set_trace()\n\n m_pred = p_m # from prediction step\n P_pred = p_P # from prediction step\n\n H = p_meas_model_callable.Hk(k, m_pred, P_pred)\n R = p_meas_model_callable.Rk(k)\n\n time_series_no = p_m.shape[1] # number of time serieses\n\n log_likelihood_update=None; dm_upd=None; dP_upd=None; d_log_likelihood_update=None\n # Update step (only if there is data)\n #if not np.any(np.isnan(measurement)): # TODO: if some dimensions are missing, do properly computations for other.\n v = measurement-p_meas_model_callable.f_h(k, m_pred, H)\n S = H.dot(P_pred).dot(H.T) + R\n if measurement.shape[0]==1: # measurements are one dimensional\n if (S < 0):\n raise ValueError(\"Kalman Filter Update: S is negative step %i\" % k )\n #import pdb; pdb.set_trace()\n\n K = P_pred.dot(H.T) / S\n if calc_log_likelihood:\n log_likelihood_update = -0.5 * ( np.log(2*np.pi) + np.log(S) +\n v*v / S)\n #log_likelihood_update = log_likelihood_update[0,0] # to make int\n if np.any(np.isnan(log_likelihood_update)): # some member in P_pred is None.\n raise ValueError(\"Nan values in likelihood update!\")\n LL = None; islower = None\n else:\n LL,islower = linalg.cho_factor(S)\n K = linalg.cho_solve((LL,islower), H.dot(P_pred.T)).T\n\n if calc_log_likelihood:\n log_likelihood_update = -0.5 * ( v.shape[0]*np.log(2*np.pi) +\n 2*np.sum( np.log(np.diag(LL)) ) +\\\n np.sum((linalg.cho_solve((LL,islower),v)) * v, axis = 0) ) # diagonal of v.T*S^{-1}*v\n\n if calc_grad_log_likelihood:\n dm_pred_all_params = p_dm # derivativas of the prediction phase\n dP_pred_all_params = p_dP\n\n param_number = p_dP.shape[2]\n\n dH_all_params = p_meas_model_callable.dHk(k)\n dR_all_params = p_meas_model_callable.dRk(k)\n\n dm_upd = np.empty(dm_pred_all_params.shape)\n dP_upd = np.empty(dP_pred_all_params.shape)\n\n # firts dimension parameter_no, second - time series number\n d_log_likelihood_update = np.empty((param_number,time_series_no))\n for param in range(param_number):\n\n dH = dH_all_params[:,:,param]\n dR = dR_all_params[:,:,param]\n\n dm_pred = dm_pred_all_params[:,:,param]\n dP_pred = dP_pred_all_params[:,:,param]\n\n # Terms in the likelihood derivatives\n dv = - np.dot( dH, m_pred) - np.dot( H, dm_pred)\n dS = np.dot(dH, np.dot( P_pred, H.T))\n dS += dS.T\n dS += np.dot(H, np.dot( dP_pred, H.T)) + dR\n\n # TODO: maybe symmetrize dS\n\n #dm and dP for the next stem\n if LL is not None: # the state vector is not a scalar\n tmp1 = linalg.cho_solve((LL,islower), H).T\n tmp2 = linalg.cho_solve((LL,islower), dH).T\n tmp3 = linalg.cho_solve((LL,islower), dS).T\n else: # the state vector is a scalar\n tmp1 = H.T / S\n tmp2 = dH.T / S\n tmp3 = dS.T / S\n\n dK = np.dot( dP_pred, tmp1) + np.dot( P_pred, tmp2) - \\\n np.dot( P_pred, np.dot( tmp1, tmp3 ) )\n\n # terms required for the next step, save this for each parameter\n dm_upd[:,:,param] = dm_pred + np.dot(dK, v) + np.dot(K, dv)\n\n dP_upd[:,:,param] = -np.dot(dK, np.dot(S, K.T))\n dP_upd[:,:,param] += dP_upd[:,:,param].T\n dP_upd[:,:,param] += dP_pred - np.dot(K , np.dot( dS, K.T))\n\n dP_upd[:,:,param] = 0.5*(dP_upd[:,:,param] + dP_upd[:,:,param].T) #symmetrize\n # computing the likelihood change for each parameter:\n if LL is not None: # the state vector is not 1D\n #tmp4 = linalg.cho_solve((LL,islower), dv)\n tmp5 = linalg.cho_solve((LL,islower), v)\n else: # the state vector is a scalar\n #tmp4 = dv / S\n tmp5 = v / S\n\n\n d_log_likelihood_update[param,:] = -(0.5*np.sum(np.diag(tmp3)) + \\\n np.sum(tmp5*dv, axis=0) - 0.5 * np.sum(tmp5 * np.dot(dS, tmp5), axis=0) )\n # Before\n #d_log_likelihood_update[param,0] = -(0.5*np.sum(np.diag(tmp3)) + \\\n #np.dot(tmp5.T, dv) - 0.5 * np.dot(tmp5.T ,np.dot(dS, tmp5)) )\n\n\n\n # Compute the actual updates for mean and variance of the states.\n m_upd = m_pred + K.dot( v )\n\n # Covariance update and ensure it is symmetric\n P_upd = K.dot(S).dot(K.T)\n P_upd = 0.5*(P_upd + P_upd.T)\n P_upd = P_pred - P_upd# this update matrix is symmetric\n\n return m_upd, P_upd, log_likelihood_update, dm_upd, dP_upd, d_log_likelihood_update", "response": "Internal function to update the state - space model for a given iteration."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ninputting k: int Iteration No. Starts at 0. Total number of iterations equal to the number of measurements. m_P: matrix of size (state_dim, time_series_no) Mean value from the previous step. For \"multiple time series mode\" it is matrix, second dimension of which correspond to different time series. p_P: tuple (P_pred, S, V) Covariance matrix from the prediction step and its SVD decomposition. P_pred = V * S * V.T The tuple is (P_pred, S, V) p_h: function (k, x_{k}, H_{k}). Measurement function. k (iteration number), starts at 0 x_{k} state H_{k} Jacobian matrices of f_h. In the linear case it is exactly H_{k}. p_f_H: function (k, m, P) return Jacobian of measurement function, it is passed into p_h. k (iteration number), starts at 0 m: point where Jacobian is evaluated P: parameter for Jacobian, usually covariance matrix. p_f_R: function (k). Returns noise matrix of measurement equation on iteration k. k (iteration number). starts at 0 p_f_iRsr: function (k). Returns the square root of the noise matrix of measurement equation on iteration k. k (iteration number). starts at 0 measurement: (measurement_dim, time_series_no) matrix One measurement used on the current update step. For \"multiple time series mode\" it is matrix, second dimension of which correspond to different time series. calc_log_likelihood: boolean Whether to calculate marginal likelihood of the state-space model. calc_grad_log_likelihood: boolean Whether to calculate gradient of the marginal likelihood of the state-space model. If true then the next parameter must provide the extra parameters for gradient calculation. p_dm: 3D array (state_dim, time_series_no, parameters_no) Mean derivatives from the prediction step. For \"multiple time series mode\" it is 3D array, second dimension of which correspond to different time series. p_dP: array Covariance derivatives from the prediction step. grad_calc_params_2: List or None List with derivatives. The first component is 'f_dH' - function(k) which returns the derivative of H. The second element is 'f_dR' - function(k). Function which returns the derivative of R. Output: ---------------------------- m_upd, P_upd, dm_upd, dP_upd: metrices, 3D objects Results of the prediction steps. log_likelihood_update: double or 1D array Update to the log_likelihood from this step d_log_likelihood_update: (grad_params_no, time_series_no) matrix Update to the gradient of log_likelihood, \"multiple time series mode\" adds extra columns to the gradient.", "response": "def _kalman_update_step_SVD(k, p_m , p_P, p_meas_model_callable, measurement, calc_log_likelihood= False,\n calc_grad_log_likelihood=False, p_dm = None, p_dP = None):\n \"\"\"\n Input:\n\n k: int\n Iteration No. Starts at 0. Total number of iterations equal to the\n number of measurements.\n\n m_P: matrix of size (state_dim, time_series_no)\n Mean value from the previous step. For \"multiple time series mode\"\n it is matrix, second dimension of which correspond to different\n time series.\n\n p_P: tuple (P_pred, S, V)\n Covariance matrix from the prediction step and its SVD decomposition.\n P_pred = V * S * V.T The tuple is (P_pred, S, V)\n\n p_h: function (k, x_{k}, H_{k}). Measurement function.\n k (iteration number), starts at 0\n x_{k} state\n H_{k} Jacobian matrices of f_h. In the linear case it is exactly H_{k}.\n\n p_f_H: function (k, m, P) return Jacobian of measurement function, it is\n passed into p_h.\n k (iteration number), starts at 0\n m: point where Jacobian is evaluated\n P: parameter for Jacobian, usually covariance matrix.\n\n p_f_R: function (k). Returns noise matrix of measurement equation\n on iteration k.\n k (iteration number). starts at 0\n\n p_f_iRsr: function (k). Returns the square root of the noise matrix of\n measurement equation on iteration k.\n k (iteration number). starts at 0\n\n measurement: (measurement_dim, time_series_no) matrix\n One measurement used on the current update step. For\n \"multiple time series mode\" it is matrix, second dimension of\n which correspond to different time series.\n\n calc_log_likelihood: boolean\n Whether to calculate marginal likelihood of the state-space model.\n\n calc_grad_log_likelihood: boolean\n Whether to calculate gradient of the marginal likelihood\n of the state-space model. If true then the next parameter must\n provide the extra parameters for gradient calculation.\n\n p_dm: 3D array (state_dim, time_series_no, parameters_no)\n Mean derivatives from the prediction step. For \"multiple time series mode\"\n it is 3D array, second dimension of which correspond to different\n time series.\n\n p_dP: array\n Covariance derivatives from the prediction step.\n\n grad_calc_params_2: List or None\n List with derivatives. The first component is 'f_dH' - function(k)\n which returns the derivative of H. The second element is 'f_dR'\n - function(k). Function which returns the derivative of R.\n\n Output:\n ----------------------------\n m_upd, P_upd, dm_upd, dP_upd: metrices, 3D objects\n Results of the prediction steps.\n\n log_likelihood_update: double or 1D array\n Update to the log_likelihood from this step\n\n d_log_likelihood_update: (grad_params_no, time_series_no) matrix\n Update to the gradient of log_likelihood, \"multiple time series mode\"\n adds extra columns to the gradient.\n\n \"\"\"\n\n #import pdb; pdb.set_trace()\n\n m_pred = p_m # from prediction step\n P_pred,S_pred,V_pred = p_P # from prediction step\n\n H = p_meas_model_callable.Hk(k, m_pred, P_pred)\n R = p_meas_model_callable.Rk(k)\n R_isr = p_meas_model_callable.R_isrk(k) # square root of the inverse of R matrix\n\n time_series_no = p_m.shape[1] # number of time serieses\n\n log_likelihood_update=None; dm_upd=None; dP_upd=None; d_log_likelihood_update=None\n # Update step (only if there is data)\n #if not np.any(np.isnan(measurement)): # TODO: if some dimensions are missing, do properly computations for other.\n v = measurement-p_meas_model_callable.f_h(k, m_pred, H)\n\n svd_2_matr = np.vstack( ( np.dot( R_isr.T, np.dot(H, V_pred)) , np.diag( 1.0/np.sqrt(S_pred) ) ) )\n\n (U,S,Vh) = sp.linalg.svd( svd_2_matr,full_matrices=False, compute_uv=True,\n overwrite_a=False,check_finite=True)\n\n # P_upd = U_upd S_upd**2 U_upd.T\n U_upd = np.dot(V_pred, Vh.T)\n S_upd = (1.0/S)**2\n\n P_upd = np.dot(U_upd * S_upd, U_upd.T) # update covariance\n P_upd = (P_upd,S_upd,U_upd) # tuple to pass to the next step\n\n # stil need to compute S and K for derivative computation\n S = H.dot(P_pred).dot(H.T) + R\n if measurement.shape[0]==1: # measurements are one dimensional\n if (S < 0):\n raise ValueError(\"Kalman Filter Update: S is negative step %i\" % k )\n #import pdb; pdb.set_trace()\n\n K = P_pred.dot(H.T) / S\n if calc_log_likelihood:\n log_likelihood_update = -0.5 * ( np.log(2*np.pi) + np.log(S) +\n v*v / S)\n #log_likelihood_update = log_likelihood_update[0,0] # to make int\n if np.any(np.isnan(log_likelihood_update)): # some member in P_pred is None.\n raise ValueError(\"Nan values in likelihood update!\")\n LL = None; islower = None\n else:\n LL,islower = linalg.cho_factor(S)\n K = linalg.cho_solve((LL,islower), H.dot(P_pred.T)).T\n\n if calc_log_likelihood:\n log_likelihood_update = -0.5 * ( v.shape[0]*np.log(2*np.pi) +\n 2*np.sum( np.log(np.diag(LL)) ) +\\\n np.sum((linalg.cho_solve((LL,islower),v)) * v, axis = 0) ) # diagonal of v.T*S^{-1}*v\n\n\n # Old method of computing updated covariance (for testing) ->\n #P_upd_tst = K.dot(S).dot(K.T)\n #P_upd_tst = 0.5*(P_upd_tst + P_upd_tst.T)\n #P_upd_tst = P_pred - P_upd_tst# this update matrix is symmetric\n # Old method of computing updated covariance (for testing) <-\n\n if calc_grad_log_likelihood:\n dm_pred_all_params = p_dm # derivativas of the prediction phase\n dP_pred_all_params = p_dP\n\n param_number = p_dP.shape[2]\n\n dH_all_params = p_meas_model_callable.dHk(k)\n dR_all_params = p_meas_model_callable.dRk(k)\n\n dm_upd = np.empty(dm_pred_all_params.shape)\n dP_upd = np.empty(dP_pred_all_params.shape)\n\n # firts dimension parameter_no, second - time series number\n d_log_likelihood_update = np.empty((param_number,time_series_no))\n for param in range(param_number):\n\n dH = dH_all_params[:,:,param]\n dR = dR_all_params[:,:,param]\n\n dm_pred = dm_pred_all_params[:,:,param]\n dP_pred = dP_pred_all_params[:,:,param]\n\n # Terms in the likelihood derivatives\n dv = - np.dot( dH, m_pred) - np.dot( H, dm_pred)\n dS = np.dot(dH, np.dot( P_pred, H.T))\n dS += dS.T\n dS += np.dot(H, np.dot( dP_pred, H.T)) + dR\n\n # TODO: maybe symmetrize dS\n\n #dm and dP for the next stem\n if LL is not None: # the state vector is not a scalar\n tmp1 = linalg.cho_solve((LL,islower), H).T\n tmp2 = linalg.cho_solve((LL,islower), dH).T\n tmp3 = linalg.cho_solve((LL,islower), dS).T\n else: # the state vector is a scalar\n tmp1 = H.T / S\n tmp2 = dH.T / S\n tmp3 = dS.T / S\n\n dK = np.dot( dP_pred, tmp1) + np.dot( P_pred, tmp2) - \\\n np.dot( P_pred, np.dot( tmp1, tmp3 ) )\n\n # terms required for the next step, save this for each parameter\n dm_upd[:,:,param] = dm_pred + np.dot(dK, v) + np.dot(K, dv)\n\n dP_upd[:,:,param] = -np.dot(dK, np.dot(S, K.T))\n dP_upd[:,:,param] += dP_upd[:,:,param].T\n dP_upd[:,:,param] += dP_pred - np.dot(K , np.dot( dS, K.T))\n\n dP_upd[:,:,param] = 0.5*(dP_upd[:,:,param] + dP_upd[:,:,param].T) #symmetrize\n # computing the likelihood change for each parameter:\n if LL is not None: # the state vector is not 1D\n tmp5 = linalg.cho_solve((LL,islower), v)\n else: # the state vector is a scalar\n tmp5 = v / S\n\n\n d_log_likelihood_update[param,:] = -(0.5*np.sum(np.diag(tmp3)) + \\\n np.sum(tmp5*dv, axis=0) - 0.5 * np.sum(tmp5 * np.dot(dS, tmp5), axis=0) )\n # Before\n #d_log_likelihood_update[param,0] = -(0.5*np.sum(np.diag(tmp3)) + \\\n #np.dot(tmp5.T, dv) - 0.5 * np.dot(tmp5.T ,np.dot(dS, tmp5)) )\n\n # Compute the actual updates for mean of the states. Variance update\n # is computed earlier.\n m_upd = m_pred + K.dot( v )\n\n return m_upd, P_upd, log_likelihood_update, dm_upd, dP_upd, d_log_likelihood_update"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _rts_smoother_update_step(k, p_m , p_P, p_m_pred, p_P_pred, p_m_prev_step,\n p_P_prev_step, p_dynamic_callables):\n \"\"\"\n Rauch\u2013Tung\u2013Striebel(RTS) update step\n\n Input:\n -----------------------------\n k: int\n Iteration No. Starts at 0. Total number of iterations equal to the\n number of measurements.\n\n p_m: matrix of size (state_dim, time_series_no)\n Filter mean on step k\n\n p_P: matrix of size (state_dim,state_dim)\n Filter Covariance on step k\n\n p_m_pred: matrix of size (state_dim, time_series_no)\n Means from the smoother prediction step.\n\n p_P_pred:\n Covariance from the smoother prediction step.\n\n p_m_prev_step\n Smoother mean from the previous step.\n\n p_P_prev_step:\n Smoother covariance from the previous step.\n\n p_f_A: function (k, m, P) return Jacobian of dynamic function, it is\n passed into p_a.\n k (iteration number), starts at 0\n m: point where Jacobian is evaluated\n P: parameter for Jacobian, usually covariance matrix.\n\n \"\"\"\n\n A = p_dynamic_callables.Ak(k,p_m,p_P) # state transition matrix (or Jacobian)\n\n tmp = np.dot( A, p_P.T)\n if A.shape[0] == 1: # 1D states\n G = tmp.T / p_P_pred # P[:,:,k] is symmetric\n else:\n try:\n LL,islower = linalg.cho_factor(p_P_pred)\n G = linalg.cho_solve((LL,islower),tmp).T\n except:\n # It happende that p_P_pred has several near zero eigenvalues\n # hence the Cholesky method does not work.\n res = sp.linalg.lstsq(p_P_pred, tmp)\n G = res[0].T\n\n m_upd = p_m + G.dot( p_m_prev_step-p_m_pred )\n P_upd = p_P + G.dot( p_P_prev_step-p_P_pred).dot(G.T)\n\n P_upd = 0.5*(P_upd + P_upd.T)\n\n return m_upd, P_upd, G", "response": "Update the state machine for a single step."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _EM_gradient(A,Q,H,R,m_init,P_init,measurements, M, P, G, dA, dQ, dH, dR, dm_init, dP_init):\n import pdb; pdb.set_trace();\n\n param_number = dA.shape[-1]\n d_log_likelihood_update = np.empty((param_number,1))\n\n sample_no = measurements.shape[0]\n P_1 = P[1:,:,:] # remove 0-th step\n P_2 = P[0:-1,:,:] # remove 0-th step\n\n M_1 = M[1:,:] # remove 0-th step\n M_2 = M[0:-1,:] # remove the last step\n\n Sigma = np.mean(P_1,axis=0) + np.dot(M_1.T, M_1) / sample_no #\n Phi = np.mean(P_2,axis=0) + np.dot(M_2.T, M_2) / sample_no #\n\n B = np.dot( measurements.T, M_1 )/ sample_no\n C = (sp.einsum( 'ijk,ikl', P_1, G) + np.dot(M_1.T, M_2)) / sample_no #\n\n# C1 = np.zeros( (P_1.shape[1],P_1.shape[1]) )\n# for k in range(P_1.shape[0]):\n# C1 += np.dot(P_1[k,:,:],G[k,:,:]) + sp.outer( M_1[k,:], M_2[k,:] )\n# C1 = C1 / sample_no\n\n D = np.dot( measurements.T, measurements ) / sample_no\n\n try:\n P_init_inv = sp.linalg.inv(P_init)\n\n if np.max( np.abs(P_init_inv)) > 10e13:\n compute_P_init_terms = False\n else:\n compute_P_init_terms = True\n except np.linalg.LinAlgError:\n compute_P_init_terms = False\n\n try:\n Q_inv = sp.linalg.inv(Q)\n\n if np.max( np.abs(Q_inv)) > 10e13:\n compute_Q_terms = False\n else:\n compute_Q_terms = True\n except np.linalg.LinAlgError:\n compute_Q_terms = False\n\n try:\n R_inv = sp.linalg.inv(R)\n\n if np.max( np.abs(R_inv)) > 10e13:\n compute_R_terms = False\n else:\n compute_R_terms = True\n except np.linalg.LinAlgError:\n compute_R_terms = False\n\n\n d_log_likelihood_update = np.zeros((param_number,1))\n for j in range(param_number):\n if compute_P_init_terms:\n d_log_likelihood_update[j,:] -= 0.5 * np.sum(P_init_inv* dP_init[:,:,j].T ) #p #m\n\n M0_smoothed = M[0]; M0_smoothed.shape = (M0_smoothed.shape[0],1)\n tmp1 = np.dot( dP_init[:,:,j], np.dot( P_init_inv, (P[0,:,:] + sp.outer( (M0_smoothed - m_init), (M0_smoothed - m_init) )) ) ) #p #m\n d_log_likelihood_update[j,:] += 0.5 * np.sum(P_init_inv* tmp1.T )\n\n tmp2 = sp.outer( dm_init[:,j], M0_smoothed )\n tmp2 += tmp2.T\n d_log_likelihood_update[j,:] += 0.5 * np.sum(P_init_inv* tmp2.T )\n\n if compute_Q_terms:\n\n d_log_likelihood_update[j,:] -= sample_no/2.0 * np.sum(Q_inv* dQ[:,:,j].T ) #m\n\n tmp1 = np.dot(C,A.T); tmp1 += tmp1.T; tmp1 = Sigma - tmp1 + np.dot(A, np.dot(Phi,A.T)) #m\n tmp1 = np.dot( dQ[:,:,j], np.dot( Q_inv, tmp1) )\n d_log_likelihood_update[j,:] += sample_no/2.0 * np.sum(Q_inv * tmp1.T)\n\n tmp2 = np.dot( dA[:,:,j], C.T); tmp2 += tmp2.T;\n tmp3 = np.dot(dA[:,:,j], np.dot(Phi,A.T)); tmp3 += tmp3.T\n d_log_likelihood_update[j,:] -= sample_no/2.0 * np.sum(Q_inv.T * (tmp3 - tmp2) )\n\n if compute_R_terms:\n d_log_likelihood_update[j,:] -= sample_no/2.0 * np.sum(R_inv* dR[:,:,j].T )\n\n tmp1 = np.dot(B,H.T); tmp1 += tmp1.T; tmp1 = D - tmp1 + np.dot(H, np.dot(Sigma,H.T))\n tmp1 = np.dot( dR[:,:,j], np.dot( R_inv, tmp1) )\n d_log_likelihood_update[j,:] += sample_no/2.0 * np.sum(R_inv * tmp1.T)\n\n tmp2 = np.dot( dH[:,:,j], B.T); tmp2 += tmp2.T;\n tmp3 = np.dot(dH[:,:,j], np.dot(Sigma,H.T)); tmp3 += tmp3.T\n d_log_likelihood_update[j,:] -= sample_no/2.0 * np.sum(R_inv.T * (tmp3 - tmp2) )\n\n return d_log_likelihood_update", "response": "Compute the gradient of the EM algorithm."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _check_SS_matrix(p_M, state_dim, measurement_dim, which='A'):\n\n old_M_shape = None\n if len(p_M.shape) < 3: # new shape is 3 dimensional\n old_M_shape = p_M.shape # save shape to restore it on exit\n if len(p_M.shape) == 2: # matrix\n p_M.shape = (p_M.shape[0],p_M.shape[1],1)\n elif len(p_M.shape) == 1: # scalar but in array already\n if (p_M.shape[0] != 1):\n raise ValueError(\"Matrix %s is an 1D array, while it must be a matrix or scalar\", which)\n else:\n p_M.shape = (1,1,1)\n\n if (which == 'A') or (which == 'Q'):\n if (p_M.shape[0] != state_dim) or (p_M.shape[1] != state_dim):\n raise ValueError(\"%s must be a square matrix of size (%i,%i)\" % (which, state_dim, state_dim))\n if (which == 'H'):\n if (p_M.shape[0] != measurement_dim) or (p_M.shape[1] != state_dim):\n raise ValueError(\"H must be of shape (measurement_dim, state_dim) (%i,%i)\" % (measurement_dim, state_dim))\n if (which == 'R'):\n if (p_M.shape[0] != measurement_dim) or (p_M.shape[1] != measurement_dim):\n raise ValueError(\"R must be of shape (measurement_dim, measurement_dim) (%i,%i)\" % (measurement_dim, measurement_dim))\n\n return (p_M,old_M_shape)", "response": "Check that the matrix p_M has correct shape for the KF algorithm."} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nfunctions checks the gradients of the state matrices for marginal likelihood and gradient parameters calculation.", "response": "def _check_grad_state_matrices(dM, state_dim, grad_params_no, which = 'dA'):\n \"\"\"\n Function checks (mostly check dimensions) matrices for marginal likelihood\n gradient parameters calculation. It check dA, dQ matrices.\n\n Input:\n -------------\n dM: None, scaler or 3D matrix\n It is supposed to be (state_dim,state_dim,grad_params_no) matrix.\n If None then zero matrix is assumed. If scalar then the function\n checks consistency with \"state_dim\" and \"grad_params_no\".\n\n state_dim: int\n State dimensionality\n\n grad_params_no: int\n How many parrameters of likelihood gradient in total.\n\n which: string\n 'dA' or 'dQ'\n\n\n Output:\n --------------\n function of (k) which returns the parameters matrix.\n\n \"\"\"\n\n\n if dM is None:\n dM=np.zeros((state_dim,state_dim,grad_params_no))\n elif isinstance(dM, np.ndarray):\n if state_dim == 1:\n if len(dM.shape) < 3:\n dM.shape = (1,1,1)\n else:\n if len(dM.shape) < 3:\n dM.shape = (state_dim,state_dim,1)\n elif isinstance(dM, np.int):\n if state_dim > 1:\n raise ValueError(\"When computing likelihood gradient wrong %s dimension.\" % which)\n else:\n dM = np.ones((1,1,1)) * dM\n\n# if not isinstance(dM, types.FunctionType):\n# f_dM = lambda k: dM\n# else:\n# f_dM = dM\n\n return dM"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _check_grad_measurement_matrices(dM, state_dim, grad_params_no, measurement_dim, which = 'dH'):\n\n if dM is None:\n if which == 'dH':\n dM=np.zeros((measurement_dim ,state_dim,grad_params_no))\n elif which == 'dR':\n dM=np.zeros((measurement_dim,measurement_dim,grad_params_no))\n elif isinstance(dM, np.ndarray):\n if state_dim == 1:\n if len(dM.shape) < 3:\n dM.shape = (1,1,1)\n else:\n if len(dM.shape) < 3:\n if which == 'dH':\n dM.shape = (measurement_dim,state_dim,1)\n elif which == 'dR':\n dM.shape = (measurement_dim,measurement_dim,1)\n elif isinstance(dM, np.int):\n if state_dim > 1:\n raise ValueError(\"When computing likelihood gradient wrong dH dimension.\")\n else:\n dM = np.ones((1,1,1)) * dM\n\n# if not isinstance(dM, types.FunctionType):\n# f_dM = lambda k: dM\n# else:\n# f_dM = dM\n\n return dM", "response": "Function checks the gradient of the marginal likelihood and gradient parameters calculation."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef cont_discr_kalman_filter(cls, F, L, Qc, p_H, p_R, P_inf, X, Y, index = None,\n m_init=None, P_init=None,\n p_kalman_filter_type='regular',\n calc_log_likelihood=False,\n calc_grad_log_likelihood=False,\n grad_params_no=0, grad_calc_params=None):\n \"\"\"\n This function implements the continuous-discrete Kalman Filter algorithm\n These notations for the State-Space model are assumed:\n d/dt x(t) = F * x(t) + L * w(t); w(t) ~ N(0, Qc)\n y_{k} = H_{k} * x_{k} + r_{k}; r_{k-1} ~ N(0, R_{k})\n\n Returns estimated filter distributions x_{k} ~ N(m_{k}, P(k))\n\n Current Features:\n ----------------------------------------\n 1) The function generaly do not modify the passed parameters. If\n it happens then it is an error. There are several exeprions: scalars\n can be modified into a matrix, in some rare cases shapes of\n the derivatives matrices may be changed, it is ignored for now.\n\n 2) Copies of F,L,Qc are created in memory because they may be used later\n in smoother. References to copies are kept in \"AQcomp\" object\n return parameter.\n\n 3) Function support \"multiple time series mode\" which means that exactly\n the same State-Space model is used to filter several sets of measurements.\n In this case third dimension of Y should include these state-space measurements\n Log_likelihood and Grad_log_likelihood have the corresponding dimensions then.\n\n 4) Calculation of Grad_log_likelihood is not supported if matrices\n H, or R changes overf time (with index k). (later may be changed)\n\n 5) Measurement may include missing values. In this case update step is\n not done for this measurement. (later may be changed)\n\n Input:\n -----------------\n\n F: (state_dim, state_dim) matrix\n F in the model.\n\n L: (state_dim, noise_dim) matrix\n L in the model.\n\n Qc: (noise_dim, noise_dim) matrix\n Q_c in the model.\n\n p_H: scalar, matrix (measurement_dim, state_dim) , 3D array\n H_{k} in the model. If matrix then H_{k} = H - constant.\n If it is 3D array then H_{k} = p_Q[:,:, index[2,k]]\n\n p_R: scalar, square symmetric matrix, 3D array\n R_{k} in the model. If matrix then R_{k} = R - constant.\n If it is 3D array then R_{k} = p_R[:,:, index[3,k]]\n\n P_inf: (state_dim, state_dim) matrix\n State varince matrix on infinity.\n\n X: 1D array\n Time points of measurements. Needed for converting continuos\n problem to the discrete one.\n\n Y: matrix or vector or 3D array\n Data. If Y is matrix then samples are along 0-th dimension and\n features along the 1-st. If 3D array then third dimension\n correspond to \"multiple time series mode\".\n\n index: vector\n Which indices (on 3-rd dimension) from arrays p_H, p_R to use\n on every time step. If this parameter is None then it is assumed\n that p_H, p_R do not change over time and indices are not needed.\n index[0,:] - correspond to H, index[1,:] - correspond to R\n If index.shape[0] == 1, it is assumed that indides for all matrices\n are the same.\n\n m_init: vector or matrix\n Initial distribution mean. If None it is assumed to be zero.\n For \"multiple time series mode\" it is matrix, second dimension of\n which correspond to different time series. In regular case (\"one\n time series mode\") it is a vector.\n\n P_init: square symmetric matrix or scalar\n Initial covariance of the states. If the parameter is scalar\n then it is assumed that initial covariance matrix is unit matrix\n multiplied by this scalar. If None the unit matrix is used instead.\n \"multiple time series mode\" does not affect it, since it does not\n affect anything related to state variaces.\n\n p_kalman_filter_type: string, one of ('regular', 'svd')\n Which Kalman Filter is used. Regular or SVD. SVD is more numerically\n stable, in particular, Covariace matrices are guarantied to be\n positive semi-definite. However, 'svd' works slower, especially for\n small data due to SVD call overhead.\n\n calc_log_likelihood: boolean\n Whether to calculate marginal likelihood of the state-space model.\n\n calc_grad_log_likelihood: boolean\n Whether to calculate gradient of the marginal likelihood\n of the state-space model. If true then \"grad_calc_params\" parameter must\n provide the extra parameters for gradient calculation.\n\n grad_params_no: int\n If previous parameter is true, then this parameters gives the\n total number of parameters in the gradient.\n\n grad_calc_params: dictionary\n Dictionary with derivatives of model matrices with respect\n to parameters \"dF\", \"dL\", \"dQc\", \"dH\", \"dR\", \"dm_init\", \"dP_init\".\n They can be None, in this case zero matrices (no dependence on parameters)\n is assumed. If there is only one parameter then third dimension is\n automatically added.\n\n Output:\n --------------\n\n M: (no_steps+1,state_dim) matrix or (no_steps+1,state_dim, time_series_no) 3D array\n Filter estimates of the state means. In the extra step the initial\n value is included. In the \"multiple time series mode\" third dimension\n correspond to different timeseries.\n\n P: (no_steps+1, state_dim, state_dim) 3D array\n Filter estimates of the state covariances. In the extra step the initial\n value is included.\n\n log_likelihood: double or (1, time_series_no) 3D array.\n\n If the parameter calc_log_likelihood was set to true, return\n logarithm of marginal likelihood of the state-space model. If\n the parameter was false, return None. In the \"multiple time series mode\" it is a vector\n providing log_likelihood for each time series.\n\n grad_log_likelihood: column vector or (grad_params_no, time_series_no) matrix\n If calc_grad_log_likelihood is true, return gradient of log likelihood\n with respect to parameters. It returns it column wise, so in\n \"multiple time series mode\" gradients for each time series is in the\n corresponding column.\n\n AQcomp: object\n Contains some pre-computed values for converting continuos model into\n discrete one. It can be used later in the smoothing pahse.\n \"\"\"\n\n p_H = np.atleast_1d(p_H)\n p_R = np.atleast_1d(p_R)\n\n X.shape, old_X_shape = cls._reshape_input_data(X.shape, 2) # represent as column\n if (X.shape[1] != 1):\n raise ValueError(\"Only one dimensional X data is supported.\")\n\n Y.shape, old_Y_shape = cls._reshape_input_data(Y.shape) # represent as column\n\n state_dim = F.shape[0]\n measurement_dim = Y.shape[1]\n time_series_no = Y.shape[2] # multiple time series mode\n\n if ((len(p_H.shape) == 3) and (len(p_H.shape[2]) != 1)) or\\\n ((len(p_R.shape) == 3) and (len(p_R.shape[2]) != 1)):\n model_matrices_chage_with_time = True\n else:\n model_matrices_chage_with_time = False\n\n # Check index\n old_index_shape = None\n if index is None:\n if (len(p_H.shape) == 3) or (len(p_R.shape) == 3):\n raise ValueError(\"Parameter index can not be None for time varying matrices (third dimension is present)\")\n else: # matrices do not change in time, so form dummy zero indices.\n index = np.zeros((1,Y.shape[0]))\n else:\n if len(index.shape) == 1:\n index.shape = (1,index.shape[0])\n old_index_shape = (index.shape[0],)\n\n if (index.shape[1] != Y.shape[0]):\n raise ValueError(\"Number of measurements must be equal the number of H_{k}, R_{k}\")\n\n if (index.shape[0] == 1):\n H_time_var_index = 0; R_time_var_index = 0\n elif (index.shape[0] == 4):\n H_time_var_index = 0; R_time_var_index = 1\n else:\n raise ValueError(\"First Dimension of index must be either 1 or 2.\")\n\n (p_H, old_H_shape) = cls._check_SS_matrix(p_H, state_dim, measurement_dim, which='H')\n (p_R, old_R_shape) = cls._check_SS_matrix(p_R, state_dim, measurement_dim, which='R')\n\n if m_init is None:\n m_init = np.zeros((state_dim, time_series_no))\n else:\n m_init = np.atleast_2d(m_init).T\n\n if P_init is None:\n P_init = P_inf.copy()\n\n if p_kalman_filter_type not in ('regular', 'svd'):\n raise ValueError(\"Kalman filer type neither 'regular nor 'svd'.\")\n\n # Functions to pass to the kalman_filter algorithm:\n # Parameters:\n # k - number of Kalman filter iteration\n # m - vector for calculating matrices. Required for EKF. Not used here.\n # f_hl = lambda k,m,H: np.dot(H, m)\n # f_H = lambda k,m,P: p_H[:,:, index[H_time_var_index, k]]\n #f_R = lambda k: p_R[:,:, index[R_time_var_index, k]]\n #o_R = R_handling( p_R, index, R_time_var_index, 20)\n\n if calc_grad_log_likelihood:\n\n dF = cls._check_grad_state_matrices(grad_calc_params.get('dF'), state_dim, grad_params_no, which = 'dA')\n dQc = cls._check_grad_state_matrices(grad_calc_params.get('dQc'), state_dim, grad_params_no, which = 'dQ')\n dP_inf = cls._check_grad_state_matrices(grad_calc_params.get('dP_inf'), state_dim, grad_params_no, which = 'dA')\n\n dH = cls._check_grad_measurement_matrices(grad_calc_params.get('dH'), state_dim, grad_params_no, measurement_dim, which = 'dH')\n dR = cls._check_grad_measurement_matrices(grad_calc_params.get('dR'), state_dim, grad_params_no, measurement_dim, which = 'dR')\n\n dm_init = grad_calc_params.get('dm_init') # Initial values for the Kalman Filter\n if dm_init is None:\n # multiple time series mode. Keep grad_params always as a last dimension\n dm_init = np.zeros( (state_dim, time_series_no, grad_params_no) )\n\n dP_init = grad_calc_params.get('dP_init') # Initial values for the Kalman Filter\n if dP_init is None:\n dP_init = dP_inf(0).copy() # get the dP_init matrix, because now it is a function\n\n else:\n dP_inf = None\n dF = None\n dQc = None\n dH = None\n dR = None\n dm_init = None\n dP_init = None\n\n measurement_callables = Std_Measurement_Callables_Class(p_H, H_time_var_index, p_R, index, R_time_var_index, 20, dH, dR)\n #import pdb; pdb.set_trace()\n\n dynamic_callables = cls._cont_to_discrete_object(X, F, L, Qc, compute_derivatives=calc_grad_log_likelihood,\n grad_params_no=grad_params_no,\n P_inf=P_inf, dP_inf=dP_inf, dF = dF, dQc=dQc)\n\n if print_verbose:\n print(\"General: run Continuos-Discrete Kalman Filter\")\n # Also for dH, dR and probably for all derivatives\n (M, P, log_likelihood, grad_log_likelihood, AQcomp) = cls._cont_discr_kalman_filter_raw(state_dim,\n dynamic_callables, measurement_callables,\n X, Y, m_init=m_init, P_init=P_init,\n p_kalman_filter_type=p_kalman_filter_type,\n calc_log_likelihood=calc_log_likelihood,\n calc_grad_log_likelihood=calc_grad_log_likelihood, grad_params_no=grad_params_no,\n dm_init=dm_init, dP_init=dP_init)\n\n if old_index_shape is not None:\n index.shape = old_index_shape\n\n if old_X_shape is not None:\n X.shape = old_X_shape\n\n if old_Y_shape is not None:\n Y.shape = old_Y_shape\n\n if old_H_shape is not None:\n p_H.shape = old_H_shape\n\n if old_R_shape is not None:\n p_R.shape = old_R_shape\n\n return (M, P, log_likelihood, grad_log_likelihood, AQcomp)", "response": "This function returns the estimated filter distributions x_k L x_k R_k and y_k in the state - space model."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _cont_to_discrete_object(cls, X, F, L, Qc, compute_derivatives=False,\n grad_params_no=None,\n P_inf=None, dP_inf=None, dF = None, dQc=None,\n dt0=None):\n \"\"\"\n Function return the object which is used in Kalman filter and/or\n smoother to obtain matrices A, Q and their derivatives for discrete model\n from the continuous model.\n\n There are 2 objects AQcompute_once and AQcompute_batch and the function\n returs the appropriate one based on the number of different time steps.\n\n Input:\n ----------------------\n X, F, L, Qc: matrices\n Continuous model matrices\n\n f_a: function\n Dynamic Function is attached to the Dynamic_Model_Callables class\n compute_derivatives: boolean\n Whether to compute derivatives\n\n grad_params_no: int\n Number of parameters in the gradient\n\n P_inf, dP_inf, dF, dQ: matrices and 3D objects\n Data necessary to compute derivatives.\n\n Output:\n --------------------------\n AQcomp: object\n Its methods return matrices (and optionally derivatives) for the\n discrete state-space model.\n\n \"\"\"\n\n unique_round_decimals = 10\n threshold_number_of_unique_time_steps = 20 # above which matrices are separately each time\n dt = np.empty((X.shape[0],))\n dt[1:] = np.diff(X[:,0],axis=0)\n if dt0 is None:\n dt[0] = 0#dt[1]\n else:\n if isinstance(dt0,str):\n dt = dt[1:]\n else:\n dt[0] = dt0\n \n unique_indices = np.unique(np.round(dt, decimals=unique_round_decimals))\n number_unique_indices = len(unique_indices)\n\n #import pdb; pdb.set_trace()\n if use_cython:\n class AQcompute_batch(state_space_cython.AQcompute_batch_Cython):\n def __init__(self, F,L,Qc,dt,compute_derivatives=False, grad_params_no=None, P_inf=None, dP_inf=None, dF = None, dQc=None):\n As, Qs, reconstruct_indices, dAs, dQs = ContDescrStateSpace.lti_sde_to_descrete(F,\n L,Qc,dt,compute_derivatives,\n grad_params_no=grad_params_no, P_inf=P_inf, dP_inf=dP_inf, dF=dF, dQc=dQc)\n\n super(AQcompute_batch,self).__init__(As, Qs, reconstruct_indices, dAs,dQs)\n else:\n AQcompute_batch = cls.AQcompute_batch_Python\n\n if number_unique_indices > threshold_number_of_unique_time_steps:\n AQcomp = cls.AQcompute_once(F,L,Qc, dt,compute_derivatives=compute_derivatives,\n grad_params_no=grad_params_no, P_inf=P_inf, dP_inf=dP_inf, dF=dF, dQc=dQc)\n if print_verbose:\n print(\"CDO: Continue-to-discrete INSTANTANEOUS object is created.\")\n print(\"CDO: Number of different time steps: %i\" % (number_unique_indices,) )\n\n else:\n AQcomp = AQcompute_batch(F,L,Qc,dt,compute_derivatives=compute_derivatives,\n grad_params_no=grad_params_no, P_inf=P_inf, dP_inf=dP_inf, dF=dF, dQc=dQc)\n if print_verbose:\n print(\"CDO: Continue-to-discrete BATCH object is created.\")\n print(\"CDO: Number of different time steps: %i\" % (number_unique_indices,) )\n print(\"CDO: Total size if its data: %i\" % (AQcomp.total_size_of_data,) )\n\n return AQcomp", "response": "This function returns the object which is used in the discrete model."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncomputing the network preactivations for the given data set.", "response": "def _f_preactivations(self, X):\n \"\"\"Computes the network preactivations, i.e. the results of all intermediate linear layers before applying the\n activation function on them\n :param X: input data\n :return: list of preactivations [X, XW+b, f(XW+b)W+b, ...]\n \"\"\"\n\n preactivations_list = list()\n net = X\n preactivations_list.append(X)\n\n for W, b, i in zip(self.W_list, self.b_list, np.arange(len(self.W_list))):\n net = np.dot(net, W)\n net = net + b\n if i < len(self.W_list) - 1:\n preactivations_list.append(net)\n net = self.act(net)\n return preactivations_list"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef numpy_invalid_op_as_exception(func):\n def func_wrapper(*args, **kwargs):\n np.seterr(invalid='raise')\n result = func(*args, **kwargs)\n np.seterr(invalid='warn')\n return result\n return func_wrapper", "response": "A decorator that allows catching numpy invalid operations as exceptions."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef dkron(A,dA,B,dB, operation='prod'):\n\n if dA is None:\n dA_param_num = 0\n dA = np.zeros((A.shape[0], A.shape[1],1))\n else:\n dA_param_num = dA.shape[2]\n\n if dB is None:\n dB_param_num = 0\n dB = np.zeros((B.shape[0], B.shape[1],1))\n else:\n dB_param_num = dB.shape[2]\n\n # Space allocation for derivative matrix\n dC = np.zeros((A.shape[0]*B.shape[0], A.shape[1]*B.shape[1], dA_param_num + dB_param_num))\n\n for k in range(dA_param_num):\n if operation == 'prod':\n dC[:,:,k] = np.kron(dA[:,:,k],B);\n else:\n dC[:,:,k] = np.kron(dA[:,:,k],np.eye( B.shape[0] ))\n\n for k in range(dB_param_num):\n if operation == 'prod':\n dC[:,:,dA_param_num+k] = np.kron(A,dB[:,:,k])\n else:\n dC[:,:,dA_param_num+k] = np.kron(np.eye( A.shape[0] ),dB[:,:,k])\n\n return dC", "response": "Function computes the derivative of the derivative of two sets of elements of A and B."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef to_dict(self):\n\n input_dict = super(Prod, self)._save_to_input_dict()\n input_dict[\"class\"] = str(\"GPy.kern.Prod\")\n return input_dict", "response": "Convert the object into a json serializable dictionary."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef sde_update_gradient_full(self, gradients):\n part_start_param_index = 0\n for p in self.parts:\n if not p.is_fixed:\n part_param_num = len(p.param_array) # number of parameters in the part\n p.sde_update_gradient_full(gradients[part_start_param_index:(part_start_param_index+part_param_num)])\n part_start_param_index += part_param_num", "response": "Update gradient in the order in which parameters are represented in the kernel\n "} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef to_dict(self):\n\n input_dict = super(StdPeriodic, self)._save_to_input_dict()\n input_dict[\"class\"] = \"GPy.kern.StdPeriodic\"\n input_dict[\"variance\"] = self.variance.values.tolist()\n input_dict[\"period\"] = self.period.values.tolist()\n input_dict[\"lengthscale\"] = self.lengthscale.values.tolist()\n input_dict[\"ARD1\"] = self.ARD1\n input_dict[\"ARD2\"] = self.ARD2\n return input_dict", "response": "Convert the object into a json serializable dictionary."} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncomputing the covariance matrix between X and X2.", "response": "def K(self, X, X2=None):\n \"\"\"Compute the covariance matrix between X and X2.\"\"\"\n if X2 is None:\n X2 = X\n\n base = np.pi * (X[:, None, :] - X2[None, :, :]) / self.period\n exp_dist = np.exp( -0.5* np.sum( np.square( np.sin( base ) / self.lengthscale ), axis = -1 ) )\n\n return self.variance * exp_dist"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncompute the diagonal of the covariance matrix associated to X.", "response": "def Kdiag(self, X):\n \"\"\"Compute the diagonal of the covariance matrix associated to X.\"\"\"\n ret = np.empty(X.shape[0])\n ret[:] = self.variance\n return ret"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef update_gradients_full(self, dL_dK, X, X2=None):\n if X2 is None:\n X2 = X\n\n base = np.pi * (X[:, None, :] - X2[None, :, :]) / self.period\n\n sin_base = np.sin( base )\n exp_dist = np.exp( -0.5* np.sum( np.square( sin_base / self.lengthscale ), axis = -1 ) )\n\n dwl = self.variance * (1.0/np.square(self.lengthscale)) * sin_base*np.cos(base) * (base / self.period)\n\n dl = self.variance * np.square( sin_base) / np.power( self.lengthscale, 3)\n\n self.variance.gradient = np.sum(exp_dist * dL_dK)\n #target[0] += np.sum( exp_dist * dL_dK)\n\n if self.ARD1: # different periods\n self.period.gradient = (dwl * exp_dist[:,:,None] * dL_dK[:, :, None]).sum(0).sum(0)\n else: # same period\n self.period.gradient = np.sum(dwl.sum(-1) * exp_dist * dL_dK)\n\n if self.ARD2: # different lengthscales\n self.lengthscale.gradient = (dl * exp_dist[:,:,None] * dL_dK[:, :, None]).sum(0).sum(0)\n else: # same lengthscales\n self.lengthscale.gradient = np.sum(dl.sum(-1) * exp_dist * dL_dK)", "response": "update the gradients of the covariance matrix with respect to the parameters."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef update_gradients_diag(self, dL_dKdiag, X):\n self.variance.gradient = np.sum(dL_dKdiag)\n self.period.gradient = 0\n self.lengthscale.gradient = 0", "response": "update the gradients of the log likelihood matrix with respect to the parameters"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef plot(shape_records,facecolor='w',edgecolor='k',linewidths=.5, ax=None,xlims=None,ylims=None):\n #Axes handle\n if ax is None:\n fig = pb.figure()\n ax = fig.add_subplot(111)\n\n #Iterate over shape_records\n for srec in shape_records:\n points = np.vstack(srec.shape.points)\n sparts = srec.shape.parts\n par = list(sparts) + [points.shape[0]]\n\n polygs = []\n for pj in range(len(sparts)):\n polygs.append(Polygon(points[par[pj]:par[pj+1]]))\n ax.add_collection(PatchCollection(polygs,facecolor=facecolor,edgecolor=edgecolor, linewidths=linewidths))\n\n #Plot limits\n _box = np.vstack([srec.shape.bbox for srec in shape_records])\n minx,miny = np.min(_box[:,:2],0)\n maxx,maxy = np.max(_box[:,2:],0)\n\n if xlims is not None:\n minx,maxx = xlims\n if ylims is not None:\n miny,maxy = ylims\n ax.set_xlim(minx,maxx)\n ax.set_ylim(miny,maxy)", "response": "Plot the geometry of a shapefile and attributes list."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn the geometry and attributes of a shapefile whose fields match a regular expression given by regex.", "response": "def string_match(sf,regex,field=2):\n \"\"\"\n Return the geometry and attributes of a shapefile whose fields match a regular expression given\n\n :param sf: shapefile\n :type sf: shapefile object\n :regex: regular expression to match\n :type regex: string\n :field: field number to be matched with the regex\n :type field: integer\n \"\"\"\n index = []\n shape_records = []\n for rec in enumerate(sf.shapeRecords()):\n m = re.search(regex,rec[1].record[field])\n if m is not None:\n index.append(rec[0])\n shape_records.append(rec[1])\n return index,shape_records"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns the geometry and attributes of a shapefile that lie within a bounding box.", "response": "def bbox_match(sf,bbox,inside_only=True):\n \"\"\"\n Return the geometry and attributes of a shapefile that lie within (or intersect) a bounding box\n\n :param sf: shapefile\n :type sf: shapefile object\n :param bbox: bounding box\n :type bbox: list of floats [x_min,y_min,x_max,y_max]\n :inside_only: True if the objects returned are those that lie within the bbox and False if the objects returned are any that intersect the bbox\n :type inside_only: Boolean\n \"\"\"\n A,B,C,D = bbox\n index = []\n shape_records = []\n for rec in enumerate(sf.shapeRecords()):\n a,b,c,d = rec[1].shape.bbox\n if inside_only:\n if A <= a and B <= b and C >= c and D >= d:\n index.append(rec[0])\n shape_records.append(rec[1])\n else:\n cond1 = A <= a and B <= b and C >= a and D >= b\n cond2 = A <= c and B <= d and C >= c and D >= d\n cond3 = A <= a and D >= d and C >= a and B <= d\n cond4 = A <= c and D >= b and C >= c and B <= b\n cond5 = a <= C and b <= B and d >= D\n cond6 = c <= A and b <= B and d >= D\n cond7 = d <= B and a <= A and c >= C\n cond8 = b <= D and a <= A and c >= C\n if cond1 or cond2 or cond3 or cond4 or cond5 or cond6 or cond7 or cond8:\n index.append(rec[0])\n shape_records.append(rec[1])\n return index,shape_records"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef plot_bbox(sf,bbox,inside_only=True):\n index,shape_records = bbox_match(sf,bbox,inside_only)\n A,B,C,D = bbox\n plot(shape_records,xlims=[bbox[0],bbox[2]],ylims=[bbox[1],bbox[3]])", "response": "Plot the geometry of a shapefile within a bounding box."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nplot the geometry of a shapefile whose fields match a regular expression given by regex.", "response": "def plot_string_match(sf,regex,field,**kwargs):\n \"\"\"\n Plot the geometry of a shapefile whose fields match a regular expression given\n\n :param sf: shapefile\n :type sf: shapefile object\n :regex: regular expression to match\n :type regex: string\n :field: field number to be matched with the regex\n :type field: integer\n \"\"\"\n index,shape_records = string_match(sf,regex,field)\n plot(shape_records,**kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef apply_bbox(sf,ax):\n limits = sf.bbox\n xlim = limits[0],limits[2]\n ylim = limits[1],limits[3]\n ax.set_xlim(xlim)\n ax.set_ylim(ylim)", "response": "Apply bounding box to a single figure."} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nupdating the warping function gradients if the posterior parameters have changed.", "response": "def parameters_changed(self):\n \"\"\"\n Notice that we update the warping function gradients here.\n \"\"\"\n self.Y_normalized[:] = self.transform_data()\n super(WarpedGP, self).parameters_changed()\n Kiy = self.posterior.woodbury_vector.flatten()\n self.warping_function.update_grads(self.Y_untransformed, Kiy)"} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nlogs likelihood of the warping function.", "response": "def log_likelihood(self):\n \"\"\"\n Notice we add the jacobian of the warping function here.\n \"\"\"\n ll = GP.log_likelihood(self)\n jacobian = self.warping_function.fgrad_y(self.Y_untransformed)\n return ll + np.log(jacobian).sum()"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _get_warped_mean(self, mean, std, pred_init=None, deg_gauss_hermite=20):\n gh_samples, gh_weights = np.polynomial.hermite.hermgauss(deg_gauss_hermite)\n gh_samples = gh_samples[:, None]\n gh_weights = gh_weights[None, :]\n return gh_weights.dot(self._get_warped_term(mean, std, gh_samples)) / np.sqrt(np.pi)", "response": "Calculate the warped mean by using Gauss - Hermite quadrature."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _get_warped_variance(self, mean, std, pred_init=None, deg_gauss_hermite=20):\n gh_samples, gh_weights = np.polynomial.hermite.hermgauss(deg_gauss_hermite)\n gh_samples = gh_samples[:, None]\n gh_weights = gh_weights[None, :]\n arg1 = gh_weights.dot(self._get_warped_term(mean, std, gh_samples, \n pred_init=pred_init) ** 2) / np.sqrt(np.pi)\n arg2 = self._get_warped_mean(mean, std, pred_init=pred_init,\n deg_gauss_hermite=deg_gauss_hermite)\n return arg1 - (arg2 ** 2)", "response": "Calculate the warped variance by using Gauss - Hermite quadrature."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\npredict the new values of the current object.", "response": "def predict(self, Xnew, kern=None, pred_init=None, Y_metadata=None,\n median=False, deg_gauss_hermite=20, likelihood=None):\n \"\"\"\n Prediction results depend on:\n - The value of the self.predict_in_warped_space flag\n - The median flag passed as argument\n The likelihood keyword is never used, it is just to follow the plotting API.\n \"\"\"\n #mu, var = GP._raw_predict(self, Xnew)\n # now push through likelihood\n #mean, var = self.likelihood.predictive_values(mu, var)\n \n mean, var = super(WarpedGP, self).predict(Xnew, kern=kern, full_cov=False, likelihood=likelihood)\n\n\n if self.predict_in_warped_space:\n std = np.sqrt(var)\n if median:\n wmean = self.warping_function.f_inv(mean, y=pred_init)\n else:\n wmean = self._get_warped_mean(mean, std, pred_init=pred_init,\n deg_gauss_hermite=deg_gauss_hermite).T\n wvar = self._get_warped_variance(mean, std, pred_init=pred_init,\n deg_gauss_hermite=deg_gauss_hermite).T\n else:\n wmean = mean\n wvar = var\n return wmean, wvar"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef predict_quantiles(self, X, quantiles=(2.5, 97.5), Y_metadata=None, likelihood=None, kern=None):\n qs = super(WarpedGP, self).predict_quantiles(X, quantiles, Y_metadata=Y_metadata, likelihood=likelihood, kern=kern)\n if self.predict_in_warped_space:\n return [self.warping_function.f_inv(q) for q in qs]\n return qs", "response": "Predict the quantiles around the prediction at X."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef log_predictive_density(self, x_test, y_test, Y_metadata=None):\n mu_star, var_star = self._raw_predict(x_test)\n fy = self.warping_function.f(y_test)\n ll_lpd = self.likelihood.log_predictive_density(fy, mu_star, var_star, Y_metadata=Y_metadata)\n return ll_lpd + np.log(self.warping_function.fgrad_y(y_test))", "response": "Calculates the log predictive density of the warping function at the given locations and observations."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef sde_update_gradient_full(self, gradients):\n \n self.variance.gradient = gradients[0]\n self.lengthscale.gradient = gradients[1]", "response": "Update the gradient of the kernel with the given gradients."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning the state space representation of the covariance.", "response": "def sde(self): \n \"\"\" \n Return the state space representation of the covariance. \n \"\"\" \n \n variance = float(self.variance.values)\n lengthscale = float(self.lengthscale.values)\n\n lamda = np.sqrt(5.0)/lengthscale\n kappa = 5.0/3.0*variance/lengthscale**2 \n \n F = np.array(((0, 1,0), (0, 0, 1), (-lamda**3, -3.0*lamda**2, -3*lamda)))\n L = np.array(((0,),(0,),(1,)))\n Qc = np.array((((variance*400.0*np.sqrt(5.0)/3.0/lengthscale**5),),))\n H = np.array(((1,0,0),)) \n \n Pinf = np.array(((variance,0,-kappa), (0, kappa, 0), (-kappa, 0, 25.0*variance/lengthscale**4)))\n P0 = Pinf.copy()\n # Allocate space for the derivatives \n dF = np.empty((3,3,2)) \n dQc = np.empty((1,1,2)) \n dPinf = np.empty((3,3,2))\n \n # The partial derivatives \n dFvariance = np.zeros((3,3))\n dFlengthscale = np.array(((0,0,0),(0,0,0),(15.0*np.sqrt(5.0)/lengthscale**4, \n 30.0/lengthscale**3, 3*np.sqrt(5.0)/lengthscale**2)))\n dQcvariance = np.array((((400*np.sqrt(5)/3/lengthscale**5,),)))\n dQclengthscale = np.array((((-variance*2000*np.sqrt(5)/3/lengthscale**6,),))) \n \n dPinf_variance = Pinf/variance\n kappa2 = -2.0*kappa/lengthscale\n dPinf_lengthscale = np.array(((0,0,-kappa2),(0,kappa2,0),(-kappa2, \n 0,-100*variance/lengthscale**5))) \n # Combine the derivatives \n dF[:,:,0] = dFvariance\n dF[:,:,1] = dFlengthscale \n dQc[:,:,0] = dQcvariance \n dQc[:,:,1] = dQclengthscale \n dPinf[:,:,0] = dPinf_variance\n dPinf[:,:,1] = dPinf_lengthscale\n dP0 = dPinf.copy()\n \n return (F, L, Qc, H, Pinf, P0, dF, dQc, dPinf, dP0)"} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nsamples the unfixed model parameters.", "response": "def sample(self, num_samples=1000, hmc_iters=20):\n \"\"\"\n Sample the (unfixed) model parameters.\n \n :param num_samples: the number of samples to draw (1000 by default)\n :type num_samples: int\n :param hmc_iters: the number of leap-frog iterations (20 by default)\n :type hmc_iters: int\n :return: the list of parameters samples with the size N x P (N - the number of samples, P - the number of parameters to sample) \n :rtype: numpy.ndarray\n \"\"\"\n params = np.empty((num_samples,self.p.size))\n for i in range(num_samples):\n self.p[:] = np.random.multivariate_normal(np.zeros(self.p.size),self.M)\n H_old = self._computeH()\n theta_old = self.model.optimizer_array.copy()\n params[i] = self.model.unfixed_param_array\n #Matropolis\n self._update(hmc_iters)\n H_new = self._computeH()\n\n if H_old>H_new:\n k = 1.\n else:\n k = np.exp(H_old-H_new)\n if np.random.rand()0 , kuyp(np.abs(rdist[ss1,ss2])), kuyn(np.abs(rdist[ss1,ss2]) ) )\n #K[ss1,ss2]= np.where( rdist[ss1,ss2]>0 , kuyp(rdist[ss1,ss2]), kuyn(rdist[ss1,ss2] ) )\n elif i==1 and j==1:\n K[ss1,ss2] = ( b**2*k1(tdist[ss1,ss2]) - 2*a*c*k2(xdist[ss1,ss2]) + a**2*k3(xdist[ss1,ss2]) + c**2 )* vyt*vyx* kyy(tdist[ss1,ss2],xdist[ss1,ss2])\n else:\n K[ss1,ss2] = (-a*k2(xdist[ss1,ss2]) - b*k4(ttdist[ss1,ss2]) + c)*vyt*vyx*kyy(tdist[ss1,ss2],xdist[ss1,ss2])\n #K[ss1,ss2]= np.where( rdist[ss1,ss2]>0 , kyup(np.abs(rdist[ss1,ss2])), kyun(np.abs(rdist[ss1,ss2]) ) )\n #K[ss1,ss2] = np.where( rdist[ss1,ss2]>0 , kyup(rdist[ss1,ss2]), kyun(rdist[ss1,ss2] ) )\n \n #stop\n return K"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ncomputes the diagonal of the covariance matrix associated to X.", "response": "def Kdiag(self, X):\n \"\"\"Compute the diagonal of the covariance matrix associated to X.\"\"\"\n vyt = self.variance_Yt\n vyx = self.variance_Yx\n\n lyt = 1./(2*self.lengthscale_Yt)\n lyx = 1./(2*self.lengthscale_Yx)\n\n a = self.a\n b = self.b\n c = self.c\n\n ## dk^2/dtdt'\n k1 = (2*lyt )*vyt*vyx\n ## dk^2/dx^2\n k2 = ( - 2*lyx )*vyt*vyx\n ## dk^4/dx^2dx'^2\n k3 = ( 4*3*lyx**2 )*vyt*vyx\n\n\n Kdiag = np.zeros(X.shape[0])\n slices = index_to_slices(X[:,-1])\n\n for i, ss1 in enumerate(slices):\n for s1 in ss1:\n if i==0:\n Kdiag[s1]+= vyt*vyx\n elif i==1:\n #i=1\n Kdiag[s1]+= b**2*k1 - 2*a*c*k2 + a**2*k3 + c**2*vyt*vyx\n #Kdiag[s1]+= Vu*Vy*(k1+k2+k3)\n else:\n raise ValueError(\"invalid input/output index\")\n\n return Kdiag"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef update_gradients_full(self, dL_dK, X, X2=None):\n #def dK_dtheta(self, dL_dK, X, X2, target):\n \"\"\"derivative of the covariance matrix with respect to the parameters.\"\"\"\n X,slices = X[:,:-1],index_to_slices(X[:,-1])\n if X2 is None:\n X2,slices2 = X,slices\n K = np.zeros((X.shape[0], X.shape[0]))\n else:\n X2,slices2 = X2[:,:-1],index_to_slices(X2[:,-1])\n \n vyt = self.variance_Yt\n vyx = self.variance_Yx\n\n lyt = 1./(2*self.lengthscale_Yt)\n lyx = 1./(2*self.lengthscale_Yx)\n\n a = self.a\n b = self.b\n c = self.c\n\n tdist = (X[:,0][:,None] - X2[:,0][None,:])**2\n xdist = (X[:,1][:,None] - X2[:,1][None,:])**2\n #rdist = [tdist,xdist]\n ttdist = (X[:,0][:,None] - X2[:,0][None,:])\n \n rd=tdist.shape[0]\n\n dka = np.zeros([rd,rd])\n dkb = np.zeros([rd,rd])\n dkc = np.zeros([rd,rd])\n dkYdvart = np.zeros([rd,rd])\n dkYdvarx = np.zeros([rd,rd])\n dkYdlent = np.zeros([rd,rd])\n dkYdlenx = np.zeros([rd,rd])\n\n\n kyy = lambda tdist,xdist: np.exp(-lyt*(tdist) -lyx*(xdist))\n #k1 = lambda tdist: (lyt - lyt**2 * (tdist) )\n #k2 = lambda xdist: ( lyx**2 * (xdist) - lyx )\n #k3 = lambda xdist: ( 3*lyx**2 - 6*xdist*lyx**3 + xdist**2*lyx**4 )\n #k4 = lambda tdist: -lyt*np.sqrt(tdist)\n\n k1 = lambda tdist: (2*lyt - 4*lyt**2 * (tdist) )\n\n k2 = lambda xdist: ( 4*lyx**2 * (xdist) - 2*lyx )\n\n k3 = lambda xdist: ( 3*4*lyx**2 - 6*8*xdist*lyx**3 + 16*xdist**2*lyx**4 )\n\n k4 = lambda ttdist: 2*lyt*(ttdist)\n\n dkyydlyx = lambda tdist,xdist: kyy(tdist,xdist)*(-xdist)\n dkyydlyt = lambda tdist,xdist: kyy(tdist,xdist)*(-tdist)\n\n dk1dlyt = lambda tdist: 2. - 4*2.*lyt*tdist\n dk2dlyx = lambda xdist: (4.*2.*lyx*xdist -2.)\n dk3dlyx = lambda xdist: (6.*4.*lyx - 18.*8*xdist*lyx**2 + 4*16*xdist**2*lyx**3)\n\n dk4dlyt = lambda ttdist: 2*(ttdist)\n\n for i, s1 in enumerate(slices):\n for j, s2 in enumerate(slices2):\n for ss1 in s1:\n for ss2 in s2:\n if i==0 and j==0:\n dka[ss1,ss2] = 0\n dkb[ss1,ss2] = 0\n dkc[ss1,ss2] = 0\n dkYdvart[ss1,ss2] = vyx*kyy(tdist[ss1,ss2],xdist[ss1,ss2])\n dkYdvarx[ss1,ss2] = vyt*kyy(tdist[ss1,ss2],xdist[ss1,ss2])\n dkYdlenx[ss1,ss2] = vyt*vyx*dkyydlyx(tdist[ss1,ss2],xdist[ss1,ss2])\n dkYdlent[ss1,ss2] = vyt*vyx*dkyydlyt(tdist[ss1,ss2],xdist[ss1,ss2])\n elif i==0 and j==1:\n dka[ss1,ss2] = -k2(xdist[ss1,ss2])*vyt*vyx*kyy(tdist[ss1,ss2],xdist[ss1,ss2])\n dkb[ss1,ss2] = k4(ttdist[ss1,ss2])*vyt*vyx*kyy(tdist[ss1,ss2],xdist[ss1,ss2])\n dkc[ss1,ss2] = vyt*vyx*kyy(tdist[ss1,ss2],xdist[ss1,ss2])\n #dkYdvart[ss1,ss2] = 0\n #dkYdvarx[ss1,ss2] = 0\n #dkYdlent[ss1,ss2] = 0\n #dkYdlenx[ss1,ss2] = 0\n dkYdvart[ss1,ss2] = (-a*k2(xdist[ss1,ss2])+b*k4(ttdist[ss1,ss2])+c)*vyx*kyy(tdist[ss1,ss2],xdist[ss1,ss2])\n dkYdvarx[ss1,ss2] = (-a*k2(xdist[ss1,ss2])+b*k4(ttdist[ss1,ss2])+c)*vyt*kyy(tdist[ss1,ss2],xdist[ss1,ss2])\n dkYdlent[ss1,ss2] = vyt*vyx*dkyydlyt(tdist[ss1,ss2],xdist[ss1,ss2])* (-a*k2(xdist[ss1,ss2])+b*k4(ttdist[ss1,ss2])+c)+\\\n vyt*vyx*kyy(tdist[ss1,ss2],xdist[ss1,ss2])*b*dk4dlyt(ttdist[ss1,ss2])\n dkYdlenx[ss1,ss2] = vyt*vyx*dkyydlyx(tdist[ss1,ss2],xdist[ss1,ss2])*(-a*k2(xdist[ss1,ss2])+b*k4(ttdist[ss1,ss2])+c)+\\\n vyt*vyx*kyy(tdist[ss1,ss2],xdist[ss1,ss2])*(-a*dk2dlyx(xdist[ss1,ss2]))\n elif i==1 and j==1:\n dka[ss1,ss2] = (2*a*k3(xdist[ss1,ss2]) - 2*c*k2(xdist[ss1,ss2]))*vyt*vyx* kyy(tdist[ss1,ss2],xdist[ss1,ss2])\n dkb[ss1,ss2] = 2*b*k1(tdist[ss1,ss2])*vyt*vyx* kyy(tdist[ss1,ss2],xdist[ss1,ss2])\n dkc[ss1,ss2] = (-2*a*k2(xdist[ss1,ss2]) + 2*c )*vyt*vyx* kyy(tdist[ss1,ss2],xdist[ss1,ss2])\n dkYdvart[ss1,ss2] = ( b**2*k1(tdist[ss1,ss2]) - 2*a*c*k2(xdist[ss1,ss2]) + a**2*k3(xdist[ss1,ss2]) + c**2 )*vyx* kyy(tdist[ss1,ss2],xdist[ss1,ss2])\n dkYdvarx[ss1,ss2] = ( b**2*k1(tdist[ss1,ss2]) - 2*a*c*k2(xdist[ss1,ss2]) + a**2*k3(xdist[ss1,ss2]) + c**2 )*vyt* kyy(tdist[ss1,ss2],xdist[ss1,ss2])\n dkYdlent[ss1,ss2] = vyt*vyx*dkyydlyt(tdist[ss1,ss2],xdist[ss1,ss2])*( b**2*k1(tdist[ss1,ss2]) - 2*a*c*k2(xdist[ss1,ss2]) + a**2*k3(xdist[ss1,ss2]) + c**2 ) +\\\n vyx*vyt*kyy(tdist[ss1,ss2],xdist[ss1,ss2])*b**2*dk1dlyt(tdist[ss1,ss2])\n dkYdlenx[ss1,ss2] = vyt*vyx*dkyydlyx(tdist[ss1,ss2],xdist[ss1,ss2])*( b**2*k1(tdist[ss1,ss2]) - 2*a*c*k2(xdist[ss1,ss2]) + a**2*k3(xdist[ss1,ss2]) + c**2 ) +\\\n vyx*vyt*kyy(tdist[ss1,ss2],xdist[ss1,ss2])* (-2*a*c*dk2dlyx(xdist[ss1,ss2]) + a**2*dk3dlyx(xdist[ss1,ss2]) )\n else:\n dka[ss1,ss2] = -k2(xdist[ss1,ss2])*vyt*vyx*kyy(tdist[ss1,ss2],xdist[ss1,ss2])\n dkb[ss1,ss2] = -k4(ttdist[ss1,ss2])*vyt*vyx*kyy(tdist[ss1,ss2],xdist[ss1,ss2])\n dkc[ss1,ss2] = vyt*vyx*kyy(tdist[ss1,ss2],xdist[ss1,ss2])\n #dkYdvart[ss1,ss2] = 0\n #dkYdvarx[ss1,ss2] = 0\n #dkYdlent[ss1,ss2] = 0\n #dkYdlenx[ss1,ss2] = 0\n dkYdvart[ss1,ss2] = (-a*k2(xdist[ss1,ss2])-b*k4(ttdist[ss1,ss2])+c)*vyx*kyy(tdist[ss1,ss2],xdist[ss1,ss2])\n dkYdvarx[ss1,ss2] = (-a*k2(xdist[ss1,ss2])-b*k4(ttdist[ss1,ss2])+c)*vyt*kyy(tdist[ss1,ss2],xdist[ss1,ss2])\n dkYdlent[ss1,ss2] = vyt*vyx*dkyydlyt(tdist[ss1,ss2],xdist[ss1,ss2])* (-a*k2(xdist[ss1,ss2])-b*k4(ttdist[ss1,ss2])+c)+\\\n vyt*vyx*kyy(tdist[ss1,ss2],xdist[ss1,ss2])*(-1)*b*dk4dlyt(ttdist[ss1,ss2])\n dkYdlenx[ss1,ss2] = vyt*vyx*dkyydlyx(tdist[ss1,ss2],xdist[ss1,ss2])*(-a*k2(xdist[ss1,ss2])-b*k4(ttdist[ss1,ss2])+c)+\\\n vyt*vyx*kyy(tdist[ss1,ss2],xdist[ss1,ss2])*(-a*dk2dlyx(xdist[ss1,ss2])) \n\n self.a.gradient = np.sum(dka * dL_dK) \n\n self.b.gradient = np.sum(dkb * dL_dK) \n\n self.c.gradient = np.sum(dkc * dL_dK)\n\n\n self.variance_Yt.gradient = np.sum(dkYdvart * dL_dK) # Vy\n\n self.variance_Yx.gradient = np.sum(dkYdvarx * dL_dK)\n\n self.lengthscale_Yt.gradient = np.sum(dkYdlent*(-0.5*self.lengthscale_Yt**(-2)) * dL_dK) #ly np.sum(dktheta2*(-self.lengthscale_Y**(-2)) * dL_dK) \n\n self.lengthscale_Yx.gradient = np.sum(dkYdlenx*(-0.5*self.lengthscale_Yx**(-2)) * dL_dK)", "response": "update the gradients of the kernel with respect to the parameters"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ngetting the gradients of the posterior distribution of X in its specific form.", "response": "def get_X_gradients(self, X):\n \"\"\"Get the gradients of the posterior distribution of X in its specific form.\"\"\"\n return X.mean.gradient, X.variance.gradient, X.binary_prob.gradient"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef sample_W(self, nSamples, raw_samples=False):\n assert isinstance(self.kern, kern.Linear)\n from ..util.linalg import pdinv\n N, D = self.Y.shape\n Q = self.X.shape[1]\n noise_var = self.likelihood.variance.values\n \n # Draw samples for X\n Xs = np.random.randn(*((nSamples,)+self.X.shape))*np.sqrt(self.X.variance.values)+self.X.mean.values\n b = np.random.rand(*((nSamples,)+self.X.shape))\n Xs[b>self.X.gamma.values] = 0\n \n invcov = (Xs[:,:,:,None]*Xs[:,:,None,:]).sum(1)/noise_var+np.eye(Q)\n cov = np.array([pdinv(invcov[s_idx])[0] for s_idx in range(invcov.shape[0])])\n Ws = np.empty((nSamples, Q, D))\n tmp = (np.transpose(Xs, (0,2,1)).reshape(nSamples*Q,N).dot(self.Y)).reshape(nSamples,Q,D)\n mean = (cov[:,:,:,None]*tmp[:,None,:,:]).sum(2)/noise_var\n zeros = np.zeros((Q,))\n for s_idx in range(Xs.shape[0]):\n Ws[s_idx] = (np.random.multivariate_normal(mean=zeros,cov=cov[s_idx],size=(D,))).T+mean[s_idx]\n \n if raw_samples:\n return Ws\n else:\n return Ws.mean(0), Ws.std(0)", "response": "Sample the load matrix for the current load set."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nconvert the object into a json serializable dictionary.", "response": "def to_dict(self):\n \"\"\"\n Convert the object into a json serializable dictionary.\n\n Note: It uses the private method _save_to_input_dict of the parent.\n\n :return dict: json serializable dictionary containing the needed information to instantiate the object\n \"\"\"\n\n #TODO: Implement a more memory efficient variant\n if self.L is None:\n return { \"mu\": self.mu.tolist(), \"Sigma\": self.Sigma.tolist()}\n else:\n return { \"mu\": self.mu.tolist(), \"Sigma\": self.Sigma.tolist(), \"L\": self.L.tolist()}"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef to_dict(self):\n\n input_dict = super(EP, self)._save_to_input_dict()\n input_dict[\"class\"] = \"GPy.inference.latent_function_inference.expectation_propagation.EP\"\n if self.ga_approx_old is not None:\n input_dict[\"ga_approx_old\"] = self.ga_approx_old.to_dict()\n if self._ep_approximation is not None:\n input_dict[\"_ep_approximation\"] = {}\n input_dict[\"_ep_approximation\"][\"post_params\"] = self._ep_approximation[0].to_dict()\n input_dict[\"_ep_approximation\"][\"ga_approx\"] = self._ep_approximation[1].to_dict()\n input_dict[\"_ep_approximation\"][\"cav_params\"] = self._ep_approximation[2].to_dict()\n input_dict[\"_ep_approximation\"][\"log_Z_tilde\"] = self._ep_approximation[3].tolist()\n\n return input_dict", "response": "Convert the object into a json serializable dictionary."} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nconverts the object into a json serializable dictionary.", "response": "def to_dict(self):\n \"\"\"\n Convert the object into a json serializable dictionary.\n\n Note: It uses the private method _save_to_input_dict of the parent.\n\n :return dict: json serializable dictionary containing the needed information to instantiate the object\n \"\"\"\n\n input_dict = super(EPDTC, self)._save_to_input_dict()\n input_dict[\"class\"] = \"GPy.inference.latent_function_inference.expectation_propagation.EPDTC\"\n if self.ga_approx_old is not None:\n input_dict[\"ga_approx_old\"] = self.ga_approx_old.to_dict()\n if self._ep_approximation is not None:\n input_dict[\"_ep_approximation\"] = {}\n input_dict[\"_ep_approximation\"][\"post_params\"] = self._ep_approximation[0].to_dict()\n input_dict[\"_ep_approximation\"][\"ga_approx\"] = self._ep_approximation[1].to_dict()\n input_dict[\"_ep_approximation\"][\"log_Z_tilde\"] = self._ep_approximation[2]\n\n return input_dict"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ncomputing the KL divergence to another NormalPosterior Object. This only holds if the two NormalPosterior objects have the same shape as the multivariate normal KL divergence.", "response": "def KL(self, other):\n \"\"\"Compute the KL divergence to another NormalPosterior Object. This only holds, if the two NormalPosterior objects have the same shape, as we do computational tricks for the multivariate normal KL divergence.\n \"\"\"\n return .5*(\n np.sum(self.variance/other.variance)\n + ((other.mean-self.mean)**2/other.variance).sum()\n - self.num_data * self.input_dim\n + np.sum(np.log(other.variance)) - np.sum(np.log(self.variance))\n )"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef plot(self, *args, **kwargs):\n import sys\n assert \"matplotlib\" in sys.modules, \"matplotlib package has not been imported.\"\n from ...plotting.matplot_dep import variational_plots\n return variational_plots.plot_SpikeSlab(self,*args, **kwargs)", "response": "Plot latent space X in 1D"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef chain_2(d2f_dg2, dg_dx, df_dg, d2g_dx2):\n if np.all(dg_dx==1.) and np.all(d2g_dx2 == 0):\n return d2f_dg2\n dg_dx_2 = np.clip(dg_dx, -np.inf, _lim_val_square)**2\n #dg_dx_2 = dg_dx**2\n return d2f_dg2*(dg_dx_2) + df_dg*d2g_dx2", "response": "Generic chaining function for second derivative of a single segment segment."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef chain_3(d3f_dg3, dg_dx, d2f_dg2, d2g_dx2, df_dg, d3g_dx3):\n if np.all(dg_dx==1.) and np.all(d2g_dx2==0) and np.all(d3g_dx3==0):\n return d3f_dg3\n dg_dx_3 = np.clip(dg_dx, -np.inf, _lim_val_cube)**3\n return d3f_dg3*(dg_dx_3) + 3*d2f_dg2*dg_dx*d2g_dx2 + df_dg*d3g_dx3", "response": "Chain the three components of a single segment."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef linear_grid(D, n = 100, min_max = (-100, 100)):\n\n g = np.linspace(min_max[0], min_max[1], n)\n G = np.ones((n, D))\n\n return G*g[:,None]", "response": "Creates a D - dimensional linearly spaced grid of n points."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef param_to_array(*param):\n import warnings\n warnings.warn(\"Please use param.values, as this function will be deprecated in the next release.\", DeprecationWarning)\n assert len(param) > 0, \"At least one parameter needed\"\n if len(param) == 1:\n return param[0].view(np.ndarray)\n return [x.view(np.ndarray) for x in param]", "response": "Convert an arbitrary number of parameters to numpy arrays."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef Kdiag(self, X):\n Kdiag = np.zeros(X.shape[0])\n ly=1/self.lengthscale_Y\n lu=np.sqrt(3)/self.lengthscale_U\n\n Vu = self.variance_U\n Vy=self.variance_Y\n\n k1 = (2*lu+ly)/(lu+ly)**2\n k2 = (ly-2*lu + 2*lu-ly ) / (ly-lu)**2\n k3 = 1/(lu+ly) + (lu)/(lu+ly)**2\n\n slices = index_to_slices(X[:,-1])\n\n for i, ss1 in enumerate(slices):\n for s1 in ss1:\n if i==0:\n Kdiag[s1]+= self.variance_U + self.ubias\n elif i==1:\n Kdiag[s1]+= Vu*Vy*(k1+k2+k3)\n else:\n raise ValueError(\"invalid input/output index\")\n #Kdiag[slices[0][0]]+= self.variance_U #matern32 diag\n #Kdiag[slices[1][0]]+= self.variance_U*self.variance_Y*(k1+k2+k3) # diag\n return Kdiag", "response": "Compute the diagonal of the covariance matrix associated to X."} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nupdating the gradients of the covariance matrix with respect to the parameters.", "response": "def update_gradients_full(self, dL_dK, X, X2=None):\n \"\"\"derivative of the covariance matrix with respect to the parameters.\"\"\"\n X,slices = X[:,:-1],index_to_slices(X[:,-1])\n if X2 is None:\n X2,slices2 = X,slices\n else:\n X2,slices2 = X2[:,:-1],index_to_slices(X2[:,-1])\n #rdist = X[:,0][:,None] - X2[:,0][:,None].T\n\n rdist = X - X2.T\n ly=1/self.lengthscale_Y\n lu=np.sqrt(3)/self.lengthscale_U\n\n Vu=self.variance_U\n Vy=self.variance_Y\n Vyu = np.sqrt(Vy*ly*2)\n dVdly = 0.5/np.sqrt(ly)*np.sqrt(2*Vy)\n dVdVy = 0.5/np.sqrt(Vy)*np.sqrt(2*ly)\n\n rd=rdist.shape[0]\n dktheta1 = np.zeros([rd,rd])\n dktheta2 = np.zeros([rd,rd])\n dkUdvar = np.zeros([rd,rd])\n dkYdvar = np.zeros([rd,rd])\n\n dkdubias = np.zeros([rd,rd])\n\n # dk dtheta for UU\n UUdtheta1 = lambda dist: np.exp(-lu* dist)*dist + (-dist)*np.exp(-lu* dist)*(1+lu*dist)\n UUdtheta2 = lambda dist: 0\n #UUdvar = lambda dist: (1 + lu*dist)*np.exp(-lu*dist)\n UUdvar = lambda dist: (1 + lu* np.abs(dist)) * np.exp(-lu * np.abs(dist))\n\n # dk dtheta for YY\n\n dk1theta1 = lambda dist: np.exp(-ly*dist)*2*(-lu)/(lu+ly)**3\n\n dk2theta1 = lambda dist: (1.0)*(\n np.exp(-lu*dist)*dist*(-ly+2*lu-lu*ly*dist+dist*lu**2)*(ly-lu)**(-2) + np.exp(-lu*dist)*(-2+ly*dist-2*dist*lu)*(ly-lu)**(-2)\n +np.exp(-dist*lu)*(ly-2*lu+ly*lu*dist-dist*lu**2)*2*(ly-lu)**(-3)\n +np.exp(-dist*ly)*2*(ly-lu)**(-2)\n +np.exp(-dist*ly)*2*(2*lu-ly)*(ly-lu)**(-3)\n )\n\n dk3theta1 = lambda dist: np.exp(-dist*lu)*(lu+ly)**(-2)*((2*lu+ly+dist*lu**2+lu*ly*dist)*(-dist-2/(lu+ly))+2+2*lu*dist+ly*dist)\n\n #dktheta1 = lambda dist: self.variance_U*self.variance_Y*(dk1theta1+dk2theta1+dk3theta1)\n\n\n\n\n dk1theta2 = lambda dist: np.exp(-ly*dist) * ((lu+ly)**(-2)) * ( (-dist)*(2*lu+ly) + 1 + (-2)*(2*lu+ly)/(lu+ly) )\n\n dk2theta2 =lambda dist: 1*(\n np.exp(-dist*lu)*(ly-lu)**(-2) * ( 1+lu*dist+(-2)*(ly-2*lu+lu*ly*dist-dist*lu**2)*(ly-lu)**(-1) )\n +np.exp(-dist*ly)*(ly-lu)**(-2) * ( (-dist)*(2*lu-ly) -1+(2*lu-ly)*(-2)*(ly-lu)**(-1) )\n )\n\n dk3theta2 = lambda dist: np.exp(-dist*lu) * (-3*lu-ly-dist*lu**2-lu*ly*dist)/(lu+ly)**3\n\n #dktheta2 = lambda dist: self.variance_U*self.variance_Y*(dk1theta2 + dk2theta2 +dk3theta2)\n\n # kyy kernel\n\n k1 = lambda dist: np.exp(-ly*dist)*(2*lu+ly)/(lu+ly)**2\n k2 = lambda dist: (np.exp(-lu*dist)*(ly-2*lu+lu*ly*dist-lu**2*dist) + np.exp(-ly*dist)*(2*lu-ly) ) / (ly-lu)**2\n k3 = lambda dist: np.exp(-lu*dist) * ( (1+lu*dist)/(lu+ly) + (lu)/(lu+ly)**2 )\n #dkdvar = k1+k2+k3\n\n\n\n # cross covariance function\n kyu3 = lambda dist:np.exp(-lu*dist)/(lu+ly)*(1+lu*(dist+1/(lu+ly)))\n\n k1cros = lambda dist:np.exp(ly*dist)/(lu-ly) * ( 1- np.exp( (lu-ly)*dist) + lu* ( dist*np.exp( (lu-ly)*dist ) + (1- np.exp( (lu-ly)*dist ) ) /(lu-ly) ) )\n\n k2cros = lambda dist:np.exp(ly*dist)*( 1/(lu+ly) + lu/(lu+ly)**2 )\n # cross covariance kuy\n kuyp = lambda dist:(kyu3(dist)) #t>0 kuy\n kuyn = lambda dist:(k1cros(dist)+k2cros(dist)) #t<0 kuy\n # cross covariance kyu\n kyup = lambda dist:(k1cros(-dist)+k2cros(-dist)) #t>0 kyu\n kyun = lambda dist:(kyu3(-dist)) #t<0 kyu\n\n # dk dtheta for UY\n\n\n dkyu3dtheta2 = lambda dist: np.exp(-lu*dist) * ( (-1)*(lu+ly)**(-2)*(1+lu*dist+lu*(lu+ly)**(-1)) + (lu+ly)**(-1)*(-lu)*(lu+ly)**(-2) )\n dkyu3dtheta1 = lambda dist: np.exp(-lu*dist)*(lu+ly)**(-1)* ( (-dist)*(1+dist*lu+lu*(lu+ly)**(-1)) -\\\n (lu+ly)**(-1)*(1+dist*lu+lu*(lu+ly)**(-1)) +dist+(lu+ly)**(-1)-lu*(lu+ly)**(-2) )\n\n dkcros2dtheta1 = lambda dist: np.exp(ly*dist)* ( -(ly+lu)**(-2) + (ly+lu)**(-2) + (-2)*lu*(lu+ly)**(-3) )\n dkcros2dtheta2 = lambda dist: np.exp(ly*dist)*dist* ( (ly+lu)**(-1) + lu*(lu+ly)**(-2) ) + \\\n np.exp(ly*dist)*( -(lu+ly)**(-2) + lu*(-2)*(lu+ly)**(-3) )\n\n dkcros1dtheta1 = lambda dist: np.exp(ly*dist)*( -(lu-ly)**(-2)*( 1-np.exp((lu-ly)*dist) + lu*dist*np.exp((lu-ly)*dist)+ \\\n lu*(1-np.exp((lu-ly)*dist))/(lu-ly) ) + (lu-ly)**(-1)*( -np.exp( (lu-ly)*dist )*dist + dist*np.exp( (lu-ly)*dist)+\\\n lu*dist**2*np.exp((lu-ly)*dist)+(1-np.exp((lu-ly)*dist))/(lu-ly) - lu*np.exp((lu-ly)*dist)*dist/(lu-ly) -\\\n lu*(1-np.exp((lu-ly)*dist))/(lu-ly)**2 ) )\n\n dkcros1dtheta2 = lambda t: np.exp(ly*t)*t/(lu-ly)*( 1-np.exp((lu-ly)*t) +lu*t*np.exp((lu-ly)*t)+\\\n lu*(1-np.exp((lu-ly)*t))/(lu-ly) )+\\\n np.exp(ly*t)/(lu-ly)**2* ( 1-np.exp((lu-ly)*t) +lu*t*np.exp((lu-ly)*t) + lu*( 1-np.exp((lu-ly)*t) )/(lu-ly) )+\\\n np.exp(ly*t)/(lu-ly)*( np.exp((lu-ly)*t)*t -lu*t*t*np.exp((lu-ly)*t) +lu*t*np.exp((lu-ly)*t)/(lu-ly)+\\\n lu*( 1-np.exp((lu-ly)*t) )/(lu-ly)**2 )\n\n dkuypdtheta1 = lambda dist:(dkyu3dtheta1(dist)) #t>0 kuy\n dkuyndtheta1 = lambda dist:(dkcros1dtheta1(dist)+dkcros2dtheta1(dist)) #t<0 kuy\n # cross covariance kyu\n dkyupdtheta1 = lambda dist:(dkcros1dtheta1(-dist)+dkcros2dtheta1(-dist)) #t>0 kyu\n dkyundtheta1 = lambda dist:(dkyu3dtheta1(-dist)) #t<0 kyu\n\n dkuypdtheta2 = lambda dist:(dkyu3dtheta2(dist)) #t>0 kuy\n dkuyndtheta2 = lambda dist:(dkcros1dtheta2(dist)+dkcros2dtheta2(dist)) #t<0 kuy\n # cross covariance kyu\n dkyupdtheta2 = lambda dist:(dkcros1dtheta2(-dist)+dkcros2dtheta2(-dist)) #t>0 kyu\n dkyundtheta2 = lambda dist:(dkyu3dtheta2(-dist)) #t<0 kyu\n\n\n for i, s1 in enumerate(slices):\n for j, s2 in enumerate(slices2):\n for ss1 in s1:\n for ss2 in s2:\n if i==0 and j==0:\n #target[ss1,ss2] = kuu(np.abs(rdist[ss1,ss2]))\n dktheta1[ss1,ss2] = Vu*UUdtheta1(np.abs(rdist[ss1,ss2]))\n dktheta2[ss1,ss2] = 0\n dkUdvar[ss1,ss2] = UUdvar(np.abs(rdist[ss1,ss2]))\n dkYdvar[ss1,ss2] = 0\n dkdubias[ss1,ss2] = 1\n elif i==0 and j==1:\n ########target[ss1,ss2] = np.where( rdist[ss1,ss2]>0 , kuyp(np.abs(rdist[ss1,ss2])), kuyn(np.abs(rdist[s1[0],s2[0]]) ) )\n #np.where( rdist[ss1,ss2]>0 , kuyp(np.abs(rdist[ss1,ss2])), kuyn(np.abs(rdist[s1[0],s2[0]]) ) )\n #dktheta1[ss1,ss2] = np.where( rdist[ss1,ss2]>0 , self.variance_U*self.variance_Y*dkcrtheta1(np.abs(rdist[ss1,ss2])) ,self.variance_U*self.variance_Y*(dk1theta1(np.abs(rdist[ss1,ss2]))+dk2theta1(np.abs(rdist[ss1,ss2]))) )\n #dktheta2[ss1,ss2] = np.where( rdist[ss1,ss2]>0 , self.variance_U*self.variance_Y*dkcrtheta2(np.abs(rdist[ss1,ss2])) ,self.variance_U*self.variance_Y*(dk1theta2(np.abs(rdist[ss1,ss2]))+dk2theta2(np.abs(rdist[ss1,ss2]))) )\n dktheta1[ss1,ss2] = np.where( rdist[ss1,ss2]>0 , Vu*Vyu*dkuypdtheta1(rdist[ss1,ss2]),Vu*Vyu*dkuyndtheta1(rdist[ss1,ss2]) )\n dkUdvar[ss1,ss2] = np.where( rdist[ss1,ss2]>0 , Vyu*kuyp(rdist[ss1,ss2]), Vyu* kuyn(rdist[ss1,ss2]) )\n dktheta2[ss1,ss2] = np.where( rdist[ss1,ss2]>0 , Vu*Vyu*dkuypdtheta2(rdist[ss1,ss2])+Vu*dVdly*kuyp(rdist[ss1,ss2]),Vu*Vyu*dkuyndtheta2(rdist[ss1,ss2])+Vu*dVdly*kuyn(rdist[ss1,ss2]) )\n dkYdvar[ss1,ss2] = np.where( rdist[ss1,ss2]>0 , Vu*dVdVy*kuyp(rdist[ss1,ss2]), Vu*dVdVy* kuyn(rdist[ss1,ss2]) )\n dkdubias[ss1,ss2] = 0\n elif i==1 and j==1:\n #target[ss1,ss2] = kyy(np.abs(rdist[ss1,ss2]))\n dktheta1[ss1,ss2] = self.variance_U*self.variance_Y*(dk1theta1(np.abs(rdist[ss1,ss2]))+dk2theta1(np.abs(rdist[ss1,ss2]))+dk3theta1(np.abs(rdist[ss1,ss2])))\n dktheta2[ss1,ss2] = self.variance_U*self.variance_Y*(dk1theta2(np.abs(rdist[ss1,ss2])) + dk2theta2(np.abs(rdist[ss1,ss2])) +dk3theta2(np.abs(rdist[ss1,ss2])))\n dkUdvar[ss1,ss2] = self.variance_Y*(k1(np.abs(rdist[ss1,ss2]))+k2(np.abs(rdist[ss1,ss2]))+k3(np.abs(rdist[ss1,ss2])) )\n dkYdvar[ss1,ss2] = self.variance_U*(k1(np.abs(rdist[ss1,ss2]))+k2(np.abs(rdist[ss1,ss2]))+k3(np.abs(rdist[ss1,ss2])) )\n dkdubias[ss1,ss2] = 0\n else:\n #######target[ss1,ss2] = np.where( rdist[ss1,ss2]>0 , kyup(np.abs(rdist[ss1,ss2])), kyun(np.abs(rdist[s1[0],s2[0]]) ) )\n #dktheta1[ss1,ss2] = np.where( rdist[ss1,ss2]>0 ,self.variance_U*self.variance_Y*(dk1theta1(np.abs(rdist[ss1,ss2]))+dk2theta1(np.abs(rdist[ss1,ss2]))) , self.variance_U*self.variance_Y*dkcrtheta1(np.abs(rdist[ss1,ss2])) )\n #dktheta2[ss1,ss2] = np.where( rdist[ss1,ss2]>0 ,self.variance_U*self.variance_Y*(dk1theta2(np.abs(rdist[ss1,ss2]))+dk2theta2(np.abs(rdist[ss1,ss2]))) , self.variance_U*self.variance_Y*dkcrtheta2(np.abs(rdist[ss1,ss2])) )\n dktheta1[ss1,ss2] = np.where( rdist[ss1,ss2]>0 , Vu*Vyu*dkyupdtheta1(rdist[ss1,ss2]),Vu*Vyu*dkyundtheta1(rdist[ss1,ss2]) )\n dkUdvar[ss1,ss2] = np.where( rdist[ss1,ss2]>0 , Vyu*kyup(rdist[ss1,ss2]),Vyu*kyun(rdist[ss1,ss2]))\n dktheta2[ss1,ss2] = np.where( rdist[ss1,ss2]>0 , Vu*Vyu*dkyupdtheta2(rdist[ss1,ss2])+Vu*dVdly*kyup(rdist[ss1,ss2]),Vu*Vyu*dkyundtheta2(rdist[ss1,ss2])+Vu*dVdly*kyun(rdist[ss1,ss2]) )\n dkYdvar[ss1,ss2] = np.where( rdist[ss1,ss2]>0 , Vu*dVdVy*kyup(rdist[ss1,ss2]), Vu*dVdVy*kyun(rdist[ss1,ss2]))\n dkdubias[ss1,ss2] = 0\n #stop\n self.variance_U.gradient = np.sum(dkUdvar * dL_dK) # Vu\n\n self.variance_Y.gradient = np.sum(dkYdvar * dL_dK) # Vy\n\n self.lengthscale_U.gradient = np.sum(dktheta1*(-np.sqrt(3)*self.lengthscale_U**(-2))* dL_dK) #lu\n\n self.lengthscale_Y.gradient = np.sum(dktheta2*(-self.lengthscale_Y**(-2)) * dL_dK) #ly\n\n self.ubias.gradient = np.sum(dkdubias * dL_dK)"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef set_data(self, X, Y):\n assert X.shape[1]==self.Z.shape[1]\n self.X, self.Y = X, Y", "response": "Set the data of the current object without calling parameters_changed to avoid wasted computation\n "} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef new_batch(self):\n i = next(self.slicer)\n return self.X_all[i], self.Y_all[i]", "response": "Return a new batch of X and Y by taking a chunk of data from the complete X and Y."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef to_dict(self):\n\n input_dict = super(Standardize, self)._save_to_input_dict()\n input_dict[\"class\"] = \"GPy.util.normalizer.Standardize\"\n if self.mean is not None:\n input_dict[\"mean\"] = self.mean.tolist()\n input_dict[\"std\"] = self.std.tolist()\n return input_dict", "response": "Convert the object into a json serializable dictionary."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef predict(self,function,args):\n param = self.model.param_array\n fs = []\n for p in self.chain:\n self.model.param_array = p\n fs.append(function(*args))\n # reset model to starting state\n self.model.param_array = param\n return fs", "response": "Make a prediction for the function to which we will pass the additional arguments"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncomputes the gradients of the parameters of the class.", "response": "def _param_grad_helper(self,X,X2,target):\n \"\"\"Return shape is NxMx(Ntheta)\"\"\"\n if X2 is None: X2 = X\n FX = np.column_stack([f(X) for f in self.F])\n FX2 = np.column_stack([f(X2) for f in self.F])\n DER = np.zeros((self.n,self.n,self.n))\n for i in range(self.n):\n DER[i,i,i] = np.sqrt(self.weights[i])\n dw = self.variance * mdot(FX,DER,self.G_1,np.diag(np.sqrt(self.weights)),FX2.T)\n dv = mdot(FX,np.diag(np.sqrt(self.weights)),self.G_1,np.diag(np.sqrt(self.weights)),FX2.T)\n np.add(target[:,:,0],np.transpose(dv,(0,2,1)), target[:,:,0])\n np.add(target[:,:,1:],np.transpose(dw,(0,2,1)), target[:,:,1:])"} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nset the input and output data of the model", "response": "def set_XY(self, X, Y):\n \"\"\"\n Set the input / output data of the model\n This is useful if we wish to change our existing data but maintain the same model\n\n :param X: input observations\n :type X: np.ndarray\n :param Y: output observations\n :type Y: np.ndarray or ObsAr\n \"\"\"\n self.update_model(False)\n self.set_Y(Y)\n self.set_X(X)\n self.update_model(True)"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef set_X(self, X):\n assert isinstance(X, np.ndarray)\n state = self.update_model()\n self.update_model(False)\n self.X = ObsAr(X)\n self.update_model(state)", "response": "Set the input data of the model containing the set of observations of the current version of the logistic logistic."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef set_Y(self, Y):\n assert isinstance(Y, (np.ndarray, ObsAr))\n state = self.update_model()\n self.update_model(False)\n if self.normalizer is not None:\n self.normalizer.scale_by(Y)\n self.Y_normalized = ObsAr(self.normalizer.normalize(Y))\n self.Y = Y\n else:\n self.Y = ObsAr(Y) if isinstance(Y, np.ndarray) else Y\n self.Y_normalized = self.Y\n self.update_model(state)", "response": "Set the output data of the model of the object."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef predict(self, Xnew, full_cov=False, kern=None, **kwargs):\n return self.predict_noiseless(Xnew, full_cov=full_cov, kern=kern)", "response": "Predict the function at the new point Xnew."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef predict_noiseless(self, Xnew, full_cov=False, kern=None):\n # Predict the latent function values\n mu, var = self._raw_predict(Xnew, full_cov=full_cov, kern=kern)\n\n # Un-apply normalization\n if self.normalizer is not None:\n mu, var = self.normalizer.inverse_mean(mu), self.normalizer.inverse_variance(var)\n\n return mu, var", "response": "Predict the underlying function f at the new point Xnew."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\npredict the quantiles around the prediction at X.", "response": "def predict_quantiles(self, X, quantiles=(2.5, 97.5), kern=None, **kwargs):\n \"\"\"\n Get the predictive quantiles around the prediction at X\n\n :param X: The points at which to make a prediction\n :type X: np.ndarray (Xnew x self.input_dim)\n :param quantiles: tuple of quantiles, default is (2.5, 97.5) which is the 95% interval\n :type quantiles: tuple\n :param kern: optional kernel to use for prediction\n :type predict_kw: dict\n :returns: list of quantiles for each X and predictive quantiles for interval combination\n :rtype: [np.ndarray (Xnew x self.output_dim), np.ndarray (Xnew x self.output_dim)]\n \"\"\"\n mu, var = self._raw_predict(X, full_cov=False, kern=kern)\n quantiles = [stats.t.ppf(q / 100., self.nu + 2 + self.num_data) * np.sqrt(var) + mu for q in quantiles]\n\n if self.normalizer is not None:\n quantiles = [self.normalizer.inverse_mean(q) for q in quantiles]\n\n return quantiles"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nsampling the posterior GP at the points X.", "response": "def posterior_samples(self, X, size=10, full_cov=False, Y_metadata=None, likelihood=None, **predict_kwargs):\n \"\"\"\n Samples the posterior GP at the points X, equivalent to posterior_samples_f due to the absence of a likelihood.\n \"\"\"\n return self.posterior_samples_f(X, size, full_cov=full_cov, **predict_kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nsample the posterior TP at the points X.", "response": "def posterior_samples_f(self, X, size=10, full_cov=True, **predict_kwargs):\n \"\"\"\n Samples the posterior TP at the points X.\n\n :param X: The points at which to take the samples.\n :type X: np.ndarray (Nnew x self.input_dim)\n :param size: the number of a posteriori samples.\n :type size: int.\n :param full_cov: whether to return the full covariance matrix, or just the diagonal.\n :type full_cov: bool.\n :returns: fsim: set of simulations\n :rtype: np.ndarray (D x N x samples) (if D==1 we flatten out the first dimension)\n \"\"\"\n mu, var = self._raw_predict(X, full_cov=full_cov, **predict_kwargs)\n if self.normalizer is not None:\n mu, var = self.normalizer.inverse_mean(mu), self.normalizer.inverse_variance(var)\n\n def sim_one_dim(m, v):\n nu = self.nu + 2 + self.num_data\n v = np.diag(v.flatten()) if not full_cov else v\n Z = np.random.multivariate_normal(np.zeros(X.shape[0]), v, size).T\n g = np.tile(np.random.gamma(nu / 2., 2. / nu, size), (X.shape[0], 1))\n return m + Z / np.sqrt(g)\n\n if self.output_dim == 1:\n return sim_one_dim(mu, var)\n else:\n fsim = np.empty((self.output_dim, self.num_data, size))\n for d in range(self.output_dim):\n if full_cov and var.ndim == 3:\n fsim[d] = sim_one_dim(mu[:, d], var[:, :, d])\n elif (not full_cov) and var.ndim == 2:\n fsim[d] = sim_one_dim(mu[:, d], var[:, d])\n else:\n fsim[d] = sim_one_dim(mu[:, d], var)\n return fsim"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ngetting a view on the diagonal elements of a 2D array.", "response": "def view(A, offset=0):\n \"\"\"\n Get a view on the diagonal elements of a 2D array.\n\n This is actually a view (!) on the diagonal of the array, so you can\n in-place adjust the view.\n\n :param :class:`ndarray` A: 2 dimensional numpy array\n :param int offset: view offset to give back (negative entries allowed)\n :rtype: :class:`ndarray` view of diag(A)\n\n >>> import numpy as np\n >>> X = np.arange(9).reshape(3,3)\n >>> view(X)\n array([0, 4, 8])\n >>> d = view(X)\n >>> d += 2\n >>> view(X)\n array([ 2, 6, 10])\n >>> view(X, offset=-1)\n array([3, 7])\n >>> subtract(X, 3, offset=-1)\n array([[ 2, 1, 2],\n [ 0, 6, 5],\n [ 6, 4, 10]])\n \"\"\"\n from numpy.lib.stride_tricks import as_strided\n assert A.ndim == 2, \"only implemented for 2 dimensions\"\n assert A.shape[0] == A.shape[1], \"attempting to get the view of non-square matrix?!\"\n if offset > 0:\n return as_strided(A[0, offset:], shape=(A.shape[0] - offset, ), strides=((A.shape[0]+1)*A.itemsize, ))\n elif offset < 0:\n return as_strided(A[-offset:, 0], shape=(A.shape[0] + offset, ), strides=((A.shape[0]+1)*A.itemsize, ))\n else:\n return as_strided(A, shape=(A.shape[0], ), strides=((A.shape[0]+1)*A.itemsize, ))"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef times(A, b, offset=0):\n return _diag_ufunc(A, b, offset, np.multiply)", "response": "Times the view of A with b in place."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef divide(A, b, offset=0):\n return _diag_ufunc(A, b, offset, np.divide)", "response": "Divides the view of A by b in place."} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nadds b to the view of A in place.", "response": "def add(A, b, offset=0):\n \"\"\"\n Add b to the view of A in place (!).\n Returns modified A.\n Broadcasting is allowed, thus b can be scalar.\n\n if offset is not zero, make sure b is of right shape!\n\n :param ndarray A: 2 dimensional array\n :param ndarray-like b: either one dimensional or scalar\n :param int offset: same as in view.\n :rtype: view of A, which is adjusted inplace\n \"\"\"\n return _diag_ufunc(A, b, offset, np.add)"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef subtract(A, b, offset=0):\n return _diag_ufunc(A, b, offset, np.subtract)", "response": "Subtract b from the view of A in place."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef from_dict(input_dict):\n\n import copy\n input_dict = copy.deepcopy(input_dict)\n inference_class = input_dict.pop('class')\n import GPy\n inference_class = eval(inference_class)\n return inference_class._build_from_input_dict(inference_class, input_dict)", "response": "Instantiate an object of a derived class from a dictionary."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef to_dict(self):\n\n input_dict = super(Add, self)._save_to_input_dict()\n input_dict[\"class\"] = str(\"GPy.kern.Add\")\n return input_dict", "response": "Convert the object into a json serializable dictionary."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncomputing the gradient of the objective function with respect to X.", "response": "def gradients_X(self, dL_dK, X, X2=None):\n \"\"\"Compute the gradient of the objective function with respect to X.\n\n :param dL_dK: An array of gradients of the objective function with respect to the covariance function.\n :type dL_dK: np.ndarray (num_samples x num_inducing)\n :param X: Observed data inputs\n :type X: np.ndarray (num_samples x input_dim)\n :param X2: Observed data inputs (optional, defaults to X)\n :type X2: np.ndarray (num_inducing x input_dim)\"\"\"\n\n target = np.zeros(X.shape)\n [target.__iadd__(p.gradients_X(dL_dK, X, X2)) for p in self.parts]\n return target"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef conf_matrix(p,labels,names=['1','0'],threshold=.5,show=True):\n assert p.size == labels.size, \"Arrays p and labels have different dimensions.\"\n decision = np.ones((labels.size,1))\n decision[p ilk"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef load(file_or_path):\n # This is the pickling pain when changing _src -> src\n import sys\n import inspect\n sys.modules['GPy.kern._src'] = kern.src\n for name, module in inspect.getmembers(kern.src):\n if not name.startswith('_'):\n sys.modules['GPy.kern._src.{}'.format(name)] = module\n sys.modules['GPy.inference.optimization'] = inference.optimization\n import paramz\n return paramz.load(file_or_path)", "response": "Load a previously pickled model using m. pickle"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef quadvgk(feval, fmin, fmax, tol1=1e-5, tol2=1e-5):\n\n XK = np.array([-0.991455371120813, -0.949107912342759, -0.864864423359769, -0.741531185599394,\n -0.586087235467691, -0.405845151377397, -0.207784955007898, 0.,\n 0.207784955007898, 0.405845151377397, 0.586087235467691,\n 0.741531185599394, 0.864864423359769, 0.949107912342759, 0.991455371120813])\n WK = np.array([0.022935322010529, 0.063092092629979, 0.104790010322250, 0.140653259715525,\n 0.169004726639267, 0.190350578064785, 0.204432940075298, 0.209482141084728,\n 0.204432940075298, 0.190350578064785, 0.169004726639267,\n 0.140653259715525, 0.104790010322250, 0.063092092629979, 0.022935322010529])\n # 7-point Gaussian weightings\n WG = np.array([0.129484966168870, 0.279705391489277, 0.381830050505119, 0.417959183673469,\n 0.381830050505119, 0.279705391489277, 0.129484966168870])\n\n NK = WK.size\n G = np.arange(2,NK,2)\n tol1 = 1e-4\n tol2 = 1e-4\n Subs = np.array([[fmin],[fmax]])\n # number of functions to evaluate in the feval vector of functions.\n NF = feval(np.zeros(1)).size\n Q = np.zeros(NF)\n neval = 0\n while Subs.size > 0:\n Subs = getSubs(Subs,XK)\n M = (Subs[1,:] - Subs[0,:]) / 2\n C = (Subs[1,:] + Subs[0,:]) / 2\n # NM = length(M);\n NM = M.size\n # x = reshape(XK * M + ones(NK, 1) * C, 1, []);\n x = XK[:,None]*M + C\n x = x.flatten()\n FV = feval(x)\n # FV = FV[:,None]\n Q1 = np.zeros((NF, NM))\n Q2 = np.zeros((NF, NM))\n\n # for n=1:NF\n # F = reshape(FV(n,:), NK, []);\n # Q1(n,:) = M. * sum((WK * ones(1, NM)). * F);\n # Q2(n,:) = M. * sum((WG * ones(1, NM)). * F(G,:));\n # end\n # for i in range(NF):\n # F = FV\n # F = F.reshape((NK,-1))\n # temp_mat = np.sum(np.multiply(WK[:,None]*np.ones((1,NM)), F),axis=0)\n # Q1[i,:] = np.multiply(M, temp_mat)\n # temp_mat = np.sum(np.multiply(WG[:,None]*np.ones((1, NM)), F[G-1,:]), axis=0)\n # Q2[i,:] = np.multiply(M, temp_mat)\n # ind = np.where(np.logical_or(np.max(np.abs(Q1 -Q2) / Q1) < tol1, (Subs[1,:] - Subs[0,:]) <= tol2) > 0)[0]\n # Q = Q + np.sum(Q1[:,ind], axis=1)\n # np.delete(Subs, ind,axis=1)\n\n Q1 = np.dot(FV.reshape(NF, NK, NM).swapaxes(2,1),WK)*M\n Q2 = np.dot(FV.reshape(NF, NK, NM).swapaxes(2,1)[:,:,1::2],WG)*M\n #ind = np.nonzero(np.logical_or(np.max(np.abs((Q1-Q2)/Q1), 0) < difftol , M < xtol))[0]\n ind = np.nonzero(np.logical_or(np.max(np.abs((Q1-Q2)), 0) < tol1 , (Subs[1,:] - Subs[0,:]) < tol2))[0]\n Q = Q + np.sum(Q1[:,ind], axis=1)\n Subs = np.delete(Subs, ind, axis=1)\n return Q", "response": "This function is used to compute the kronrod integration of a single dimensional kronrod."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nintegrate f from fmin to fmax with quadgk function", "response": "def quadgk_int(f, fmin=-np.inf, fmax=np.inf, difftol=0.1):\n \"\"\"\n Integrate f from fmin to fmax,\n do integration by substitution\n x = r / (1-r**2)\n when r goes from -1 to 1 , x goes from -inf to inf.\n the interval for quadgk function is from -1 to +1, so we transform the space from (-inf,inf) to (-1,1)\n :param f:\n :param fmin:\n :param fmax:\n :param difftol:\n :return:\n \"\"\"\n difftol = 1e-4\n def trans_func(r):\n r2 = np.square(r)\n x = r / (1-r2)\n dx_dr = (1 + r2)/(1-r2)**2\n return f(x)*dx_dr\n\n integrand = quadvgk(trans_func, -1., 1., difftol, difftol)\n return integrand"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nupdates internal status by gathering the variational posteriors for all the individual models.", "response": "def _update_inernal(self, varp_list):\n \"\"\"Make an update of the internal status by gathering the variational posteriors for all the individual models.\"\"\"\n # The probability for the binary variable for the same latent dimension of any of the models is on.\n if self.group_spike:\n self._b_prob_all = 1.-param_to_array(varp_list[0].gamma_group)\n [np.multiply(self._b_prob_all, 1.-vp.gamma_group, self._b_prob_all) for vp in varp_list[1:]]\n else:\n self._b_prob_all = 1.-param_to_array(varp_list[0].binary_prob)\n [np.multiply(self._b_prob_all, 1.-vp.binary_prob, self._b_prob_all) for vp in varp_list[1:]]"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nmake an update of the internal status by gathering the variational posteriors for all the individual models.", "response": "def _update_inernal(self, varp_list):\n \"\"\"Make an update of the internal status by gathering the variational posteriors for all the individual models.\"\"\"\n # The probability for the binary variable for the same latent dimension of any of the models is on.\n self._b_prob_all = 1.-param_to_array(varp_list[0].gamma_group)\n [np.multiply(self._b_prob_all, 1.-vp.gamma_group, self._b_prob_all) for vp in varp_list[1:]]"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncompute the covariance matrix between X and X2.", "response": "def K(self,X,X2,target):\n \"\"\"Compute the covariance matrix between X and X2.\"\"\"\n AX = np.dot(X,self.transform)\n if X2 is None:\n X2 = X\n AX2 = AX\n else:\n AX2 = np.dot(X2, self.transform)\n self.k.K(X,X2,target)\n self.k.K(AX,X2,target)\n self.k.K(X,AX2,target)\n self.k.K(AX,AX2,target)"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _param_grad_helper(self,dL_dK,X,X2,target):\n AX = np.dot(X,self.transform)\n if X2 is None:\n X2 = X\n ZX2 = AX\n else:\n AX2 = np.dot(X2, self.transform)\n self.k._param_grad_helper(dL_dK,X,X2,target)\n self.k._param_grad_helper(dL_dK,AX,X2,target)\n self.k._param_grad_helper(dL_dK,X,AX2,target)\n self.k._param_grad_helper(dL_dK,AX,AX2,target)", "response": "derivative of the covariance matrix with respect to the parameters."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncomputes the diagonal of the covariance matrix associated to X.", "response": "def Kdiag(self,X,target):\n \"\"\"Compute the diagonal of the covariance matrix associated to X.\"\"\"\n foo = np.zeros((X.shape[0],X.shape[0]))\n self.K(X,X,foo)\n target += np.diag(foo)"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nshape N num_inducing Ntheta", "response": "def dpsi2_dtheta(self, dL_dpsi2, Z, mu, S, target):\n \"\"\"Shape N,num_inducing,num_inducing,Ntheta\"\"\"\n self._psi_computations(Z, mu, S)\n d_var = 2.*self._psi2 / self.variance\n # d_length = 2.*self._psi2[:, :, :, None] * (self._psi2_Zdist_sq * self._psi2_denom + self._psi2_mudist_sq + S[:, None, None, :] / self.lengthscale2) / (self.lengthscale * self._psi2_denom)\n d_length = -2.*self._psi2[:, :, :, None] * (self._psi2_Zdist_sq * self._psi2_denom + self._psi2_mudist_sq + S[:, None, None, :] * self.inv_lengthscale2) / (self.inv_lengthscale * self._psi2_denom)\n target[0] += np.sum(dL_dpsi2 * d_var)\n dpsi2_dlength = d_length * dL_dpsi2[:, :, :, None]\n if not self.ARD:\n target[1] += dpsi2_dlength.sum() # *(-self.lengthscale2)\n else:\n target[1:] += dpsi2_dlength.sum(0).sum(0).sum(0)"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef dpsi2_dmuS(self, dL_dpsi2, Z, mu, S, target_mu, target_S):\n self._psi_computations(Z, mu, S)\n tmp = (self.inv_lengthscale2 * self._psi2[:, :, :, None]) / self._psi2_denom\n target_mu += -2.*(dL_dpsi2[:, :, :, None] * tmp * self._psi2_mudist).sum(1).sum(1)\n target_S += (dL_dpsi2[:, :, :, None] * tmp * (2.*self._psi2_mudist_sq - 1)).sum(1).sum(1)", "response": "Think N num_inducing input_dim N - 1"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _psicomputations(self, kern, Z, variational_posterior, return_psi2_n=False):\n variance, lengthscale = kern.variance, kern.lengthscale\n N,M,Q = self.get_dimensions(Z, variational_posterior)\n self._initGPUCache(N,M,Q)\n self.sync_params(lengthscale, Z, variational_posterior.mean, variational_posterior.variance)\n \n psi1_gpu = self.gpuCache['psi1_gpu']\n psi2_gpu = self.gpuCache['psi2_gpu']\n psi2n_gpu = self.gpuCache['psi2n_gpu']\n l_gpu = self.gpuCache['l_gpu']\n Z_gpu = self.gpuCache['Z_gpu']\n mu_gpu = self.gpuCache['mu_gpu']\n S_gpu = self.gpuCache['S_gpu']\n log_denom1_gpu = self.gpuCache['log_denom1_gpu']\n log_denom2_gpu = self.gpuCache['log_denom2_gpu']\n\n psi0 = np.empty((N,))\n psi0[:] = variance\n self.g_psi1computations.prepared_call((self.blocknum,1),(self.threadnum,1,1),psi1_gpu.gpudata, log_denom1_gpu.gpudata, np.float64(variance),l_gpu.gpudata,Z_gpu.gpudata,mu_gpu.gpudata,S_gpu.gpudata, np.int32(N), np.int32(M), np.int32(Q))\n self.g_psi2computations.prepared_call((self.blocknum,1),(self.threadnum,1,1),psi2_gpu.gpudata, psi2n_gpu.gpudata, log_denom2_gpu.gpudata, np.float64(variance),l_gpu.gpudata,Z_gpu.gpudata,mu_gpu.gpudata,S_gpu.gpudata, np.int32(N), np.int32(M), np.int32(Q))\n # t = self.g_psi1computations(psi1_gpu, log_denom1_gpu, np.float64(variance),l_gpu,Z_gpu,mu_gpu,S_gpu, np.int32(N), np.int32(M), np.int32(Q), block=(self.threadnum,1,1), grid=(self.blocknum,1),time_kernel=True)\n # print 'g_psi1computations '+str(t)\n # t = self.g_psi2computations(psi2_gpu, psi2n_gpu, log_denom2_gpu, np.float64(variance),l_gpu,Z_gpu,mu_gpu,S_gpu, np.int32(N), np.int32(M), np.int32(Q), block=(self.threadnum,1,1), grid=(self.blocknum,1),time_kernel=True)\n # print 'g_psi2computations '+str(t)\n \n if self.GPU_direct:\n return psi0, psi1_gpu, psi2_gpu\n else:\n if return_psi2_n:\n return psi0, psi1_gpu.get(), psi2n_gpu.get()\n else:\n return psi0, psi1_gpu.get(), psi2_gpu.get()", "response": "This function is used to compute the psicomputations for the given kernel and Z."} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nsets the prior for this object to prior.", "response": "def set_prior(self, prior, warning=True):\n \"\"\"\n Set the prior for this object to prior.\n :param :class:`~GPy.priors.Prior` prior: a prior to set for this parameter\n :param bool warning: whether to warn if another prior was set for this parameter\n \"\"\"\n repriorized = self.unset_priors()\n self._add_to_index_operations(self.priors, repriorized, prior, warning)\n\n from paramz.domains import _REAL, _POSITIVE, _NEGATIVE\n if prior.domain is _POSITIVE:\n self.constrain_positive(warning)\n elif prior.domain is _NEGATIVE:\n self.constrain_negative(warning)\n elif prior.domain is _REAL:\n rav_i = self._raveled_index()\n assert all(all(False if c is __fixed__ else c.domain is _REAL for c in con) for con in self.constraints.properties_for(rav_i)), 'Domain of prior and constraint have to match, please unconstrain if you REALLY wish to use this prior'"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _log_prior_gradients(self):\n if self.priors.size == 0:\n return 0.\n x = self.param_array\n ret = np.zeros(x.size)\n #compute derivate of prior density\n [np.put(ret, ind, p.lnpdf_grad(x[ind])) for p, ind in self.priors.items()]\n #add in jacobian derivatives if transformed\n priored_indexes = np.hstack([i for p, i in self.priors.items()])\n for c,j in self.constraints.items():\n if not isinstance(c, Transformation):continue\n for jj in j:\n if jj in priored_indexes:\n ret[jj] += c.log_jacobian_grad(x[jj])\n return ret", "response": "evaluate the gradients of the priors"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _preprocess(self, data, train):\n if train:\n inputs, labels = data\n self.data_mean = inputs.mean(axis=0)\n self.data_std = inputs.std(axis=0)\n self.labels_mean = labels.mean(axis=0)\n self.labels_std = labels.std(axis=0)\n return ((inputs-self.data_mean)/self.data_std, (labels-self.labels_mean)/self.labels_std)\n else:\n return (data-self.data_mean)/self.data_std", "response": "Zero - mean unit - variance normalization by default"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_blocks_3d(A, blocksizes, pagesizes=None):\n assert (A.shape[0]==A.shape[1]) and len(A.shape)==3, \"can't blockify this non-square matrix, may need to use 2d version\"\n N = np.sum(blocksizes)\n assert A.shape[0] == N, \"bad blocksizes\"\n num_blocks = len(blocksizes)\n if pagesizes == None:\n #Assume each page of A should be its own dimension\n pagesizes = range(A.shape[2])#[0]*A.shape[2]\n num_pages = len(pagesizes)\n B = np.empty(shape=(num_blocks, num_blocks, num_pages), dtype=np.object)\n count_k = 0\n #for Bk, k in enumerate(pagesizes):\n for Bk in pagesizes:\n count_i = 0\n for Bi, i in enumerate(blocksizes):\n count_j = 0\n for Bj, j in enumerate(blocksizes):\n #We want to have it count_k:count_k + k but its annoying as it makes a NxNx1 array is page sizes are set to 1\n B[Bi, Bj, Bk] = A[count_i:count_i + i, count_j:count_j + j, Bk]\n count_j += j\n count_i += i\n #count_k += k\n return B", "response": "Given a 3d matrix A make a block matrix where the first and second dimensions are blocked according\n to blocksizes and the pages are blocked according\n to pagesizes."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef oil(num_inducing=50, max_iters=100, kernel=None, optimize=True, plot=True):\n try:import pods\n except ImportError:raise ImportWarning('Need pods for example datasets. See https://github.com/sods/ods, or pip install pods.')\n data = pods.datasets.oil()\n X = data['X']\n Xtest = data['Xtest']\n Y = data['Y'][:, 0:1]\n Ytest = data['Ytest'][:, 0:1]\n Y[Y.flatten()==-1] = 0\n Ytest[Ytest.flatten()==-1] = 0\n\n # Create GP model\n m = GPy.models.SparseGPClassification(X, Y, kernel=kernel, num_inducing=num_inducing)\n m.Ytest = Ytest\n\n # Contrain all parameters to be positive\n #m.tie_params('.*len')\n m['.*len'] = 10.\n\n # Optimize\n if optimize:\n m.optimize(messages=1)\n print(m)\n\n #Test\n probs = m.predict(Xtest)[0]\n GPy.util.classification.conf_matrix(probs, Ytest)\n return m", "response": "Run a Gaussian process classification on the three phase oil data."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef toy_linear_1d_classification_laplace(seed=default_seed, optimize=True, plot=True):\n\n try:import pods\n except ImportError:print('pods unavailable, see https://github.com/sods/ods for example datasets')\n data = pods.datasets.toy_linear_1d_classification(seed=seed)\n Y = data['Y'][:, 0:1]\n Y[Y.flatten() == -1] = 0\n\n likelihood = GPy.likelihoods.Bernoulli()\n laplace_inf = GPy.inference.latent_function_inference.Laplace()\n kernel = GPy.kern.RBF(1)\n\n # Model definition\n m = GPy.core.GP(data['X'], Y, kernel=kernel, likelihood=likelihood, inference_method=laplace_inf)\n\n # Optimize\n if optimize:\n try:\n m.optimize('scg', messages=1)\n except Exception as e:\n return m\n\n # Plot\n if plot:\n from matplotlib import pyplot as plt\n fig, axes = plt.subplots(2, 1)\n m.plot_f(ax=axes[0])\n m.plot(ax=axes[1])\n\n print(m)\n return m", "response": "Simple 1D classification example using Laplace approximation"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef sparse_toy_linear_1d_classification_uncertain_input(num_inducing=10, seed=default_seed, optimize=True, plot=True):\n\n try:import pods\n except ImportError:print('pods unavailable, see https://github.com/sods/ods for example datasets')\n import numpy as np\n data = pods.datasets.toy_linear_1d_classification(seed=seed)\n Y = data['Y'][:, 0:1]\n Y[Y.flatten() == -1] = 0\n X = data['X']\n X_var = np.random.uniform(0.3,0.5,X.shape)\n\n # Model definition\n m = GPy.models.SparseGPClassificationUncertainInput(X, X_var, Y, num_inducing=num_inducing)\n m['.*len'] = 4.\n\n # Optimize\n if optimize:\n m.optimize()\n\n # Plot\n if plot:\n from matplotlib import pyplot as plt\n fig, axes = plt.subplots(2, 1)\n m.plot_f(ax=axes[0])\n m.plot(ax=axes[1])\n\n print(m)\n return m", "response": "Generate a Sparse 1D classification example with uncertain input."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef crescent_data(model_type='Full', num_inducing=10, seed=default_seed, kernel=None, optimize=True, plot=True):\n try:import pods\n except ImportError:print('pods unavailable, see https://github.com/sods/ods for example datasets')\n data = pods.datasets.crescent_data(seed=seed)\n Y = data['Y']\n Y[Y.flatten()==-1] = 0\n\n if model_type == 'Full':\n m = GPy.models.GPClassification(data['X'], Y, kernel=kernel)\n\n elif model_type == 'DTC':\n m = GPy.models.SparseGPClassification(data['X'], Y, kernel=kernel, num_inducing=num_inducing)\n m['.*len'] = 10.\n\n elif model_type == 'FITC':\n m = GPy.models.FITCClassification(data['X'], Y, kernel=kernel, num_inducing=num_inducing)\n m['.*len'] = 3.\n if optimize:\n m.optimize(messages=1)\n\n if plot:\n m.plot()\n\n print(m)\n return m", "response": "Run a Gaussian process classification on the crescent data."} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\napply warping_function to some input data", "response": "def f(self, X, test_data=False):\n \"\"\"Apply warping_function to some Input data\n\n Parameters:\n -----------\n X : array_like, shape = (n_samples, n_features)\n\n test_data: bool, optional\n Default to False, should set to True when transforming test data\n\n Returns\n -------\n X_warped : array_like, shape = (n_samples, n_features)\n The warped input data\n\n Math\n ----\n f(x) = 1 - (1 - x^a)^b\n \"\"\"\n X_warped = X.copy()\n if test_data:\n X_normalized = (X - self.Xmin) / (self.Xmax - self.Xmin)\n else:\n X_normalized = self.X_normalized\n\n for i_seq, i_fea in enumerate(self.warping_indices):\n a, b = self.params[i_seq][0], self.params[i_seq][1]\n X_warped[:, i_fea] = 1 - np.power(1 - np.power(X_normalized[:, i_fea], a), b)\n return X_warped"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncomputing the gradient of warping function with respect to X", "response": "def fgrad_X(self, X):\n \"\"\"Compute the gradient of warping function with respect to X\n\n Parameters\n ----------\n X : array_like, shape = (n_samples, n_features)\n The location to compute gradient\n\n Returns\n -------\n grad : array_like, shape = (n_samples, n_features)\n The gradient for every location at X\n\n Math\n ----\n grad = a * b * x ^(a-1) * (1 - x^a)^(b-1)\n \"\"\"\n grad = np.zeros(X.shape)\n for i_seq, i_fea in enumerate(self.warping_indices):\n a, b = self.params[i_seq][0], self.params[i_seq][1]\n grad[:, i_fea] = a * b * np.power(self.X_normalized[:, i_fea], a-1) * \\\n np.power(1 - np.power(self.X_normalized[:, i_fea], a), b-1) * self.scaling[i_fea]\n return grad"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef update_grads(self, X, dL_dW):\n for i_seq, i_fea in enumerate(self.warping_indices):\n ai, bi = self.params[i_seq][0], self.params[i_seq][1]\n\n # cache some value for save some computation\n x_pow_a = np.power(self.X_normalized[:, i_fea], ai)\n\n # compute gradient for ai, bi on all X\n dz_dai = bi * np.power(1 - x_pow_a, bi-1) * x_pow_a * np.log(self.X_normalized[:, i_fea])\n dz_dbi = - np.power(1 - x_pow_a, bi) * np.log(1 - x_pow_a)\n\n # sum gradients on all the data\n dL_dai = np.sum(dL_dW[:, i_fea] * dz_dai)\n dL_dbi = np.sum(dL_dW[:, i_fea] * dz_dbi)\n self.params[i_seq][0].gradient[:] = dL_dai\n self.params[i_seq][1].gradient[:] = dL_dbi", "response": "Update the gradients of the marginal log likelihood with respect to the parameters of warping function."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef inference(self, kern, X, Z, likelihood, Y, indexD, output_dim, Y_metadata=None, Lm=None, dL_dKmm=None, Kuu_sigma=None):\n\n input_dim = Z.shape[0]\n\n uncertain_inputs = isinstance(X, VariationalPosterior)\n\n beta = 1./likelihood.variance\n if len(beta)==1:\n beta = np.zeros(output_dim)+beta\n\n beta_exp = np.zeros(indexD.shape[0])\n for d in range(output_dim):\n beta_exp[indexD==d] = beta[d]\n\n psi0, psi1, psi2 = self.gatherPsiStat(kern, X, Z, Y, beta, uncertain_inputs)\n\n psi2_sum = (beta_exp[:,None,None]*psi2).sum(0)/output_dim\n\n #======================================================================\n # Compute Common Components\n #======================================================================\n\n Kmm = kern.K(Z).copy()\n if Kuu_sigma is not None:\n diag.add(Kmm, Kuu_sigma)\n else:\n diag.add(Kmm, self.const_jitter)\n Lm = jitchol(Kmm)\n\n logL = 0.\n dL_dthetaL = np.zeros(output_dim)\n dL_dKmm = np.zeros_like(Kmm)\n dL_dpsi0 = np.zeros_like(psi0)\n dL_dpsi1 = np.zeros_like(psi1)\n dL_dpsi2 = np.zeros_like(psi2)\n wv = np.empty((Kmm.shape[0],output_dim))\n\n for d in range(output_dim):\n idx_d = indexD==d\n Y_d = Y[idx_d]\n N_d = Y_d.shape[0]\n beta_d = beta[d]\n\n psi2_d = psi2[idx_d].sum(0)*beta_d\n psi1Y = Y_d.T.dot(psi1[idx_d])*beta_d\n psi0_d = psi0[idx_d].sum()*beta_d\n YRY_d = np.square(Y_d).sum()*beta_d\n\n LmInvPsi2LmInvT = backsub_both_sides(Lm, psi2_d, 'right')\n\n Lambda = np.eye(Kmm.shape[0])+LmInvPsi2LmInvT\n LL = jitchol(Lambda)\n LmLL = Lm.dot(LL)\n\n b = dtrtrs(LmLL, psi1Y.T)[0].T\n bbt = np.square(b).sum()\n v = dtrtrs(LmLL, b.T, trans=1)[0].T\n LLinvPsi1TYYTPsi1LLinvT = tdot(b.T)\n\n tmp = -backsub_both_sides(LL, LLinvPsi1TYYTPsi1LLinvT)\n dL_dpsi2R = backsub_both_sides(Lm, tmp+np.eye(input_dim))/2\n\n logL_R = -N_d*np.log(beta_d)\n logL += -((N_d*log_2_pi+logL_R+psi0_d-np.trace(LmInvPsi2LmInvT))+YRY_d- bbt)/2.\n\n dL_dKmm += dL_dpsi2R - backsub_both_sides(Lm, LmInvPsi2LmInvT)/2\n\n dL_dthetaL[d:d+1] = (YRY_d*beta_d + beta_d*psi0_d - N_d*beta_d)/2. - beta_d*(dL_dpsi2R*psi2_d).sum() - beta_d*np.trace(LLinvPsi1TYYTPsi1LLinvT)\n\n dL_dpsi0[idx_d] = -beta_d/2.\n dL_dpsi1[idx_d] = beta_d*np.dot(Y_d,v)\n dL_dpsi2[idx_d] = beta_d*dL_dpsi2R\n wv[:,d] = v\n\n LmInvPsi2LmInvT = backsub_both_sides(Lm, psi2_sum, 'right')\n\n Lambda = np.eye(Kmm.shape[0])+LmInvPsi2LmInvT\n LL = jitchol(Lambda)\n LmLL = Lm.dot(LL)\n logdet_L = 2.*np.sum(np.log(np.diag(LL)))\n dL_dpsi2R_common = dpotri(LmLL)[0]/-2.\n dL_dpsi2 += dL_dpsi2R_common[None,:,:]*beta_exp[:,None,None]\n\n for d in range(output_dim):\n dL_dthetaL[d] += (dL_dpsi2R_common*psi2[indexD==d].sum(0)).sum()*-beta[d]*beta[d]\n\n dL_dKmm += dL_dpsi2R_common*output_dim\n\n logL += -output_dim*logdet_L/2.\n\n #======================================================================\n # Compute dL_dKmm\n #======================================================================\n\n # dL_dKmm = dL_dpsi2R - output_dim* backsub_both_sides(Lm, LmInvPsi2LmInvT)/2 #LmInv.T.dot(LmInvPsi2LmInvT).dot(LmInv)/2.\n\n #======================================================================\n # Compute the Posterior distribution of inducing points p(u|Y)\n #======================================================================\n\n LLInvLmT = dtrtrs(LL, Lm.T)[0]\n cov = tdot(LLInvLmT.T)\n\n wd_inv = backsub_both_sides(Lm, np.eye(input_dim)- backsub_both_sides(LL, np.identity(input_dim), transpose='left'), transpose='left')\n post = Posterior(woodbury_inv=wd_inv, woodbury_vector=wv, K=Kmm, mean=None, cov=cov, K_chol=Lm)\n\n #======================================================================\n # Compute dL_dthetaL for uncertian input and non-heter noise\n #======================================================================\n\n # for d in range(output_dim):\n # dL_dthetaL[d:d+1] += - beta[d]*beta[d]*(dL_dpsi2R[None,:,:] * psi2[indexD==d]/output_dim).sum()\n # dL_dthetaL += - (dL_dpsi2R[None,:,:] * psi2_sum*D beta*(dL_dpsi2R*psi2).sum()\n\n #======================================================================\n # Compute dL_dpsi\n #======================================================================\n\n if not uncertain_inputs:\n dL_dpsi1 += (psi1[:,None,:]*dL_dpsi2).sum(2)*2.\n\n if uncertain_inputs:\n grad_dict = {'dL_dKmm': dL_dKmm,\n 'dL_dpsi0':dL_dpsi0,\n 'dL_dpsi1':dL_dpsi1,\n 'dL_dpsi2':dL_dpsi2,\n 'dL_dthetaL':dL_dthetaL}\n else:\n grad_dict = {'dL_dKmm': dL_dKmm,\n 'dL_dKdiag':dL_dpsi0,\n 'dL_dKnm':dL_dpsi1,\n 'dL_dthetaL':dL_dthetaL}\n\n return post, logL, grad_dict", "response": "In the first phase of inference compute log - likelihood and log - likelihood inv for each entry in the output set."} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nplot the latent space X in 1D.", "response": "def plot(parameterized, fignum=None, ax=None, colors=None, figsize=(12, 6)):\n \"\"\"\n Plot latent space X in 1D:\n\n - if fig is given, create input_dim subplots in fig and plot in these\n - if ax is given plot input_dim 1D latent space plots of X into each `axis`\n - if neither fig nor ax is given create a figure with fignum and plot in there\n\n colors:\n colors of different latent space dimensions input_dim\n\n \"\"\"\n if ax is None:\n fig = pb.figure(num=fignum, figsize=figsize)\n if colors is None:\n from ..Tango import mediumList\n from itertools import cycle\n colors = cycle(mediumList)\n pb.clf()\n else:\n colors = iter(colors)\n lines = []\n fills = []\n bg_lines = []\n means, variances = parameterized.mean.values, parameterized.variance.values\n x = np.arange(means.shape[0])\n for i in range(means.shape[1]):\n if ax is None:\n a = fig.add_subplot(means.shape[1], 1, i + 1)\n elif isinstance(ax, (tuple, list)):\n a = ax[i]\n else:\n raise ValueError(\"Need one ax per latent dimension input_dim\")\n bg_lines.append(a.plot(means, c='k', alpha=.3))\n lines.extend(a.plot(x, means.T[i], c=next(colors), label=r\"$\\mathbf{{X_{{{}}}}}$\".format(i)))\n fills.append(a.fill_between(x,\n means.T[i] - 2 * np.sqrt(variances.T[i]),\n means.T[i] + 2 * np.sqrt(variances.T[i]),\n facecolor=lines[-1].get_color(),\n alpha=.3))\n a.legend(borderaxespad=0.)\n a.set_xlim(x.min(), x.max())\n if i < means.shape[1] - 1:\n a.set_xticklabels('')\n pb.draw()\n a.figure.tight_layout(h_pad=.01) # , rect=(0, 0, 1, .95))\n return dict(lines=lines, fills=fills, bg_lines=bg_lines)"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef plot_SpikeSlab(parameterized, fignum=None, ax=None, colors=None, side_by_side=True):\n if ax is None:\n if side_by_side:\n fig = pb.figure(num=fignum, figsize=(16, min(12, (2 * parameterized.mean.shape[1]))))\n else:\n fig = pb.figure(num=fignum, figsize=(8, min(12, (2 * parameterized.mean.shape[1]))))\n if colors is None:\n from ..Tango import mediumList\n from itertools import cycle\n colors = cycle(mediumList)\n pb.clf()\n else:\n colors = iter(colors)\n plots = []\n means, variances, gamma = parameterized.mean, parameterized.variance, parameterized.binary_prob\n x = np.arange(means.shape[0])\n for i in range(means.shape[1]):\n if side_by_side:\n sub1 = (means.shape[1],2,2*i+1)\n sub2 = (means.shape[1],2,2*i+2)\n else:\n sub1 = (means.shape[1]*2,1,2*i+1)\n sub2 = (means.shape[1]*2,1,2*i+2)\n\n # mean and variance plot\n a = fig.add_subplot(*sub1)\n a.plot(means, c='k', alpha=.3)\n plots.extend(a.plot(x, means.T[i], c=next(colors), label=r\"$\\mathbf{{X_{{{}}}}}$\".format(i)))\n a.fill_between(x,\n means.T[i] - 2 * np.sqrt(variances.T[i]),\n means.T[i] + 2 * np.sqrt(variances.T[i]),\n facecolor=plots[-1].get_color(),\n alpha=.3)\n a.legend(borderaxespad=0.)\n a.set_xlim(x.min(), x.max())\n if i < means.shape[1] - 1:\n a.set_xticklabels('')\n # binary prob plot\n a = fig.add_subplot(*sub2)\n a.bar(x,gamma[:,i],bottom=0.,linewidth=1.,width=1.0,align='center')\n a.set_xlim(x.min(), x.max())\n a.set_ylim([0.,1.])\n pb.draw()\n fig.tight_layout(h_pad=.01) # , rect=(0, 0, 1, .95))\n return fig", "response": "Plots the latent space X in 1D."} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nplots the mapping of the posterior functions and the posterior data.", "response": "def plot_mapping(self, plot_limits=None, which_data='all', which_parts='all', resolution=None, levels=20, samples=0, fignum=None, ax=None, fixed_inputs=[], linecol=Tango.colorsHex['darkBlue']):\n \"\"\"\n Plots the mapping associated with the model.\n - In one dimension, the function is plotted.\n - In two dimsensions, a contour-plot shows the function\n - In higher dimensions, we've not implemented this yet !TODO!\n\n Can plot only part of the data and part of the posterior functions\n using which_data and which_functions\n\n :param plot_limits: The limits of the plot. If 1D [xmin,xmax], if 2D [[xmin,ymin],[xmax,ymax]]. Defaluts to data limits\n :type plot_limits: np.array\n :param which_data: which if the training data to plot (default all)\n :type which_data: 'all' or a slice object to slice self.X, self.Y\n :param which_parts: which of the kernel functions to plot (additively)\n :type which_parts: 'all', or list of bools\n :param resolution: the number of intervals to sample the GP on. Defaults to 200 in 1D and 50 (a 50x50 grid) in 2D\n :type resolution: int\n :param levels: number of levels to plot in a contour plot.\n :type levels: int\n :param samples: the number of a posteriori samples to plot\n :type samples: int\n :param fignum: figure to plot on.\n :type fignum: figure number\n :param ax: axes to plot on.\n :type ax: axes handle\n :param fixed_inputs: a list of tuple [(i,v), (i,v)...], specifying that input index i should be set to value v.\n :type fixed_inputs: a list of tuples\n :param linecol: color of line to plot.\n :type linecol:\n :param levels: for 2D plotting, the number of contour levels to use is ax is None, create a new figure\n\n \"\"\"\n # TODO include samples\n if which_data == 'all':\n which_data = slice(None)\n\n if ax is None:\n fig = pb.figure(num=fignum)\n ax = fig.add_subplot(111)\n\n plotdims = self.input_dim - len(fixed_inputs)\n from ..gpy_plot.plot_util import x_frame1D, x_frame2D\n\n if plotdims == 1:\n\n Xu = self.X * self._Xscale + self._Xoffset # NOTE self.X are the normalized values now\n\n fixed_dims = np.array([i for i,v in fixed_inputs])\n freedim = np.setdiff1d(np.arange(self.input_dim),fixed_dims)\n\n Xnew, xmin, xmax = x_frame1D(Xu[:,freedim], plot_limits=plot_limits)\n Xgrid = np.empty((Xnew.shape[0],self.input_dim))\n Xgrid[:,freedim] = Xnew\n for i,v in fixed_inputs:\n Xgrid[:,i] = v\n\n f = self.predict(Xgrid, which_parts=which_parts)\n for d in range(y.shape[1]):\n ax.plot(Xnew, f[:, d], edgecol=linecol)\n\n elif self.X.shape[1] == 2:\n resolution = resolution or 50\n Xnew, _, _, xmin, xmax = x_frame2D(self.X, plot_limits, resolution)\n x, y = np.linspace(xmin[0], xmax[0], resolution), np.linspace(xmin[1], xmax[1], resolution)\n f = self.predict(Xnew, which_parts=which_parts)\n m = m.reshape(resolution, resolution).T\n ax.contour(x, y, f, levels, vmin=m.min(), vmax=m.max(), cmap=pb.cm.jet) # @UndefinedVariable\n ax.set_xlim(xmin[0], xmax[0])\n ax.set_ylim(xmin[1], xmax[1])\n\n else:\n raise NotImplementedError(\"Cannot define a frame with more than two input dimensions\")"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _raw_predict(self, Xnew=None, Ynew=None, filteronly=False, p_balance=False, **kw):\n\n # Set defaults\n if Ynew is None:\n Ynew = self.Y\n\n # Make a single matrix containing training and testing points\n if Xnew is not None:\n X = np.vstack((self.X, Xnew))\n Y = np.vstack((Ynew, np.nan*np.zeros(Xnew.shape)))\n predict_only_training = False\n else:\n X = self.X\n Y = Ynew\n predict_only_training = True\n\n # Sort the matrix (save the order)\n _, return_index, return_inverse = np.unique(X,True,True)\n X = X[return_index] # TODO they are not used\n Y = Y[return_index]\n\n # Get the model matrices from the kernel\n (F,L,Qc,H,P_inf, P0, dF,dQc,dP_inf,dP0) = self.kern.sde()\n state_dim = F.shape[0]\n \n # Balancing\n if (p_balance==True):\n (F,L,Qc,H,P_inf,P0, dF,dQc,dP_inf,dP0) = ssm.balance_ss_model(F,L,Qc,H,P_inf,P0, dF,dQc,dP_inf, dP0)\n print(\"SSM _raw_predict balancing!\")\n \n #Y = self.Y[:, 0,0]\n # Run the Kalman filter\n #import pdb; pdb.set_trace()\n kalman_filter_type = self.kalman_filter_type\n\n (M, P, log_likelihood,\n grad_log_likelihood,SmootherMatrObject) = ssm.ContDescrStateSpace.cont_discr_kalman_filter(\n F,L,Qc,H,float(self.Gaussian_noise.variance),P_inf,X,Y,m_init=None,\n P_init=P0, p_kalman_filter_type = kalman_filter_type,\n calc_log_likelihood=False,\n calc_grad_log_likelihood=False)\n\n# (filter_means, filter_covs, log_likelihood,\n# grad_log_likelihood,SmootherMatrObject) = ssm.ContDescrStateSpace.cont_discr_kalman_filter(F,L,Qc,H,\n# float(self.Gaussian_noise.variance),P_inf,self.X,self.Y,m_init=None,\n# P_init=P0, p_kalman_filter_type = kalman_filter_type, calc_log_likelihood=True,\n# calc_grad_log_likelihood=True,\n# grad_params_no=grad_params_no,\n# grad_calc_params=grad_calc_params)\n\n # Run the Rauch-Tung-Striebel smoother\n if not filteronly:\n (M, P) = ssm.ContDescrStateSpace.cont_discr_rts_smoother(state_dim, M, P,\n p_dynamic_callables=SmootherMatrObject, X=X, F=F,L=L,Qc=Qc)\n\n # remove initial values\n M = M[1:,:,:]\n P = P[1:,:,:]\n\n # Put the data back in the original order\n M = M[return_inverse,:,:]\n P = P[return_inverse,:,:]\n\n # Only return the values for Xnew\n if not predict_only_training:\n M = M[self.num_data:,:,:]\n P = P[self.num_data:,:,:]\n\n # Calculate the mean and variance\n # after einsum m has dimension in 3D (sample_num, dim_no,time_series_no)\n m = np.einsum('ijl,kj', M, H)# np.dot(M,H.T)\n m.shape = (m.shape[0], m.shape[1]) # remove the third dimension\n\n V = np.einsum('ij,ajk,kl', H, P, H.T)\n\n V.shape = (V.shape[0], V.shape[1]) # remove the third dimension\n\n # Return the posterior of the state\n return (m, V)", "response": "This function performs the actual prediction for new time points."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\npredicting the state of a set of state items.", "response": "def predict(self, Xnew=None, filteronly=False, include_likelihood=True, balance=None, **kw):\n \"\"\"\n Inputs:\n ------------------\n \n balance: bool\n Whether to balance or not the model as a whole\n \n \"\"\"\n \n if balance is None:\n p_balance = self.balance\n else:\n p_balance = balance\n \n # Run the Kalman filter to get the state\n (m, V) = self._raw_predict(Xnew,filteronly=filteronly, p_balance=p_balance)\n\n # Add the noise variance to the state variance\n if include_likelihood:\n V += float(self.likelihood.variance)\n\n # Lower and upper bounds\n #lower = m - 2*np.sqrt(V)\n #upper = m + 2*np.sqrt(V)\n\n # Return mean and variance\n return m, V"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef predict_quantiles(self, Xnew=None, quantiles=(2.5, 97.5), balance=None, **kw):\n if balance is None:\n p_balance = self.balance\n else:\n p_balance = balance\n \n \n mu, var = self._raw_predict(Xnew, p_balance=p_balance)\n #import pdb; pdb.set_trace()\n return [stats.norm.ppf(q/100.)*np.sqrt(var + float(self.Gaussian_noise.variance)) + mu for q in quantiles]", "response": "Predicts the quantiles of the current object."} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nplay a data set using the data_show object given.", "response": "def data_play(Y, visualizer, frame_rate=30):\n \"\"\"Play a data set using the data_show object given.\n\n :Y: the data set to be visualized.\n :param visualizer: the data show objectwhether to display during optimisation\n :type visualizer: data_show\n\n Example usage:\n\n This example loads in the CMU mocap database (http://mocap.cs.cmu.edu) subject number 35 motion number 01. It then plays it using the mocap_show visualize object.\n\n .. code-block:: python\n\n data = GPy.util.datasets.cmu_mocap(subject='35', train_motions=['01'])\n Y = data['Y']\n Y[:, 0:3] = 0. # Make figure walk in place\n visualize = GPy.util.visualize.skeleton_show(Y[0, :], data['skel'])\n GPy.util.visualize.data_play(Y, visualize)\n\n \"\"\"\n\n\n for y in Y:\n visualizer.modify(y[None, :])\n time.sleep(1./float(frame_rate))"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nsetting up the axes with the right limits and scaling.", "response": "def initialize_axes(self, boundary=0.05):\n \"\"\"Set up the axes with the right limits and scaling.\"\"\"\n bs = [(self.vals[:, i].max()-self.vals[:, i].min())*boundary for i in range(3)]\n self.x_lim = np.array([self.vals[:, 0].min()-bs[0], self.vals[:, 0].max()+bs[0]])\n self.y_lim = np.array([self.vals[:, 1].min()-bs[1], self.vals[:, 1].max()+bs[1]])\n self.z_lim = np.array([self.vals[:, 2].min()-bs[2], self.vals[:, 2].max()+bs[2]])"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ntake a set of angles and converts them to the x y z coordinates in the internal prepresentation of the class ready for plotting.", "response": "def process_values(self):\n \"\"\"Takes a set of angles and converts them to the x,y,z coordinates in the internal prepresentation of the class, ready for plotting.\n\n :param vals: the values that are being modelled.\"\"\"\n\n if self.padding>0:\n channels = np.zeros((self.vals.shape[0], self.vals.shape[1]+self.padding))\n channels[:, 0:self.vals.shape[0]] = self.vals\n else:\n channels = self.vals\n vals_mat = self.skel.to_xyz(channels.flatten())\n self.vals = np.zeros_like(vals_mat)\n # Flip the Y and Z axes\n self.vals[:, 0] = vals_mat[:, 0].copy()\n self.vals[:, 1] = vals_mat[:, 2].copy()\n self.vals[:, 2] = vals_mat[:, 1].copy()"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ncreating a new canvas with the specified parameters", "response": "def new_canvas(self, figure=None, col=1, row=1, projection='2d', xlabel=None, ylabel=None, zlabel=None, title=None, xlim=None, ylim=None, zlim=None, **kwargs):\n \"\"\"\n Return a canvas, kwargupdate for your plotting library. \n\n if figure is not None, create a canvas in the figure\n at subplot position (col, row).\n \n This method does two things, it creates an empty canvas\n and updates the kwargs (deletes the unnecessary kwargs)\n for further usage in normal plotting.\n \n the kwargs are plotting library specific kwargs!\n\n :param {'2d'|'3d'} projection: The projection to use.\n\n E.g. in matplotlib this means it deletes references to ax, as\n plotting is done on the axis itself and is not a kwarg. \n\n :param xlabel: the label to put on the xaxis\n :param ylabel: the label to put on the yaxis\n :param zlabel: the label to put on the zaxis (if plotting in 3d)\n :param title: the title of the plot\n :param legend: if True, plot a legend, if int make legend rows in the legend\n :param (float, float) xlim: the limits for the xaxis\n :param (float, float) ylim: the limits for the yaxis\n :param (float, float) zlim: the limits for the zaxis (if plotting in 3d)\n \"\"\"\n raise NotImplementedError(\"Implement all plot functions in AbstractPlottingLibrary in order to use your own plotting library\")"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nplotting a line from for X Y and Z on the canvas.", "response": "def plot(self, cavas, X, Y, Z=None, color=None, label=None, **kwargs):\n \"\"\"\n Make a line plot from for Y on X (Y = f(X)) on the canvas.\n If Z is not None, plot in 3d!\n \n the kwargs are plotting library specific kwargs!\n \"\"\"\n raise NotImplementedError(\"Implement all plot functions in AbstractPlottingLibrary in order to use your own plotting library\")"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef surface(self, canvas, X, Y, Z, color=None, label=None, **kwargs):\n raise NotImplementedError(\"Implement all plot functions in AbstractPlottingLibrary in order to use your own plotting library\")", "response": "Plot a surface for the input X Y and Z."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nmaking a scatter plot between X and Y on the canvas given.", "response": "def scatter(self, canvas, X, Y, Z=None, color=None, vmin=None, vmax=None, label=None, **kwargs):\n \"\"\"\n Make a scatter plot between X and Y on the canvas given.\n \n the kwargs are plotting library specific kwargs!\n \n :param canvas: the plotting librarys specific canvas to plot on.\n :param array-like X: the inputs to plot.\n :param array-like Y: the outputs to plot.\n :param array-like Z: the Z level to plot (if plotting 3d).\n :param array-like c: the colorlevel for each point.\n :param float vmin: minimum colorscale\n :param float vmax: maximum colorscale\n :param kwargs: the specific kwargs for your plotting library\n \"\"\"\n raise NotImplementedError(\"Implement all plot functions in AbstractPlottingLibrary in order to use your own plotting library\")"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef barplot(self, canvas, x, height, width=0.8, bottom=0, color=None, label=None, **kwargs):\n raise NotImplementedError(\"Implement all plot functions in AbstractPlottingLibrary in order to use your own plotting library\")", "response": "Plot a vertical bar plot centered at x with height and width of bars."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nmake an errorbar along the xaxis for points at X Y.", "response": "def xerrorbar(self, canvas, X, Y, error, color=None, label=None, **kwargs):\n \"\"\"\n Make an errorbar along the xaxis for points at (X,Y) on the canvas.\n if error is two dimensional, the lower error is error[:,0] and\n the upper error is error[:,1]\n \n the kwargs are plotting library specific kwargs!\n \"\"\"\n raise NotImplementedError(\"Implement all plot functions in AbstractPlottingLibrary in order to use your own plotting library\")"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nshow the image stored in X on the canvas. The origin of the image show is (0,0), such that X[0,0] gets plotted at [0,0] of the image! the kwargs are plotting library specific kwargs!", "response": "def imshow(self, canvas, X, extent=None, label=None, vmin=None, vmax=None, **kwargs):\n \"\"\"\n Show the image stored in X on the canvas.\n \n The origin of the image show is (0,0), such that X[0,0] gets plotted at [0,0] of the image!\n \n the kwargs are plotting library specific kwargs!\n \"\"\"\n raise NotImplementedError(\"Implement all plot functions in AbstractPlottingLibrary in order to use your own plotting library\")"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef contour(self, canvas, X, Y, C, Z=None, color=None, label=None, **kwargs):\n raise NotImplementedError(\"Implement all plot functions in AbstractPlottingLibrary in order to use your own plotting library\")", "response": "Make a contour plot at X Y C with heights and colors stored in C on the canvas."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef fill_between(self, canvas, X, lower, upper, color=None, label=None, **kwargs):\n raise NotImplementedError(\"Implement all plot functions in AbstractPlottingLibrary in order to use your own plotting library\")", "response": "Fill along the xaxis between lower and upper"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nrandomizing the model. Make this draw from the prior if one exists, else draw from given random generator :param rand_gen: np random number generator which takes args and kwargs :param flaot loc: loc parameter for random number generator :param float scale: scale parameter for random number generator :param args, kwargs: will be passed through to random number generator", "response": "def randomize(self, rand_gen=None, *args, **kwargs):\n \"\"\"\n Randomize the model.\n Make this draw from the prior if one exists, else draw from given random generator\n\n :param rand_gen: np random number generator which takes args and kwargs\n :param flaot loc: loc parameter for random number generator\n :param float scale: scale parameter for random number generator\n :param args, kwargs: will be passed through to random number generator\n \"\"\"\n if rand_gen is None:\n rand_gen = np.random.normal\n # first take care of all parameters (from N(0,1))\n x = rand_gen(size=self._size_transformed(), *args, **kwargs)\n updates = self.update_model()\n self.update_model(False) # Switch off the updates\n self.optimizer_array = x # makes sure all of the tied parameters get the same init (since there's only one prior object...)\n # now draw from prior where possible\n x = self.param_array.copy()\n [np.put(x, ind, p.rvs(ind.size)) for p, ind in self.priors.items() if not p is None]\n unfixlist = np.ones((self.size,),dtype=np.bool)\n from paramz.transformations import __fixed__\n unfixlist[self.constraints[__fixed__]] = False\n self.param_array.flat[unfixlist] = x.view(np.ndarray).ravel()[unfixlist]\n self.update_model(updates)"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncalculating the K_xx term without the variance term", "response": "def calc_K_xx_wo_variance(self,X):\n \"\"\"Calculates K_xx without the variance term\"\"\"\n K_xx = np.ones([X.shape[0],X.shape[0]]) #ones now as a product occurs over each dimension\n for i,x in enumerate(X):\n for j,x2 in enumerate(X):\n for il,l in enumerate(self.lengthscale):\n idx = il*2 #each pair of input dimensions describe the limits on one actual dimension in the data\n K_xx[i,j] *= self.k_xx(x[idx],x2[idx],x[idx+1],x2[idx+1],l)\n return K_xx"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef checkgrad(self, target_param=None, verbose=False, step=1e-6, tolerance=1e-3, block_indices=None, plot=False):\n try:\n import numdifftools as nd\n except:\n raise ImportError(\"Don't have numdifftools package installed, it is not a GPy dependency as of yet, it is only used for hessian tests\")\n\n if target_param:\n raise NotImplementedError('Only basic functionality is provided with this gradchecker')\n\n #Repeat for each parameter, not the nicest but shouldn't be many cases where there are many\n #variables\n current_index = 0\n for name, shape in zip(self.names, self.shapes):\n current_size = numpy.prod(shape)\n x = self.optimizer_array.copy()\n #x = self._get_params_transformed().copy()\n x = x[current_index:current_index + current_size].reshape(shape)\n\n # Check gradients\n analytic_hess = self._ddf(x)\n if analytic_hess.shape[1] == 1:\n analytic_hess = numpy.diagflat(analytic_hess)\n\n #From the docs:\n #x0 : vector location\n #at which to differentiate fun\n #If x0 is an N x M array, then fun is assumed to be a function\n #of N*M variables., thus we must have it flat, not (N,1), but just (N,)\n #numeric_hess_partial = nd.Hessian(self._f, vectorized=False)\n numeric_hess_partial = nd.Jacobian(self._df, vectorized=False)\n #numeric_hess_partial = nd.Derivative(self._df, vectorized=True)\n numeric_hess = numeric_hess_partial(x)\n\n check_passed = self.checkgrad_block(analytic_hess, numeric_hess, verbose=verbose, step=step, tolerance=tolerance, block_indices=block_indices, plot=plot)\n current_index += current_size\n return check_passed", "response": "Overwrites checkgrad method to check the gradients of the parameters of the object."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef dKd_dVar(self, X, X2=None):\n\t\tr = self._scaled_dist(X, X2)\n\t\treturn self.dKdVar_of_r(r)", "response": "Derivative of Kernel function wrt variance applied on inputs X and X2."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncomputing derivative of kernel for dimension wrt lengthscale", "response": "def dKdLen_of_r(self, r, dimCheck, lengthscale):\n\t\t\"\"\"\n\t\tCompute derivative of kernel for dimension wrt lengthscale\n\t\tComputation of derivative changes when lengthscale corresponds to\n\t\tthe dimension of the kernel whose derivate is being computed. \n\t\t\"\"\"\n\t\tif (dimCheck == True):\n\t\t\treturn (self.variance**(float(1)/self.originalDimensions)) * np.exp(-0.5 * r**2) * (r**2) / (lengthscale**(float(1)/self.originalDimensions))\n\t\telse:\n\t\t\treturn (self.variance**(float(1)/self.originalDimensions)) * np.exp(-0.5 * r**2) / (lengthscale**(float(1)/self.originalDimensions))"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef from_dict(input_dict):\n\n import copy\n input_dict = copy.deepcopy(input_dict)\n likelihood_class = input_dict.pop('class')\n input_dict[\"name\"] = str(input_dict[\"name\"])\n name = input_dict.pop('name')\n import GPy\n likelihood_class = eval(likelihood_class)\n return likelihood_class._build_from_input_dict(likelihood_class, input_dict)", "response": "Instantiate an object of a derived class from a dictionary."} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncalculating the moments of the observation and returns the entry point for the entry point that matches the observation and the specified parameter.", "response": "def moments_match_ep(self,obs,tau,v,Y_metadata_i=None):\n \"\"\"\n Calculation of moments using quadrature\n\n :param obs: observed output\n :param tau: cavity distribution 1st natural parameter (precision)\n :param v: cavity distribution 2nd natural paramenter (mu*precision)\n \"\"\"\n #Compute first integral for zeroth moment.\n #NOTE constant np.sqrt(2*pi/tau) added at the end of the function\n mu = v/tau\n sigma2 = 1./tau\n #Lets do these for now based on the same idea as Gaussian quadrature\n # i.e. multiply anything by close to zero, and its zero.\n f_min = mu - 20*np.sqrt(sigma2)\n f_max = mu + 20*np.sqrt(sigma2)\n\n def int_1(f):\n return self.pdf(f, obs, Y_metadata=Y_metadata_i)*np.exp(-0.5*tau*np.square(mu-f))\n z_scaled, accuracy = quad(int_1, f_min, f_max)\n\n #Compute second integral for first moment\n def int_2(f):\n return f*self.pdf(f, obs, Y_metadata=Y_metadata_i)*np.exp(-0.5*tau*np.square(mu-f))\n mean, accuracy = quad(int_2, f_min, f_max)\n mean /= z_scaled\n\n #Compute integral for variance\n def int_3(f):\n return (f**2)*self.pdf(f, obs, Y_metadata=Y_metadata_i)*np.exp(-0.5*tau*np.square(mu-f))\n Ef2, accuracy = quad(int_3, f_min, f_max)\n Ef2 /= z_scaled\n variance = Ef2 - mean**2\n\n #Add constant to the zeroth moment\n #NOTE: this constant is not needed in the other moments because it cancells out.\n z = z_scaled/np.sqrt(2*np.pi/tau)\n\n return z, mean, variance"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef variational_expectations(self, Y, m, v, gh_points=None, Y_metadata=None):\n\n if gh_points is None:\n gh_x, gh_w = self._gh_points()\n else:\n gh_x, gh_w = gh_points\n\n shape = m.shape\n m,v,Y = m.flatten(), v.flatten(), Y.flatten()\n\n #make a grid of points\n X = gh_x[None,:]*np.sqrt(2.*v[:,None]) + m[:,None]\n\n #evaluate the likelhood for the grid. First ax indexes the data (and mu, var) and the second indexes the grid.\n # broadcast needs to be handled carefully.\n logp = self.logpdf(X,Y[:,None], Y_metadata=Y_metadata)\n dlogp_dx = self.dlogpdf_df(X, Y[:,None], Y_metadata=Y_metadata)\n d2logp_dx2 = self.d2logpdf_df2(X, Y[:,None], Y_metadata=Y_metadata)\n\n #clipping for numerical stability\n #logp = np.clip(logp,-1e9,1e9)\n #dlogp_dx = np.clip(dlogp_dx,-1e9,1e9)\n #d2logp_dx2 = np.clip(d2logp_dx2,-1e9,1e9)\n\n #average over the gird to get derivatives of the Gaussian's parameters\n #division by pi comes from fact that for each quadrature we need to scale by 1/sqrt(pi)\n F = np.dot(logp, gh_w)/np.sqrt(np.pi)\n dF_dm = np.dot(dlogp_dx, gh_w)/np.sqrt(np.pi)\n dF_dv = np.dot(d2logp_dx2, gh_w)/np.sqrt(np.pi)\n dF_dv /= 2.\n\n if np.any(np.isnan(dF_dv)) or np.any(np.isinf(dF_dv)):\n stop\n if np.any(np.isnan(dF_dm)) or np.any(np.isinf(dF_dm)):\n stop\n\n if self.size:\n dF_dtheta = self.dlogpdf_dtheta(X, Y[:,None], Y_metadata=Y_metadata) # Ntheta x (orig size) x N_{quad_points}\n dF_dtheta = np.dot(dF_dtheta, gh_w)/np.sqrt(np.pi)\n dF_dtheta = dF_dtheta.reshape(self.size, shape[0], shape[1])\n else:\n dF_dtheta = None # Not yet implemented\n return F.reshape(*shape), dF_dm.reshape(*shape), dF_dv.reshape(*shape), dF_dtheta", "response": "Compute variational expectations for a set of data points Y m and v."} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncalculates the predictive mean of the posterior with respect to variance.", "response": "def predictive_mean(self, mu, variance, Y_metadata=None):\n \"\"\"\n Quadrature calculation of the predictive mean: E(Y_star|Y) = E( E(Y_star|f_star, Y) )\n\n :param mu: mean of posterior\n :param sigma: standard deviation of posterior\n\n \"\"\"\n #conditional_mean: the edpected value of y given some f, under this likelihood\n fmin = -np.inf\n fmax = np.inf\n def int_mean(f,m,v):\n exponent = -(0.5/v)*np.square(f - m)\n #If exponent is under -30 then exp(exponent) will be very small, so don't exp it!)\n #If p is zero then conditional_mean will overflow\n assert v.all() > 0\n p = safe_exp(exponent)\n\n #If p is zero then conditional_variance will overflow\n if p < 1e-10:\n return 0.\n else:\n return self.conditional_mean(f)*p\n scaled_mean = [quad(int_mean, fmin, fmax,args=(mj,s2j))[0] for mj,s2j in zip(mu,variance)]\n mean = np.array(scaled_mean)[:,None] / np.sqrt(2*np.pi*(variance))\n return mean"} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nevaluates the link function link then computes the likelihood of the link function link to the inversely transposed link function and returns the likelihood of the inversely transposed link function.", "response": "def pdf(self, f, y, Y_metadata=None):\n \"\"\"\n Evaluates the link function link(f) then computes the likelihood (pdf) using it\n\n .. math:\n p(y|\\\\lambda(f))\n\n :param f: latent variables f\n :type f: Nx1 array\n :param y: data\n :type y: Nx1 array\n :param Y_metadata: Y_metadata which is not used in student t distribution - not used\n :returns: likelihood evaluated for this point\n :rtype: float\n \"\"\"\n if isinstance(self.gp_link, link_functions.Identity):\n return self.pdf_link(f, y, Y_metadata=Y_metadata)\n else:\n inv_link_f = self.gp_link.transf(f)\n return self.pdf_link(inv_link_f, y, Y_metadata=Y_metadata)"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef logpdf_sum(self, f, y, Y_metadata=None):\n return np.sum(self.logpdf(f, y, Y_metadata=Y_metadata))", "response": "Compute the log PDF for a given frequency and a given set of data points."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef logpdf(self, f, y, Y_metadata=None):\n if isinstance(self.gp_link, link_functions.Identity):\n return self.logpdf_link(f, y, Y_metadata=Y_metadata)\n else:\n inv_link_f = self.gp_link.transf(f)\n return self.logpdf_link(inv_link_f, y, Y_metadata=Y_metadata)", "response": "Evaluates the log pdf of a latent variable f using the link function link and computes the log likelihood of the link function link using the identity link."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef dlogpdf_df(self, f, y, Y_metadata=None):\n if isinstance(self.gp_link, link_functions.Identity):\n return self.dlogpdf_dlink(f, y, Y_metadata=Y_metadata)\n else:\n inv_link_f = self.gp_link.transf(f)\n dlogpdf_dlink = self.dlogpdf_dlink(inv_link_f, y, Y_metadata=Y_metadata)\n dlink_df = self.gp_link.dtransf_df(f)\n return chain_1(dlogpdf_dlink, dlink_df)", "response": "Evaluates the derivative of log likelihood using the link function link and the derivative of log likelihood using it\n "} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef d2logpdf_df2(self, f, y, Y_metadata=None):\n if isinstance(self.gp_link, link_functions.Identity):\n d2logpdf_df2 = self.d2logpdf_dlink2(f, y, Y_metadata=Y_metadata)\n else:\n inv_link_f = self.gp_link.transf(f)\n d2logpdf_dlink2 = self.d2logpdf_dlink2(inv_link_f, y, Y_metadata=Y_metadata)\n dlink_df = self.gp_link.dtransf_df(f)\n dlogpdf_dlink = self.dlogpdf_dlink(inv_link_f, y, Y_metadata=Y_metadata)\n d2link_df2 = self.gp_link.d2transf_df2(f)\n d2logpdf_df2 = chain_2(d2logpdf_dlink2, dlink_df, dlogpdf_dlink, d2link_df2)\n return d2logpdf_df2", "response": "Evaluates the second derivative of log likelihood using it"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nevaluate the third derivative of log likelihood using it", "response": "def d3logpdf_df3(self, f, y, Y_metadata=None):\n \"\"\"\n Evaluates the link function link(f) then computes the third derivative of log likelihood using it\n Uses the Faa di Bruno's formula for the chain rule\n\n .. math::\n \\\\frac{d^{3}\\\\log p(y|\\\\lambda(f))}{df^{3}} = \\\\frac{d^{3}\\\\log p(y|\\\\lambda(f)}{d\\\\lambda(f)^{3}}\\\\left(\\\\frac{d\\\\lambda(f)}{df}\\\\right)^{3} + 3\\\\frac{d^{2}\\\\log p(y|\\\\lambda(f)}{d\\\\lambda(f)^{2}}\\\\frac{d\\\\lambda(f)}{df}\\\\frac{d^{2}\\\\lambda(f)}{df^{2}} + \\\\frac{d\\\\log p(y|\\\\lambda(f)}{d\\\\lambda(f)}\\\\frac{d^{3}\\\\lambda(f)}{df^{3}}\n\n :param f: latent variables f\n :type f: Nx1 array\n :param y: data\n :type y: Nx1 array\n :param Y_metadata: Y_metadata which is not used in student t distribution - not used\n :returns: third derivative of log likelihood evaluated for this point\n :rtype: float\n \"\"\"\n if isinstance(self.gp_link, link_functions.Identity):\n d3logpdf_df3 = self.d3logpdf_dlink3(f, y, Y_metadata=Y_metadata)\n else:\n inv_link_f = self.gp_link.transf(f)\n d3logpdf_dlink3 = self.d3logpdf_dlink3(inv_link_f, y, Y_metadata=Y_metadata)\n dlink_df = self.gp_link.dtransf_df(f)\n d2logpdf_dlink2 = self.d2logpdf_dlink2(inv_link_f, y, Y_metadata=Y_metadata)\n d2link_df2 = self.gp_link.d2transf_df2(f)\n dlogpdf_dlink = self.dlogpdf_dlink(inv_link_f, y, Y_metadata=Y_metadata)\n d3link_df3 = self.gp_link.d3transf_df3(f)\n d3logpdf_df3 = chain_3(d3logpdf_dlink3, dlink_df, d2logpdf_dlink2, d2link_df2, dlogpdf_dlink, d3link_df3)\n return d3logpdf_df3"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef dlogpdf_df_dtheta(self, f, y, Y_metadata=None):\n if self.size > 0:\n if self.not_block_really:\n raise NotImplementedError(\"Need to make a decorator for this!\")\n if isinstance(self.gp_link, link_functions.Identity):\n return self.dlogpdf_dlink_dtheta(f, y, Y_metadata=Y_metadata)\n else:\n inv_link_f = self.gp_link.transf(f)\n dlink_df = self.gp_link.dtransf_df(f)\n dlogpdf_dlink_dtheta = self.dlogpdf_dlink_dtheta(inv_link_f, y, Y_metadata=Y_metadata)\n\n dlogpdf_df_dtheta = np.zeros((self.size, f.shape[0], f.shape[1]))\n #Chain each parameter of hte likelihood seperately\n for p in range(self.size):\n dlogpdf_df_dtheta[p, :, :] = chain_1(dlogpdf_dlink_dtheta[p,:,:], dlink_df)\n return dlogpdf_df_dtheta\n #return chain_1(dlogpdf_dlink_dtheta, dlink_df)\n else:\n # There are no parameters so return an empty array for derivatives\n return np.zeros((0, f.shape[0], f.shape[1]))", "response": "Function to compute the derivative of the log - likelihood of a single parameter of the class."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef predictive_values(self, mu, var, full_cov=False, Y_metadata=None):\n try:\n pred_mean = self.predictive_mean(mu, var, Y_metadata=Y_metadata)\n pred_var = self.predictive_variance(mu, var, pred_mean, Y_metadata=Y_metadata)\n except NotImplementedError:\n print(\"Finding predictive mean and variance via sampling rather than quadrature\")\n Nf_samp = 300\n Ny_samp = 1\n s = np.random.randn(mu.shape[0], Nf_samp)*np.sqrt(var) + mu\n ss_y = self.samples(s, Y_metadata, samples=Ny_samp)\n pred_mean = np.mean(ss_y, axis=1)[:, None]\n pred_var = np.var(ss_y, axis=1)[:, None]\n\n return pred_mean, pred_var", "response": "Compute mean and variance of the predictive distribution."} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncompute psi - statistics for ss - linear kernels.", "response": "def psicomputations(variance, Z, variational_posterior, return_psi2_n=False):\n \"\"\"\n Compute psi-statistics for ss-linear kernel\n \"\"\"\n # here are the \"statistics\" for psi0, psi1 and psi2\n # Produced intermediate results:\n # psi0 N\n # psi1 NxM\n # psi2 MxM\n mu = variational_posterior.mean\n S = variational_posterior.variance\n gamma = variational_posterior.binary_prob\n\n psi0 = (gamma*(np.square(mu)+S)*variance).sum(axis=-1)\n psi1 = np.inner(variance*gamma*mu,Z)\n psi2 = np.inner(np.square(variance)*(gamma*((1-gamma)*np.square(mu)+S)).sum(axis=0)*Z,Z)+tdot(psi1.T)\n\n return psi0, psi1, psi2"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncompute the psi2 statistics for psi1 and psi2.", "response": "def _psi2computations(dL_dpsi2, variance, Z, mu, S, gamma):\n \"\"\"\n Z - MxQ\n mu - NxQ\n S - NxQ\n gamma - NxQ\n \"\"\"\n # here are the \"statistics\" for psi1 and psi2\n # Produced intermediate results:\n # _psi2_dvariance Q\n # _psi2_dZ MxQ\n # _psi2_dgamma NxQ\n # _psi2_dmu NxQ\n # _psi2_dS NxQ\n \n mu2 = np.square(mu)\n gamma2 = np.square(gamma)\n variance2 = np.square(variance)\n mu2S = mu2+S # NxQ\n gvm = gamma*mu*variance\n common_sum = gvm.dot(Z.T)\n Z_expect = (np.dot(dL_dpsi2,Z)*Z).sum(axis=0)\n Z_expect_var2 = Z_expect*variance2\n dL_dpsi2T = dL_dpsi2+dL_dpsi2.T\n common_expect = common_sum.dot(dL_dpsi2T).dot(Z)\n Z2_expect = common_sum.dot(dL_dpsi2T)\n Z1_expect = dL_dpsi2T.dot(Z)\n \n dL_dvar = variance*Z_expect*2.*(gamma*mu2S-gamma2*mu2).sum(axis=0)+(common_expect*gamma*mu).sum(axis=0)\n \n dL_dgamma = Z_expect_var2*(mu2S-2.*gamma*mu2)+common_expect*mu*variance\n \n dL_dmu = Z_expect_var2*mu*2.*(gamma-gamma2) + common_expect*gamma*variance\n\n dL_dS = gamma*Z_expect_var2\n \n dL_dZ = (gamma*(mu2S-gamma*mu2)).sum(axis=0)*variance2*Z1_expect+ Z2_expect.T.dot(gamma*mu)*variance\n\n return dL_dvar, dL_dgamma, dL_dmu, dL_dS, dL_dZ"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ngetting the mean of the species", "response": "def mean(self):\n \"\"\"\n Posterior mean\n $$\n K_{xx}v\n v := \\texttt{Woodbury vector}\n $$\n \"\"\"\n if self._mean is None:\n self._mean = np.dot(self._K, self.woodbury_vector)\n return self._mean"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef covariance_between_points(self, kern, X, X1, X2):\n # ndim == 3 is a model for missing data\n if self.woodbury_chol.ndim != 2:\n raise RuntimeError(\"This method does not support posterior for missing data models\")\n\n Kx1 = kern.K(X, X1)\n Kx2 = kern.K(X, X2)\n K12 = kern.K(X1, X2)\n\n tmp1 = dtrtrs(self.woodbury_chol, Kx1)[0]\n tmp2 = dtrtrs(self.woodbury_chol, Kx2)[0]\n var = K12 - tmp1.T.dot(tmp2)\n\n return var", "response": "Computes the posterior covariance between two points."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef precision(self):\n if self._precision is None:\n cov = np.atleast_3d(self.covariance)\n self._precision = np.zeros(cov.shape) # if one covariance per dimension\n for p in range(cov.shape[-1]):\n self._precision[:, :, p] = pdinv(cov[:, :, p])[0]\n return self._precision", "response": "Inverse of posterior covariance\n "} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncompute the lower triangular Cholesky decomposition of the Woodbury matrix.", "response": "def woodbury_chol(self):\n \"\"\"\n return $L_{W}$ where L is the lower triangular Cholesky decomposition of the Woodbury matrix\n $$\n L_{W}L_{W}^{\\top} = W^{-1}\n W^{-1} := \\texttt{Woodbury inv}\n $$\n \"\"\"\n if self._woodbury_chol is None:\n # compute woodbury chol from\n if self._woodbury_inv is not None:\n winv = np.atleast_3d(self._woodbury_inv)\n self._woodbury_chol = np.zeros(winv.shape)\n for p in range(winv.shape[-1]):\n self._woodbury_chol[:, :, p] = pdinv(winv[:, :, p])[2]\n # Li = jitchol(self._woodbury_inv)\n # self._woodbury_chol, _ = dtrtri(Li)\n # W, _, _, _, = pdinv(self._woodbury_inv)\n # symmetrify(W)\n # self._woodbury_chol = jitchol(W)\n # try computing woodbury chol from cov\n elif self._covariance is not None:\n raise NotImplementedError(\"TODO: check code here\")\n B = self._K - self._covariance\n tmp, _ = dpotrs(self.K_chol, B)\n self._woodbury_inv, _ = dpotrs(self.K_chol, tmp.T)\n _, _, self._woodbury_chol, _ = pdinv(self._woodbury_inv)\n else:\n raise ValueError(\"insufficient information to compute posterior\")\n return self._woodbury_chol"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning the inverse of the woodbury matrix in the gaussian likelihood case.", "response": "def woodbury_inv(self):\n \"\"\"\n The inverse of the woodbury matrix, in the gaussian likelihood case it is defined as\n $$\n (K_{xx} + \\Sigma_{xx})^{-1}\n \\Sigma_{xx} := \\texttt{Likelihood.variance / Approximate likelihood covariance}\n $$\n \"\"\"\n if self._woodbury_inv is None:\n if self._woodbury_chol is not None:\n self._woodbury_inv, _ = dpotri(self._woodbury_chol, lower=1)\n # self._woodbury_inv, _ = dpotrs(self.woodbury_chol, np.eye(self.woodbury_chol.shape[0]), lower=1)\n symmetrify(self._woodbury_inv)\n elif self._covariance is not None:\n B = np.atleast_3d(self._K) - np.atleast_3d(self._covariance)\n self._woodbury_inv = np.empty_like(B)\n for i in range(B.shape[-1]):\n tmp, _ = dpotrs(self.K_chol, B[:, :, i])\n self._woodbury_inv[:, :, i], _ = dpotrs(self.K_chol, tmp.T)\n return self._woodbury_inv"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef K_chol(self):\n if self._K_chol is None:\n self._K_chol = jitchol(self._K)\n return self._K_chol", "response": "Cholesky of the prior covariance of the prior covariance"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef Kdiag(self, X, target):\n np.add(target, self.variance, target)", "response": "Compute the diagonal of the covariance matrix for X."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _param_grad_helper(self, dL_dK, X, X2, target):\n self._K_computations(X, X2)\n self._dK_computations(dL_dK)\n if X2==None:\n gmapping = self.mapping.df_dtheta(2*self._dL_dl[:, None], X)\n else:\n gmapping = self.mapping.df_dtheta(self._dL_dl[:, None], X)\n gmapping += self.mapping.df_dtheta(self._dL_dl_two[:, None], X2)\n\n target+= np.hstack([(dL_dK*self._K_dvar).sum(), gmapping])", "response": "Derivative of the covariance with respect to the parameters."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef gradients_X(self, dL_dK, X, X2, target):\n # First account for gradients arising from presence of X in exponent.\n self._K_computations(X, X2)\n if X2 is None:\n _K_dist = 2*(X[:, None, :] - X[None, :, :])\n else:\n _K_dist = X[:, None, :] - X2[None, :, :] # don't cache this in _K_co\n gradients_X = (-2.*self.variance)*np.transpose((self._K_dvar/self._w2)[:, :, None]*_K_dist, (1, 0, 2))\n target += np.sum(gradients_X*dL_dK.T[:, :, None], 0)\n # Now account for gradients arising from presence of X in lengthscale.\n self._dK_computations(dL_dK)\n if X2 is None:\n target += 2.*self.mapping.df_dX(self._dL_dl[:, None], X)\n else:\n target += self.mapping.df_dX(self._dL_dl[:, None], X)", "response": "Derivative of the covariance matrix with respect to X."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _K_computations(self, X, X2=None):\n self._lengthscales=self.mapping.f(X)\n self._lengthscales2=np.square(self._lengthscales)\n if X2==None:\n self._lengthscales_two = self._lengthscales\n self._lengthscales_two2 = self._lengthscales2\n Xsquare = np.square(X).sum(1)\n self._K_dist2 = -2.*tdot(X) + Xsquare[:, None] + Xsquare[None, :]\n else:\n self._lengthscales_two = self.mapping.f(X2)\n self._lengthscales_two2 = np.square(self._lengthscales_two)\n self._K_dist2 = -2.*np.dot(X, X2.T) + np.square(X).sum(1)[:, None] + np.square(X2).sum(1)[None, :]\n self._w2 = self._lengthscales2 + self._lengthscales_two2.T\n prod_length = self._lengthscales*self._lengthscales_two.T\n self._K_exponential = np.exp(-self._K_dist2/self._w2)\n self._K_dvar = np.sign(prod_length)*(2*np.abs(prod_length)/self._w2)**(self.input_dim/2.)*np.exp(-self._K_dist2/self._w2)", "response": "Pre - computations for the covariance function. Here self. _dK_dvar and self. _K_dist2 are updated. Here self. _K_exponential and self. _K_dist2 are updated."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ngetting the sensitivity of the current parameter kernel.", "response": "def input_sensitivity(self, summarize=True):\n \"\"\"\n If summize is true, we want to get the summerized view of the sensitivities,\n otherwise put everything into an array with shape (#kernels, input_dim)\n in the order of appearance of the kernels in the parameterized object.\n \"\"\"\n if not summarize:\n num_params = [0]\n parts = []\n def sum_params(x):\n if (not isinstance(x, CombinationKernel)) and isinstance(x, Kern):\n num_params[0] += 1\n parts.append(x)\n self.traverse(sum_params)\n i_s = np.zeros((num_params[0], self.input_dim))\n from operator import setitem\n [setitem(i_s, (i, k._all_dims_active), k.input_sensitivity(summarize)) for i, k in enumerate(parts)]\n return i_s\n else:\n raise NotImplementedError(\"Choose the kernel you want to get the sensitivity for. You need to override the default behaviour for getting the input sensitivity to be able to get the input sensitivity. For sum kernel it is the sum of all sensitivities, TODO: product kernel? Other kernels?, also TODO: shall we return all the sensitivities here in the combination kernel? So we can combine them however we want? This could lead to just plot all the sensitivities here...\")"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _simulate_matern(D1, D2, D3, N, num_inducing, plot_sim=False):\n Q_signal = 4\n import GPy\n import numpy as np\n np.random.seed(3000)\n\n k = GPy.kern.Matern32(Q_signal, 1., lengthscale=(np.random.uniform(1, 6, Q_signal)), ARD=1)\n for i in range(Q_signal):\n k += GPy.kern.PeriodicExponential(1, variance=1., active_dims=[i], period=3., lower=-2, upper=6)\n t = np.c_[[np.linspace(-1, 5, N) for _ in range(Q_signal)]].T\n K = k.K(t)\n s2, s1, s3, sS = np.random.multivariate_normal(np.zeros(K.shape[0]), K, size=(4))[:, :, None]\n\n Y1, Y2, Y3, S1, S2, S3 = _generate_high_dimensional_output(D1, D2, D3, s1, s2, s3, sS)\n\n slist = [sS, s1, s2, s3]\n slist_names = [\"sS\", \"s1\", \"s2\", \"s3\"]\n Ylist = [Y1, Y2, Y3]\n\n if plot_sim:\n from matplotlib import pyplot as plt\n import matplotlib.cm as cm\n import itertools\n fig = plt.figure(\"MRD Simulation Data\", figsize=(8, 6))\n fig.clf()\n ax = fig.add_subplot(2, 1, 1)\n labls = slist_names\n for S, lab in zip(slist, labls):\n ax.plot(S, label=lab)\n ax.legend()\n for i, Y in enumerate(Ylist):\n ax = fig.add_subplot(2, len(Ylist), len(Ylist) + 1 + i)\n ax.imshow(Y, aspect='auto', cmap=cm.gray) # @UndefinedVariable\n ax.set_title(\"Y{}\".format(i + 1))\n plt.draw()\n plt.tight_layout()\n\n return slist, [S1, S2, S3], Ylist", "response": "Simulate some data drawn from a matern covariance and a periodic exponential for use in MRD demos."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _simulate_sincos(D1, D2, D3, N, num_inducing, plot_sim=False):\n _np.random.seed(1234)\n\n x = _np.linspace(0, 4 * _np.pi, N)[:, None]\n s1 = _np.vectorize(lambda x: _np.sin(x))\n s2 = _np.vectorize(lambda x: _np.cos(x))\n s3 = _np.vectorize(lambda x:-_np.exp(-_np.cos(2 * x)))\n sS = _np.vectorize(lambda x: _np.cos(x))\n\n s1 = s1(x)\n s2 = s2(x)\n s3 = s3(x)\n sS = sS(x)\n\n s1 -= s1.mean(); s1 /= s1.std(0)\n s2 -= s2.mean(); s2 /= s2.std(0)\n s3 -= s3.mean(); s3 /= s3.std(0)\n sS -= sS.mean(); sS /= sS.std(0)\n\n Y1, Y2, Y3, S1, S2, S3 = _generate_high_dimensional_output(D1, D2, D3, s1, s2, s3, sS)\n\n slist = [sS, s1, s2, s3]\n slist_names = [\"sS\", \"s1\", \"s2\", \"s3\"]\n Ylist = [Y1, Y2, Y3]\n\n if plot_sim:\n from matplotlib import pyplot as plt\n import matplotlib.cm as cm\n import itertools\n fig = plt.figure(\"MRD Simulation Data\", figsize=(8, 6))\n fig.clf()\n ax = fig.add_subplot(2, 1, 1)\n labls = slist_names\n for S, lab in zip(slist, labls):\n ax.plot(S, label=lab)\n ax.legend()\n for i, Y in enumerate(Ylist):\n ax = fig.add_subplot(2, len(Ylist), len(Ylist) + 1 + i)\n ax.imshow(Y, aspect='auto', cmap=cm.gray) # @UndefinedVariable\n ax.set_title(\"Y{}\".format(i + 1))\n plt.draw()\n plt.tight_layout()\n\n return slist, [S1, S2, S3], Ylist", "response": "Simulate some data drawn from sine and cosine for use in demos of MRD"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_log_likelihood(inputs,data,clust):\n \n S = data[0].shape[0] #number of time series\n \n #build a new dataset from the clusters, by combining all clusters together\n X = np.zeros([0,1])\n Y = np.zeros([0,S])\n \n #for each person in the cluster,\n #add their inputs and data to the new dataset\n for p in clust:\n X = np.vstack([X,inputs[p]])\n Y = np.vstack([Y,data[p].T])\n \n #find the loglikelihood. We just add together the LL for each time series.\n #ll=0\n #for s in range(S):\n # m = GPy.models.GPRegression(X,Y[:,s][:,None])\n # m.optimize()\n # ll+=m.log_likelihood()\n\n m = GPy.models.GPRegression(X,Y)\n m.optimize()\n ll=m.log_likelihood() \n return ll,0", "response": "This function calculates the log likelihood of a combined set of clusters ignoring time series offsets."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_log_likelihood_offset(inputs,data,clust):\n \n #if we've only got one cluster, the model has an error, so we want to just\n #use normal GPRegression.\n if len(clust)==1: \n return get_log_likelihood(inputs,data,clust)\n \n S = data[0].shape[0] #number of time series\n \n X = np.zeros([0,2]) #notice the extra column, this is for the cluster index\n Y = np.zeros([0,S])\n \n #for each person in the cluster, add their inputs and data to the new\n #dataset. Note we add an index identifying which person is which data point.\n #This is for the offset model to use, to allow it to know which data points\n #to shift.\n for i,p in enumerate(clust):\n idx = i*np.ones([inputs[p].shape[0],1])\n X = np.vstack([X,np.hstack([inputs[p],idx])])\n Y = np.vstack([Y,data[p].T])\n \n m = GPy.models.GPOffsetRegression(X,Y)\n #TODO: How to select a sensible prior?\n m.offset.set_prior(GPy.priors.Gaussian(0,20)) \n #TODO: Set a sensible start value for the length scale,\n #make it long to help the offset fit.\n \n m.optimize()\n \n ll = m.log_likelihood()\n offset = m.offset.values[0]\n return ll,offset", "response": "This function calculates the log likelihood of a combined set of clusters fitting the offsets\n "} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nclustering data Using the new offset model, this method uses a greedy algorithm to cluster the data. It starts with all the data points in separate clusters and tests whether combining them increases the overall log-likelihood (LL). It then iteratively joins pairs of clusters which cause the greatest increase in the LL, until no join increases the LL. arguments: inputs -- the 'X's in a list, one item per cluster data -- the 'Y's in a list, one item per cluster returns a list of the clusters.", "response": "def cluster(data,inputs,verbose=False):\n \"\"\"Clusters data\n \n Using the new offset model, this method uses a greedy algorithm to cluster\n the data. It starts with all the data points in separate clusters and tests\n whether combining them increases the overall log-likelihood (LL). It then\n iteratively joins pairs of clusters which cause the greatest increase in\n the LL, until no join increases the LL.\n \n arguments:\n inputs -- the 'X's in a list, one item per cluster\n data -- the 'Y's in a list, one item per cluster\n \n returns a list of the clusters. \n \"\"\"\n N=len(data)\n \n \n #Define a set of N active cluster\n active = []\n for p in range(0,N):\n active.append([p])\n\n loglikes = np.zeros(len(active))\n loglikes[:] = None\n\n pairloglikes = np.zeros([len(active),len(active)])\n pairloglikes[:] = None\n pairoffset = np.zeros([len(active),len(active)])\n\n it = 0\n while True:\n \n if verbose:\n it +=1\n print(\"Iteration %d\" % it)\n \n #Compute the log-likelihood of each cluster (add them together)\n for clusti in range(len(active)):\n if verbose:\n sys.stdout.write('.')\n sys.stdout.flush()\n if np.isnan(loglikes[clusti]):\n loglikes[clusti], unused_offset = get_log_likelihood_offset(inputs,data,[clusti])\n\n #try combining with each other cluster...\n for clustj in range(clusti): #count from 0 to clustj-1\n temp = [clusti,clustj]\n if np.isnan(pairloglikes[clusti,clustj]):\n pairloglikes[clusti,clustj],pairoffset[clusti,clustj] = get_log_likelihood_offset(inputs,data,temp)\n\n seploglikes = np.repeat(loglikes[:,None].T,len(loglikes),0)+np.repeat(loglikes[:,None],len(loglikes),1)\n loglikeimprovement = pairloglikes - seploglikes #how much likelihood improves with clustering\n top = np.unravel_index(np.nanargmax(pairloglikes-seploglikes), pairloglikes.shape)\n\n #if loglikeimprovement.shape[0]<3:\n # #no more clustering to do - this shouldn't happen really unless\n # #we've set the threshold to apply clustering to less than 0\n # break \n \n #if theres further clustering to be done...\n if loglikeimprovement[top[0],top[1]]>0:\n active[top[0]].extend(active[top[1]])\n offset=pairoffset[top[0],top[1]]\n inputs[top[0]] = np.vstack([inputs[top[0]],inputs[top[1]]-offset])\n data[top[0]] = np.hstack([data[top[0]],data[top[1]]])\n del inputs[top[1]]\n del data[top[1]]\n del active[top[1]]\n\n #None = message to say we need to recalculate\n pairloglikes[:,top[0]] = None \n pairloglikes[top[0],:] = None \n pairloglikes = np.delete(pairloglikes,top[1],0)\n pairloglikes = np.delete(pairloglikes,top[1],1)\n loglikes[top[0]] = None\n loglikes = np.delete(loglikes,top[1])\n else:\n break\n \n #if loglikeimprovement[top[0],top[1]]>0:\n # print \"joined\"\n # print top\n # print offset\n # print offsets\n # print offsets[top[1]]-offsets[top[0]]\n\n #TODO Add a way to return the offsets applied to all the time series\n return active"} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns covariance between X and X2.", "response": "def K(self, X, X2, target):\n \"\"\"Return covariance between X and X2.\"\"\"\n if (X2 is None) or (X2 is X):\n target[np.diag_indices_from(target)] += self._Kdiag(X)"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef dKdiag_dtheta(self, dL_dKdiag, X, target):\n target += 2.*self.mapping.df_dtheta(dL_dKdiag[:, None]*self.mapping.f(X), X)", "response": "Gradient of diagonal of covariance with respect to parameters."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef gradients_X(self, dL_dK, X, X2, target):\n if X2==None or X2 is X:\n dL_dKdiag = dL_dK.flat[::dL_dK.shape[0]+1]\n self.dKdiag_dX(dL_dKdiag, X, target)", "response": "Derivative of the covariance matrix with respect to X."} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nlog PDF for the current point of the log pdf", "response": "def logpdf_link(self, link_f, y, Y_metadata=None):\n \"\"\"\n Log Likelihood Function given link(f)\n\n .. math::\n\n\n :param link_f: latent variables (link(f))\n :type link_f: Nx1 array\n :param y: data\n :type y: Nx1 array\n :param Y_metadata: includes censoring information in dictionary key 'censored'\n :returns: likelihood evaluated for this point\n :rtype: float\n\n \"\"\"\n # c = np.zeros((y.shape[0],))\n c = np.zeros_like(link_f)\n if Y_metadata is not None and 'censored' in Y_metadata.keys():\n c = Y_metadata['censored']\n\n link_f = np.clip(link_f, 1e-150, 1e100)\n # y_link_f = y/link_f\n # y_link_f_r = y_link_f**self.r\n # y_link_f_r = np.clip(y**self.r, 1e-150, 1e200) / np.clip(link_f**self.r, 1e-150, 1e200)\n # y_link_f_r = np.clip((y/link_f)**self.r, 1e-150, 1e200)\n y_r = np.clip(y**self.r, 1e-150, 1e200)\n link_f_r = np.clip(link_f**self.r, 1e-150, 1e200)\n y_link_f_r = np.clip(y_r / link_f_r, 1e-150, 1e200)\n #uncensored = (1-c)*(np.log(self.r) + (self.r+1)*np.log(y) - self.r*np.log(link_f) - 2*np.log1p(y_link_f_r))\n #uncensored = (1-c)*(np.log((self.r/link_f)*y_link_f**(self.r-1)) - 2*np.log1p(y_link_f_r))\n\n # clever way tp break it into censored and uncensored-parts ..\n uncensored = (1-c)*(np.log(self.r) + (self.r-1)*np.log(y) - self.r*np.log(link_f) - 2*np.log1p(y_link_f_r))\n censored = (c)*(-np.log1p(y_link_f_r))\n #\n return uncensored + censored"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef dlogpdf_dlink(self, link_f, y, Y_metadata=None):\n # c = Y_metadata['censored']\n # for debugging\n # c = np.zeros((y.shape[0],))\n c = np.zeros_like(link_f)\n\n if Y_metadata is not None and 'censored' in Y_metadata.keys():\n c = Y_metadata['censored']\n\n #y_link_f = y/link_f\n #y_link_f_r = y_link_f**self.r\n y_link_f_r = np.clip(y**self.r, 1e-150, 1e200) / np.clip(link_f**self.r, 1e-150, 1e200)\n\n #In terms of link_f\n # uncensored = (1-c)*( (2*self.r*y**r)/(link_f**self.r + y**self.r) - link_f*self.r)\n uncensored = (1-c)*self.r*(y_link_f_r - 1)/(link_f*(1 + y_link_f_r))\n censored = c*(self.r*y_link_f_r/(link_f*y_link_f_r + link_f))\n return uncensored + censored", "response": "Gradient of the log likelihood function at y given link f w. r. t link f"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef d2logpdf_dlink2(self, link_f, y, Y_metadata=None):\n # c = Y_metadata['censored']\n # c = np.zeros((y.shape[0],))\n c = np.zeros_like(link_f)\n\n if Y_metadata is not None and 'censored' in Y_metadata.keys():\n c = Y_metadata['censored']\n\n y_link_f = y/link_f\n y_link_f_r = y_link_f**self.r\n\n #In terms of link_f\n censored = c*(-self.r*y_link_f_r*(y_link_f_r + self.r + 1)/((link_f**2)*(y_link_f_r + 1)**2))\n uncensored = (1-c)*(-self.r*(2*self.r*y_link_f_r + y_link_f**(2*self.r) - 1) / ((link_f**2)*(1+ y_link_f_r)**2))\n hess = censored + uncensored\n return hess", "response": "d2logpdf_dlink2 - 2d logpdf at y given link f and y"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nfunctioning to compute the third order derivative log - likelihood function at y given link f.", "response": "def d3logpdf_dlink3(self, link_f, y, Y_metadata=None):\n \"\"\"\n Third order derivative log-likelihood function at y given link(f) w.r.t link(f)\n\n .. math::\n\n\n :param link_f: latent variables link(f)\n :type link_f: Nx1 array\n :param y: data\n :type y: Nx1 array\n :param Y_metadata: includes censoring information in dictionary key 'censored'\n :returns: third derivative of likelihood evaluated at points f\n :rtype: Nx1 array\n \"\"\"\n # c = Y_metadata['censored']\n # for debugging\n # c = np.zeros((y.shape[0],))\n c = np.zeros_like(link_f)\n\n if Y_metadata is not None and 'censored' in Y_metadata.keys():\n c = Y_metadata['censored']\n y_link_f = y/link_f\n y_link_f_r = y_link_f**self.r\n\n #In terms of link_f\n censored = c*(self.r*y_link_f_r*(((self.r**2)*(-(y_link_f_r - 1))) + 3*self.r*(y_link_f_r + 1) + 2*(y_link_f_r + 1)**2)\n / ((link_f**3)*(y_link_f_r + 1)**3))\n uncensored = (1-c)*(2*self.r*(-(self.r**2)*(y_link_f_r -1)*y_link_f_r + 3*self.r*(y_link_f_r + 1)*y_link_f_r + (y_link_f_r - 1)*(y_link_f_r + 1)**2)\n / ((link_f**3)*(y_link_f_r + 1)**3))\n\n d3lik_dlink3 = censored + uncensored\n return d3lik_dlink3"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef dlogpdf_link_dr(self, inv_link_f, y, Y_metadata=None):\n # c = Y_metadata['censored']\n # c = np.zeros((y.shape[0],))\n c = np.zeros_like(y)\n if Y_metadata is not None and 'censored' in Y_metadata.keys():\n c = Y_metadata['censored']\n\n link_f = inv_link_f #FIXME: Change names consistently...\n y_link_f = y/link_f\n log_y_link_f = np.log(y) - np.log(link_f)\n y_link_f_r = y_link_f**self.r\n\n #In terms of link_f\n censored = c*(-y_link_f_r*log_y_link_f/(1 + y_link_f_r))\n uncensored = (1-c)*(1./self.r + np.log(y) - np.log(link_f) - (2*y_link_f_r*log_y_link_f) / (1 + y_link_f_r))\n\n dlogpdf_dr = censored + uncensored\n return dlogpdf_dr", "response": "Gradient of the log - likelihood function at y given f w. r. t shape parameter\n "} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef optimize_auto(self,max_iters=10000,verbose=True):\n self.Z.fix(warning=False)\n self.kern.fix(warning=False)\n self.kern_row.fix(warning=False)\n self.Zr.fix(warning=False)\n self.Xr.fix(warning=False)\n self.optimize(max_iters=int(0.1*max_iters),messages=verbose)\n self.unfix()\n self.optimize(max_iters=max_iters,messages=verbose)", "response": "Optimize the model parameters through a pre - defined protocol."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef ln_diff_erfs(x1, x2, return_sign=False):\n x1 = np.require(x1).real\n x2 = np.require(x2).real\n if x1.size==1:\n x1 = np.reshape(x1, (1, 1))\n if x2.size==1:\n x2 = np.reshape(x2, (1, 1))\n\n if x1.shape==x2.shape:\n v = np.zeros_like(x1)\n else:\n if x1.size==1:\n v = np.zeros(x2.shape)\n elif x2.size==1:\n v = np.zeros(x1.shape)\n else:\n raise ValueError(\"This function does not broadcast unless provided with a scalar.\")\n\n if x1.size == 1:\n x1 = np.tile(x1, x2.shape)\n\n if x2.size == 1:\n x2 = np.tile(x2, x1.shape)\n\n sign = np.sign(x1 - x2)\n if x1.size == 1:\n if sign== -1:\n swap = x1\n x1 = x2\n x2 = swap\n else:\n I = sign == -1\n swap = x1[I]\n x1[I] = x2[I]\n x2[I] = swap\n\n with np.errstate(divide='ignore'):\n # switch off log of zero warnings.\n\n # Case 0: arguments of different sign, no problems with loss of accuracy\n I0 = np.logical_or(np.logical_and(x1>0, x2<0), np.logical_and(x2>0, x1<0)) # I1=(x1*x2)<0\n\n # Case 1: x1 = x2 so we have log of zero.\n I1 = (x1 == x2)\n\n # Case 2: Both arguments are non-negative\n I2 = np.logical_and(x1 > 0, np.logical_and(np.logical_not(I0),\n np.logical_not(I1)))\n # Case 3: Both arguments are non-positive\n I3 = np.logical_and(np.logical_and(np.logical_not(I0),\n np.logical_not(I1)),\n np.logical_not(I2))\n _x2 = x2.flatten()\n _x1 = x1.flatten()\n for group, flags in zip((0, 1, 2, 3), (I0, I1, I2, I3)):\n\n if np.any(flags):\n if not x1.size==1:\n _x1 = x1[flags]\n if not x2.size==1:\n _x2 = x2[flags]\n if group==0:\n v[flags] = np.log( erf(_x1) - erf(_x2) )\n elif group==1:\n v[flags] = -np.inf\n elif group==2:\n v[flags] = np.log(erfcx(_x2)\n -erfcx(_x1)*np.exp(_x2**2\n -_x1**2)) - _x2**2\n elif group==3:\n v[flags] = np.log(erfcx(-_x1)\n -erfcx(-_x2)*np.exp(_x1**2\n -_x2**2))-_x1**2\n\n # TODO: switch back on log of zero warnings.\n\n if return_sign:\n return v, sign\n else:\n if v.size==1:\n if sign==-1:\n v = v.view('complex64')\n v += np.pi*1j\n else:\n # Need to add in a complex part because argument is negative.\n v = v.view('complex64')\n v[I] += np.pi*1j\n\n return v", "response": "Function for stably computing the log of difference of two erfs."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a Posterior class containing essential quantities of the posterior", "response": "def inference(self, kern, X, likelihood, Y, mean_function=None, Y_metadata=None, K=None, variance=None, Z_tilde=None):\n \"\"\"\n Returns a Posterior class containing essential quantities of the posterior\n \"\"\"\n\n if mean_function is None:\n m = 0\n else:\n m = mean_function.f(X)\n\n if variance is None:\n variance = likelihood.gaussian_variance(Y_metadata)\n\n YYT_factor = Y-m\n\n if K is None:\n K = kern.K(X)\n\n Ky = K.copy()\n diag.add(Ky, variance+1e-8)\n\n Wi, LW, LWi, W_logdet = pdinv(Ky)\n\n alpha, _ = dpotrs(LW, YYT_factor, lower=1)\n\n log_marginal = 0.5*(-Y.size * log_2_pi - Y.shape[1] * W_logdet - np.sum(alpha * YYT_factor))\n\n if Z_tilde is not None:\n # This is a correction term for the log marginal likelihood\n # In EP this is log Z_tilde, which is the difference between the\n # Gaussian marginal and Z_EP\n log_marginal += Z_tilde\n\n dL_dK = 0.5 * (tdot(alpha) - Y.shape[1] * Wi)\n\n dL_dthetaL = likelihood.exact_inference_gradients(np.diag(dL_dK), Y_metadata)\n\n return Posterior(woodbury_chol=LW, woodbury_vector=alpha, K=K), log_marginal, {'dL_dK':dL_dK, 'dL_dthetaL':dL_dthetaL, 'dL_dm':alpha}"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nleave one out error as found in \"Bayesian leave-one-out cross-validation approximations for Gaussian latent variable models\" Vehtari et al. 2014.", "response": "def LOO(self, kern, X, Y, likelihood, posterior, Y_metadata=None, K=None):\n \"\"\"\n Leave one out error as found in\n \"Bayesian leave-one-out cross-validation approximations for Gaussian latent variable models\"\n Vehtari et al. 2014.\n \"\"\"\n g = posterior.woodbury_vector\n c = posterior.woodbury_inv\n c_diag = np.diag(c)[:, None]\n neg_log_marginal_LOO = 0.5*np.log(2*np.pi) - 0.5*np.log(c_diag) + 0.5*(g**2)/c_diag\n #believe from Predictive Approaches for Choosing Hyperparameters in Gaussian Processes\n #this is the negative marginal LOO\n return -neg_log_marginal_LOO"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nleave one out log predictive density as found in \"Bayesian leave-one-out cross-validation approximations for Gaussian latent variable models\" Vehtari et al. 2014.", "response": "def LOO(self, kern, X, Y, likelihood, posterior, Y_metadata=None, K=None, f_hat=None, W=None, Ki_W_i=None):\n \"\"\"\n Leave one out log predictive density as found in\n \"Bayesian leave-one-out cross-validation approximations for Gaussian latent variable models\"\n Vehtari et al. 2014.\n \"\"\"\n Ki_f_init = np.zeros_like(Y)\n\n if K is None:\n K = kern.K(X)\n\n if f_hat is None:\n f_hat, _ = self.rasm_mode(K, Y, likelihood, Ki_f_init, Y_metadata=Y_metadata)\n\n if W is None:\n W = -likelihood.d2logpdf_df2(f_hat, Y, Y_metadata=Y_metadata)\n\n if Ki_W_i is None:\n _, _, _, Ki_W_i = self._compute_B_statistics(K, W, likelihood.log_concave)\n\n logpdf_dfhat = likelihood.dlogpdf_df(f_hat, Y, Y_metadata=Y_metadata)\n\n if W.shape[1] == 1:\n W = np.diagflat(W)\n\n #Eq 14, and 16\n var_site = 1./np.diag(W)[:, None]\n mu_site = f_hat + var_site*logpdf_dfhat\n prec_site = 1./var_site\n #Eq 19\n marginal_cov = Ki_W_i\n marginal_mu = marginal_cov.dot(np.diagflat(prec_site)).dot(mu_site)\n marginal_var = np.diag(marginal_cov)[:, None]\n #Eq 30 with using site parameters instead of Gaussian site parameters\n #(var_site instead of sigma^{2} )\n posterior_cav_var = 1./(1./marginal_var - 1./var_site)\n posterior_cav_mean = posterior_cav_var*((1./marginal_var)*marginal_mu - (1./var_site)*Y)\n\n flat_y = Y.flatten()\n flat_mu = posterior_cav_mean.flatten()\n flat_var = posterior_cav_var.flatten()\n\n if Y_metadata is not None:\n #Need to zip individual elements of Y_metadata aswell\n Y_metadata_flat = {}\n if Y_metadata is not None:\n for key, val in Y_metadata.items():\n Y_metadata_flat[key] = np.atleast_1d(val).reshape(-1, 1)\n\n zipped_values = []\n\n for i in range(Y.shape[0]):\n y_m = {}\n for key, val in Y_metadata_flat.items():\n if np.isscalar(val) or val.shape[0] == 1:\n y_m[key] = val\n else:\n #Won't broadcast yet\n y_m[key] = val[i]\n zipped_values.append((flat_y[i], flat_mu[i], flat_var[i], y_m))\n else:\n #Otherwise just pass along None's\n zipped_values = zip(flat_y, flat_mu, flat_var, [None]*Y.shape[0])\n\n def integral_generator(yi, mi, vi, yi_m):\n def f(fi_star):\n #More stable in the log space\n p_fi = np.exp(likelihood.logpdf(fi_star, yi, yi_m)\n - 0.5*np.log(2*np.pi*vi)\n - 0.5*np.square(mi-fi_star)/vi)\n return p_fi\n return f\n\n #Eq 30\n p_ystar, _ = zip(*[quad(integral_generator(y, m, v, yi_m), -np.inf, np.inf)\n for y, m, v, yi_m in zipped_values])\n p_ystar = np.array(p_ystar).reshape(-1, 1)\n return np.log(p_ystar)"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning a Posterior class containing essential quantities of the posterior", "response": "def inference(self, kern, X, likelihood, Y, mean_function=None, Y_metadata=None):\n \"\"\"\n Returns a Posterior class containing essential quantities of the posterior\n \"\"\"\n assert mean_function is None, \"inference with a mean function not implemented\"\n\n # Compute K\n K = kern.K(X)\n\n #Find mode\n if self.bad_fhat or self.first_run:\n Ki_f_init = np.zeros_like(Y)\n self.first_run = False\n else:\n Ki_f_init = self._previous_Ki_fhat\n\n Ki_f_init = np.zeros_like(Y)# FIXME: take this out\n\n f_hat, Ki_fhat = self.rasm_mode(K, Y, likelihood, Ki_f_init, Y_metadata=Y_metadata)\n\n #Compute hessian and other variables at mode\n log_marginal, woodbury_inv, dL_dK, dL_dthetaL = self.mode_computations(f_hat, Ki_fhat, K, Y, likelihood, kern, Y_metadata)\n\n self._previous_Ki_fhat = Ki_fhat.copy()\n return Posterior(woodbury_vector=Ki_fhat, woodbury_inv=woodbury_inv, K=K), log_marginal, {'dL_dK':dL_dK, 'dL_dthetaL':dL_dthetaL}"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef mode_computations(self, f_hat, Ki_f, K, Y, likelihood, kern, Y_metadata):\n #At this point get the hessian matrix (or vector as W is diagonal)\n W = -likelihood.d2logpdf_df2(f_hat, Y, Y_metadata=Y_metadata)\n if np.any(np.isnan(W)):\n raise ValueError('One or more element(s) of W is NaN')\n\n K_Wi_i, logdet_I_KW, I_KW_i, Ki_W_i = self._compute_B_statistics(K, W, likelihood.log_concave)\n\n #compute the log marginal\n log_marginal = -0.5*np.sum(np.dot(Ki_f.T, f_hat)) + np.sum(likelihood.logpdf(f_hat, Y, Y_metadata=Y_metadata)) - 0.5*logdet_I_KW\n\n # Compute matrices for derivatives\n dW_df = -likelihood.d3logpdf_df3(f_hat, Y, Y_metadata=Y_metadata) # -d3lik_d3fhat\n if np.any(np.isnan(dW_df)):\n raise ValueError('One or more element(s) of dW_df is NaN')\n\n dL_dfhat = -0.5*(np.diag(Ki_W_i)[:, None]*dW_df) # s2 in R&W p126 line 9.\n #BiK, _ = dpotrs(L, K, lower=1)\n #dL_dfhat = 0.5*np.diag(BiK)[:, None]*dW_df\n I_KW_i = np.eye(Y.shape[0]) - np.dot(K, K_Wi_i)\n\n ####################\n # compute dL_dK #\n ####################\n if kern.size > 0 and not kern.is_fixed:\n #Explicit\n explicit_part = 0.5*(np.dot(Ki_f, Ki_f.T) - K_Wi_i)\n\n #Implicit\n implicit_part = np.dot(Ki_f, dL_dfhat.T).dot(I_KW_i)\n\n dL_dK = explicit_part + implicit_part\n else:\n dL_dK = np.zeros(likelihood.size)\n\n ####################\n #compute dL_dthetaL#\n ####################\n if likelihood.size > 0 and not likelihood.is_fixed:\n dlik_dthetaL, dlik_grad_dthetaL, dlik_hess_dthetaL = likelihood._laplace_gradients(f_hat, Y, Y_metadata=Y_metadata)\n\n num_params = likelihood.size\n # make space for one derivative for each likelihood parameter\n dL_dthetaL = np.zeros(num_params)\n for thetaL_i in range(num_params):\n #Explicit\n dL_dthetaL_exp = ( np.sum(dlik_dthetaL[thetaL_i,:, :])\n # The + comes from the fact that dlik_hess_dthetaL == -dW_dthetaL\n + 0.5*np.sum(np.diag(Ki_W_i)*np.squeeze(dlik_hess_dthetaL[thetaL_i, :, :]))\n )\n\n #Implicit\n dfhat_dthetaL = mdot(I_KW_i, K, dlik_grad_dthetaL[thetaL_i, :, :])\n #dfhat_dthetaL = mdot(Ki_W_i, dlik_grad_dthetaL[thetaL_i, :, :])\n dL_dthetaL_imp = np.dot(dL_dfhat.T, dfhat_dthetaL)\n dL_dthetaL[thetaL_i] = np.sum(dL_dthetaL_exp + dL_dthetaL_imp)\n\n else:\n dL_dthetaL = np.zeros(likelihood.size)\n\n #Cache some things for speedy LOO\n self.Ki_W_i = Ki_W_i\n self.K = K\n self.W = W\n self.f_hat = f_hat\n return log_marginal, K_Wi_i, dL_dK, dL_dthetaL", "response": "Compute the hessian and effective covariance matrix for the current state of the class."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _compute_B_statistics(self, K, W, log_concave, *args, **kwargs):\n if not log_concave:\n #print \"Under 1e-10: {}\".format(np.sum(W < 1e-6))\n W = np.clip(W, 1e-6, 1e+30)\n # For student-T we can clip this more intelligently. If the\n # objective has hardly changed, we can increase the clipping limit\n # by ((v+1)/v)/sigma2\n # NOTE: when setting a parameter inside parameters_changed it will allways come to closed update circles!!!\n #W.__setitem__(W < 1e-6, 1e-6, update=False) # FIXME-HACK: This is a hack since GPy can't handle negative variances which can occur\n # If the likelihood is non-log-concave. We wan't to say that there is a negative variance\n # To cause the posterior to become less certain than the prior and likelihood,\n # This is a property only held by non-log-concave likelihoods\n if np.any(np.isnan(W)):\n raise ValueError('One or more element(s) of W is NaN')\n #W is diagonal so its sqrt is just the sqrt of the diagonal elements\n W_12 = np.sqrt(W)\n B = np.eye(K.shape[0]) + W_12*K*W_12.T\n L = jitchol(B)\n\n LiW12, _ = dtrtrs(L, np.diagflat(W_12), lower=1, trans=0)\n K_Wi_i = np.dot(LiW12.T, LiW12) # R = W12BiW12, in R&W p 126, eq 5.25\n\n #here's a better way to compute the required matrix.\n # you could do the model finding witha backsub, instead of a dot...\n #L2 = L/W_12\n #K_Wi_i_2 , _= dpotri(L2)\n #symmetrify(K_Wi_i_2)\n\n #compute vital matrices\n C = np.dot(LiW12, K)\n Ki_W_i = K - C.T.dot(C)\n\n I_KW_i = np.eye(K.shape[0]) - np.dot(K, K_Wi_i)\n logdet_I_KW = 2*np.sum(np.log(np.diag(L)))\n\n return K_Wi_i, logdet_I_KW, I_KW_i, Ki_W_i", "response": "Compute the B statistics for a specific entry in the log - concave distribution."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncomputing the statistics for the B matrix and the log concave matrix.", "response": "def _compute_B_statistics(self, K, W, log_concave, *args, **kwargs):\n \"\"\"\n Rasmussen suggests the use of a numerically stable positive definite matrix B\n Which has a positive diagonal element and can be easyily inverted\n\n :param K: Prior Covariance matrix evaluated at locations X\n :type K: NxN matrix\n :param W: Negative hessian at a point (diagonal matrix)\n :type W: Vector of diagonal values of hessian (1xN)\n :returns: (K_Wi_i, L_B, not_provided)\n \"\"\"\n #w = GPy.util.diag.view(W)\n #W[:] = np.where(w<1e-6, 1e-6, w)\n\n #B = I + KW\n B = np.eye(K.shape[0]) + np.dot(K, W)\n #Bi, L, Li, logdetB = pdinv(B)\n Bi = np.linalg.inv(B)\n\n #K_Wi_i = np.eye(K.shape[0]) - mdot(W, Bi, K)\n K_Wi_i = np.dot(W, Bi)\n\n #self.K_Wi_i_brute = np.linalg.inv(K + np.linalg.inv(W))\n #self.B = B\n #self.Bi = Bi\n Ki_W_i = np.dot(Bi, K)\n\n sign, logdetB = np.linalg.slogdet(B)\n return K_Wi_i, sign*logdetB, Bi, Ki_W_i"} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ngives the derivative of an objective fn with respect to the cholesky L compute the derivate with respect to the original matrix K", "response": "def _backprop_gradient_pure(dL, L):\n \"\"\"\n Given the derivative of an objective fn with respect to the cholesky L,\n compute the derivate with respect to the original matrix K, defined as\n\n K = LL^T\n\n where L was obtained by Cholesky decomposition\n \"\"\"\n dL_dK = np.tril(dL).copy()\n N = L.shape[0]\n for k in range(N - 1, -1, -1):\n for j in range(k + 1, N):\n for i in range(j, N):\n dL_dK[i, k] -= dL_dK[i, j] * L[j, k]\n dL_dK[j, k] -= dL_dK[i, j] * L[i, k]\n for j in range(k + 1, N):\n dL_dK[j, k] /= L[k, k]\n dL_dK[k, k] -= L[j, k] * dL_dK[j, k]\n dL_dK[k, k] /= (2 * L[k, k])\n return dL_dK"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nwork out which indexes of the flatteneed array should be fixed if we want the cholesky to represent a low rank matrix", "response": "def indexes_to_fix_for_low_rank(rank, size):\n \"\"\"\n Work out which indexes of the flatteneed array should be fixed if we want\n the cholesky to represent a low rank matrix\n \"\"\"\n #first we'll work out what to keep, and the do the set difference.\n\n #here are the indexes of the first column, which are the triangular numbers\n n = np.arange(size)\n triangulars = (n**2 + n) / 2\n keep = []\n for i in range(rank):\n keep.append(triangulars[i:] + i)\n #add the diagonal\n keep.append(triangulars[1:]-1)\n keep.append((size**2 + size)/2 -1)# the very last element\n keep = np.hstack(keep)\n\n return np.setdiff1d(np.arange((size**2+size)/2), keep)"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef pdf_link(self, inv_link_f, y, Y_metadata):\n return np.exp(self.logpdf_link(inv_link_f, y, Y_metadata))", "response": "Evaluate the logpdf of the inverse link of f."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef logpdf_link(self, inv_link_f, y, Y_metadata=None):\n N = Y_metadata['trials']\n np.testing.assert_array_equal(N.shape, y.shape)\n\n nchoosey = special.gammaln(N+1) - special.gammaln(y+1) - special.gammaln(N-y+1)\n \n Ny = N-y\n t1 = np.zeros(y.shape)\n t2 = np.zeros(y.shape)\n t1[y>0] = y[y>0]*np.log(inv_link_f[y>0])\n t2[Ny>0] = Ny[Ny>0]*np.log(1.-inv_link_f[Ny>0])\n \n return nchoosey + t1 + t2", "response": "Log - likelihood function given inverse link of f and data y."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef dlogpdf_dlink(self, inv_link_f, y, Y_metadata=None):\n N = Y_metadata['trials']\n np.testing.assert_array_equal(N.shape, y.shape)\n\n Ny = N-y\n t1 = np.zeros(y.shape)\n t2 = np.zeros(y.shape)\n t1[y>0] = y[y>0]/inv_link_f[y>0]\n t2[Ny>0] = (Ny[Ny>0])/(1.-inv_link_f[Ny>0]) \n\n return t1 - t2", "response": "Gradient of the pdf at y given inverse link of f w. r. t inverse link of f."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef samples(self, gp, Y_metadata=None, **kw):\n orig_shape = gp.shape\n gp = gp.flatten()\n N = Y_metadata['trials']\n Ysim = np.random.binomial(N, self.gp_link.transf(gp))\n return Ysim.reshape(orig_shape)", "response": "Returns a set of samples based on a given value of the latent variable."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreading image data from Netpbm file as numpy array.", "response": "def imread(filename, *args, **kwargs):\n \"\"\"Return image data from Netpbm file as numpy array.\n\n `args` and `kwargs` are arguments to NetpbmFile.asarray().\n\n Examples\n --------\n >>> image = imread('_tmp.pgm')\n\n \"\"\"\n try:\n netpbm = NetpbmFile(filename)\n image = netpbm.asarray()\n finally:\n netpbm.close()\n return image"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nwrite image data to Netpbm file.", "response": "def imsave(filename, data, maxval=None, pam=False):\n \"\"\"Write image data to Netpbm file.\n\n Examples\n --------\n >>> image = numpy.array([[0, 1],[65534, 65535]], dtype=numpy.uint16)\n >>> imsave('_tmp.pgm', image)\n\n \"\"\"\n try:\n netpbm = NetpbmFile(data, maxval=maxval)\n netpbm.write(filename, pam=pam)\n finally:\n netpbm.close()"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn image data from file as numpy array.", "response": "def asarray(self, copy=True, cache=False, **kwargs):\n \"\"\"Return image data from file as numpy array.\"\"\"\n data = self._data\n if data is None:\n data = self._read_data(self._fh, **kwargs)\n if cache:\n self._data = data\n else:\n return data\n return deepcopy(data) if copy else data"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nwrites instance to file.", "response": "def write(self, arg, **kwargs):\n \"\"\"Write instance to file.\"\"\"\n if hasattr(arg, 'seek'):\n self._tofile(arg, **kwargs)\n else:\n with open(arg, 'wb') as fid:\n self._tofile(fid, **kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef close(self):\n if self._filename and self._fh:\n self._fh.close()\n self._fh = None", "response": "Close open file. Future asarray calls might fail."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ninitializing instance from open file.", "response": "def _fromfile(self, fh):\n \"\"\"Initialize instance from open file.\"\"\"\n fh.seek(0)\n data = fh.read(4096)\n if (len(data) < 7) or not (b'0' < data[1:2] < b'8'):\n raise ValueError(\"Not a Netpbm file:\\n%s\" % data[:32])\n try:\n self._read_pam_header(data)\n except Exception:\n try:\n self._read_pnm_header(data)\n except Exception:\n raise ValueError(\"Not a Netpbm file:\\n%s\" % data[:32])"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _read_pam_header(self, data):\n regroups = re.search(\n b\"(^P7[\\n\\r]+(?:(?:[\\n\\r]+)|(?:#.*)|\"\n b\"(HEIGHT\\s+\\d+)|(WIDTH\\s+\\d+)|(DEPTH\\s+\\d+)|(MAXVAL\\s+\\d+)|\"\n b\"(?:TUPLTYPE\\s+\\w+))*ENDHDR\\n)\", data).groups()\n self.header = regroups[0]\n self.magicnum = b'P7'\n for group in regroups[1:]:\n key, value = group.split()\n setattr(self, unicode(key).lower(), int(value))\n matches = re.findall(b\"(TUPLTYPE\\s+\\w+)\", self.header)\n self.tupltypes = [s.split(None, 1)[1] for s in matches]", "response": "Read PAM header and initialize instance."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _read_pnm_header(self, data):\n bpm = data[1:2] in b\"14\"\n regroups = re.search(b\"\".join((\n b\"(^(P[123456]|P7 332)\\s+(?:#.*[\\r\\n])*\",\n b\"\\s*(\\d+)\\s+(?:#.*[\\r\\n])*\",\n b\"\\s*(\\d+)\\s+(?:#.*[\\r\\n])*\" * (not bpm),\n b\"\\s*(\\d+)\\s(?:\\s*#.*[\\r\\n]\\s)*)\")), data).groups() + (1, ) * bpm\n self.header = regroups[0]\n self.magicnum = regroups[1]\n self.width = int(regroups[2])\n self.height = int(regroups[3])\n self.maxval = int(regroups[4])\n self.depth = 3 if self.magicnum in b\"P3P6P7 332\" else 1\n self.tupltypes = [self._types[self.magicnum]]", "response": "Read PNM header and initialize instance."} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns image data from open file as numpy array.", "response": "def _read_data(self, fh, byteorder='>'):\n \"\"\"Return image data from open file as numpy array.\"\"\"\n fh.seek(len(self.header))\n data = fh.read()\n dtype = 'u1' if self.maxval < 256 else byteorder + 'u2'\n depth = 1 if self.magicnum == b\"P7 332\" else self.depth\n shape = [-1, self.height, self.width, depth]\n size = numpy.prod(shape[1:])\n if self.magicnum in b\"P1P2P3\":\n data = numpy.array(data.split(None, size)[:size], dtype)\n data = data.reshape(shape)\n elif self.maxval == 1:\n shape[2] = int(math.ceil(self.width / 8))\n data = numpy.frombuffer(data, dtype).reshape(shape)\n data = numpy.unpackbits(data, axis=-2)[:, :, :self.width, :]\n else:\n data = numpy.frombuffer(data, dtype)\n data = data[:size * (data.size // size)].reshape(shape)\n if data.shape[0] < 2:\n data = data.reshape(data.shape[1:])\n if data.shape[-1] < 2:\n data = data.reshape(data.shape[:-1])\n if self.magicnum == b\"P7 332\":\n rgb332 = numpy.array(list(numpy.ndindex(8, 8, 4)), numpy.uint8)\n rgb332 *= [36, 36, 85]\n data = numpy.take(rgb332, data, axis=0)\n return data"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ninitialize instance from numpy array.", "response": "def _fromdata(self, data, maxval=None):\n \"\"\"Initialize instance from numpy array.\"\"\"\n data = numpy.array(data, ndmin=2, copy=True)\n if data.dtype.kind not in \"uib\":\n raise ValueError(\"not an integer type: %s\" % data.dtype)\n if data.dtype.kind == 'i' and numpy.min(data) < 0:\n raise ValueError(\"data out of range: %i\" % numpy.min(data))\n if maxval is None:\n maxval = numpy.max(data)\n maxval = 255 if maxval < 256 else 65535\n if maxval < 0 or maxval > 65535:\n raise ValueError(\"data out of range: %i\" % maxval)\n data = data.astype('u1' if maxval < 256 else '>u2')\n self._data = data\n if data.ndim > 2 and data.shape[-1] in (3, 4):\n self.depth = data.shape[-1]\n self.width = data.shape[-2]\n self.height = data.shape[-3]\n self.magicnum = b'P7' if self.depth == 4 else b'P6'\n else:\n self.depth = 1\n self.width = data.shape[-1]\n self.height = data.shape[-2]\n self.magicnum = b'P5' if maxval > 1 else b'P4'\n self.maxval = maxval\n self.tupltypes = [self._types[self.magicnum]]\n self.header = self._header()"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nwriting the current object to a Netbm file.", "response": "def _tofile(self, fh, pam=False):\n \"\"\"Write Netbm file.\"\"\"\n fh.seek(0)\n fh.write(self._header(pam))\n data = self.asarray(copy=False)\n if self.maxval == 1:\n data = numpy.packbits(data, axis=-1)\n data.tofile(fh)"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _header(self, pam=False):\n if pam or self.magicnum == b'P7':\n header = \"\\n\".join((\n \"P7\",\n \"HEIGHT %i\" % self.height,\n \"WIDTH %i\" % self.width,\n \"DEPTH %i\" % self.depth,\n \"MAXVAL %i\" % self.maxval,\n \"\\n\".join(\"TUPLTYPE %s\" % unicode(i) for i in self.tupltypes),\n \"ENDHDR\\n\"))\n elif self.maxval == 1:\n header = \"P4 %i %i\\n\" % (self.width, self.height)\n elif self.depth == 1:\n header = \"P5 %i %i %i\\n\" % (self.width, self.height, self.maxval)\n else:\n header = \"P6 %i %i %i\\n\" % (self.width, self.height, self.maxval)\n if sys.version_info[0] > 2:\n header = bytes(header, 'ascii')\n return header", "response": "Return the file header as byte string."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef sde(self): \n \n #import pdb; pdb.set_trace()\n # Params to use: (in that order)\n #self.variance\n #self.period\n #self.lengthscale\n if self.approx_order is not None:\n N = int(self.approx_order)\n else:\n N = 7 # approximation order \n \n p_period = float(self.period) \n p_lengthscale = 2*float(self.lengthscale)\n p_variance = float(self.variance) \n \n w0 = 2*np.pi/p_period # frequency\n # lengthscale is multiplied by 2 because of different definition of lengthscale\n \n [q2,dq2l] = seriescoeff(N, p_lengthscale, p_variance) \n \n dq2l = 2*dq2l # This is because the lengthscale if multiplied by 2.\n \n eps = 1e-12\n if np.any( np.isfinite(q2) == False) or np.any( np.abs(q2) > 1.0/eps) or np.any( np.abs(q2) < eps):\n warnings.warn(\"sde_Periodic: Infinite, too small, or too large (eps={0:e}) values in q2 :\".format(eps) + q2.__format__(\"\") )\n \n if np.any( np.isfinite(dq2l) == False) or np.any( np.abs(dq2l) > 1.0/eps) or np.any( np.abs(dq2l) < eps):\n warnings.warn(\"sde_Periodic: Infinite, too small, or too large (eps={0:e}) values in dq2l :\".format(eps) + q2.__format__(\"\") )\n \n \n F = np.kron(np.diag(range(0,N+1)),np.array( ((0, -w0), (w0, 0)) ) )\n L = np.eye(2*(N+1))\n Qc = np.zeros((2*(N+1), 2*(N+1)))\n P_inf = np.kron(np.diag(q2),np.eye(2))\n H = np.kron(np.ones((1,N+1)),np.array((1,0)) )\n P0 = P_inf.copy()\n \n # Derivatives\n dF = np.empty((F.shape[0], F.shape[1], 3))\n dQc = np.empty((Qc.shape[0], Qc.shape[1], 3))\n dP_inf = np.empty((P_inf.shape[0], P_inf.shape[1], 3)) \n \n # Derivatives wrt self.variance\n dF[:,:,0] = np.zeros(F.shape)\n dQc[:,:,0] = np.zeros(Qc.shape)\n dP_inf[:,:,0] = P_inf / p_variance\n\n # Derivatives self.period\n dF[:,:,1] = np.kron(np.diag(range(0,N+1)),np.array( ((0, w0), (-w0, 0)) ) / p_period );\n dQc[:,:,1] = np.zeros(Qc.shape)\n dP_inf[:,:,1] = np.zeros(P_inf.shape) \n \n # Derivatives self.lengthscales \n dF[:,:,2] = np.zeros(F.shape)\n dQc[:,:,2] = np.zeros(Qc.shape)\n dP_inf[:,:,2] = np.kron(np.diag(dq2l),np.eye(2))\n dP0 = dP_inf.copy()\n \n if self.balance:\n # Benefits of this are not very sound.\n import GPy.models.state_space_main as ssm\n (F, L, Qc, H, P_inf, P0, dF, dQc, dP_inf,dP0) = ssm.balance_ss_model(F, L, Qc, H, P_inf, P0, dF, dQc, dP_inf, dP0 )\n \n return (F, L, Qc, H, P_inf, P0, dF, dQc, dP_inf, dP0)", "response": "Return the state space representation of the standard periodic covariance."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef pdf_link(self, link_f, y, Y_metadata=None):\n assert np.atleast_1d(link_f).shape == np.atleast_1d(y).shape\n log_objective = link_f*np.exp(-y*link_f)\n return np.exp(np.sum(np.log(log_objective)))", "response": "Evaluate the pdf of a latent variable link."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef logpdf_link(self, link_f, y, Y_metadata=None):\n log_objective = np.log(link_f) - y*link_f\n return log_objective", "response": "Log PDF for the given link"} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nevaluates the log PDF for a given latent variables link f and y.", "response": "def pdf_link(self, link_f, y, Y_metadata=None):\n \"\"\"\n Likelihood function given link(f)\n\n .. math::\n p(y_{i}|\\\\lambda(f_{i})) = \\\\frac{\\\\lambda(f_{i})^{y_{i}}}{y_{i}!}e^{-\\\\lambda(f_{i})}\n\n :param link_f: latent variables link(f)\n :type link_f: Nx1 array\n :param y: data\n :type y: Nx1 array\n :param Y_metadata: Y_metadata which is not used in poisson distribution\n :returns: likelihood evaluated for this point\n :rtype: float\n \"\"\"\n assert np.atleast_1d(link_f).shape == np.atleast_1d(y).shape\n return np.exp(self.logpdf_link(link_f, y, Y_metadata))"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef logpdf_link(self, link_f, y, Y_metadata=None):\n return -link_f + y*np.log(link_f) - special.gammaln(y+1)", "response": "Log PDF for the point - by - point"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a set of samples based on a given value of the latent variable.", "response": "def samples(self, gp, Y_metadata=None):\n \"\"\"\n Returns a set of samples of observations based on a given value of the latent variable.\n\n :param gp: latent variable\n \"\"\"\n orig_shape = gp.shape\n gp = gp.flatten()\n # Ysim = np.random.poisson(self.gp_link.transf(gp), [samples, gp.size]).T\n # return Ysim.reshape(orig_shape+(samples,))\n Ysim = np.random.poisson(self.gp_link.transf(gp))\n return Ysim.reshape(orig_shape)"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncalculate the numerical inverse of f. This should be overwritten for specific warping functions where the numerical inverse of f can be found in closed form.", "response": "def f_inv(self, z, max_iterations=250, y=None):\n \"\"\"\n Calculate the numerical inverse of f. This should be\n overwritten for specific warping functions where the\n inverse can be found in closed form.\n\n :param max_iterations: maximum number of N.R. iterations\n \"\"\"\n\n z = z.copy()\n y = np.ones_like(z)\n\n it = 0\n update = np.inf\n while np.abs(update).sum() > 1e-10 and it < max_iterations:\n fy = self.f(y)\n fgrady = self.fgrad_y(y)\n update = (fy - z) / fgrady\n y -= self.rate * update\n it += 1\n #if it == max_iterations:\n # print(\"WARNING!!! Maximum number of iterations reached in f_inv \")\n # print(\"Sum of roots: %.4f\" % np.sum(fy - z))\n return y"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef f(self, y):\n d = self.d\n mpsi = self.psi\n z = d * y.copy()\n for i in range(len(mpsi)):\n a, b, c = mpsi[i]\n z += a * np.tanh(b * (y + c))\n return z", "response": "Transform y with f using parameter vector psi\n "} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef fgrad_y(self, y, return_precalc=False):\n d = self.d\n mpsi = self.psi\n\n # vectorized version\n S = (mpsi[:,1] * (y[:,:,None] + mpsi[:,2])).T\n R = np.tanh(S)\n D = 1 - (R ** 2)\n\n GRAD = (d + (mpsi[:,0:1][:,:,None] * mpsi[:,1:2][:,:,None] * D).sum(axis=0)).T\n\n if return_precalc:\n return GRAD, S, R, D\n\n return GRAD", "response": "gradients of f w. r. t to y"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef fgrad_y_psi(self, y, return_covar_chain=False):\n mpsi = self.psi\n\n w, s, r, d = self.fgrad_y(y, return_precalc=True)\n gradients = np.zeros((y.shape[0], y.shape[1], len(mpsi), 4))\n for i in range(len(mpsi)):\n a,b,c = mpsi[i]\n gradients[:, :, i, 0] = (b * (1.0/np.cosh(s[i])) ** 2).T\n gradients[:, :, i, 1] = a * (d[i] - 2.0 * s[i] * r[i] * (1.0/np.cosh(s[i])) ** 2).T\n gradients[:, :, i, 2] = (-2.0 * a * (b ** 2) * r[i] * ((1.0 / np.cosh(s[i])) ** 2)).T\n gradients[:, :, 0, 3] = 1.0\n\n if return_covar_chain:\n covar_grad_chain = np.zeros((y.shape[0], y.shape[1], len(mpsi), 4))\n for i in range(len(mpsi)):\n a,b,c = mpsi[i]\n covar_grad_chain[:, :, i, 0] = (r[i]).T\n covar_grad_chain[:, :, i, 1] = (a * (y + c) * ((1.0 / np.cosh(s[i])) ** 2).T)\n covar_grad_chain[:, :, i, 2] = a * b * ((1.0 / np.cosh(s[i])) ** 2).T\n covar_grad_chain[:, :, 0, 3] = y\n return gradients, covar_grad_chain\n\n return gradients", "response": "gradients of f w. r. t to y and psi\n "} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef inv_std_norm_cdf(x):\n z = 2*x -1\n ln1z2 = np.log(1-z**2)\n a = 8*(np.pi -3)/(3*np.pi*(4-np.pi))\n b = 2/(np.pi * a) + ln1z2/2\n inv_erf = np.sign(z) * np.sqrt( np.sqrt(b**2 - ln1z2/a) - b )\n return np.sqrt(2) * inv_erf", "response": "Inverse cumulative standard normal distribution of the given value."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef cdfNormal(z):\n if (abs(z) < ERF_CODY_LIMIT1):\n # Phi(z) approx (1+y R_3(y^2))/2, y=z/sqrt(2)\n return 0.5 * (1.0 + (z / M_SQRT2) * _erfRationalHelperR3(0.5 * z * z))\n elif (z < 0.0):\n # Phi(z) approx N(z)Q(-z)/(-z), z<0\n return np.exp(logPdfNormal(z)) * _erfRationalHelper(-z) / (-z)\n else:\n return 1.0 - np.exp(logPdfNormal(z)) * _erfRationalHelper(z) / z", "response": "Implementation of cdf of a standard normal."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef derivLogCdfNormal(z):\n if (abs(z) < ERF_CODY_LIMIT1):\n # Phi(z) approx (1 + y R_3(y^2))/2, y = z/sqrt(2)\n return 2.0 * np.exp(logPdfNormal(z)) / (1.0 + (z / M_SQRT2) * _erfRationalHelperR3(0.5 * z * z))\n elif (z < 0.0):\n # Phi(z) approx N(z) Q(-z)/(-z), z<0\n return -z / _erfRationalHelper(-z)\n else:\n t = np.exp(logPdfNormal(z))\n return t / (1.0 - t * _erfRationalHelper(z) / z)", "response": "Derivative of the log cdf of a standard normal."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ntaking a numpy array of integers and returns a nested list of slices such that the slices describe the start and stop points for each integer in the index.", "response": "def index_to_slices(index):\n \"\"\"\n take a numpy array of integers (index) and return a nested list of slices such that the slices describe the start, stop points for each integer in the index.\n\n e.g.\n >>> index = np.asarray([0,0,0,1,1,1,2,2,2])\n returns\n >>> [[slice(0,3,None)],[slice(3,6,None)],[slice(6,9,None)]]\n\n or, a more complicated example\n >>> index = np.asarray([0,0,1,1,0,2,2,2,1,1])\n returns\n >>> [[slice(0,2,None),slice(4,5,None)],[slice(2,4,None),slice(8,10,None)],[slice(5,8,None)]]\n \"\"\"\n if len(index)==0:\n return[]\n\n #contruct the return structure\n ind = np.asarray(index,dtype=np.int)\n ret = [[] for i in range(ind.max()+1)]\n\n #find the switchpoints\n ind_ = np.hstack((ind,ind[0]+ind[-1]+1))\n switchpoints = np.nonzero(ind_ - np.roll(ind_,+1))[0]\n\n [ret[ind_i].append(slice(*indexes_i)) for ind_i,indexes_i in zip(ind[switchpoints[:-1]],zip(switchpoints,switchpoints[1:]))]\n return ret"} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nasks user for agreeing to data set licenses.", "response": "def prompt_user(prompt):\n \"\"\"Ask user for agreeing to data set licenses.\"\"\"\n # raw_input returns the empty string for \"enter\"\n yes = set(['yes', 'y'])\n no = set(['no','n'])\n\n try:\n print(prompt)\n choice = input().lower()\n # would like to test for exception here, but not sure if we can do that without importing IPython\n except:\n print('Stdin is not implemented.')\n print('You need to set')\n print('overide_manual_authorize=True')\n print('to proceed with the download. Please set that variable and continue.')\n raise\n\n\n if choice in yes:\n return True\n elif choice in no:\n return False\n else:\n print((\"Your response was a \" + choice))\n print(\"Please respond with 'yes', 'y' or 'no', 'n'\")"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef data_available(dataset_name=None):\n try:\n from itertools import zip_longest\n except ImportError:\n from itertools import izip_longest as zip_longest\n dr = data_resources[dataset_name]\n zip_urls = (dr['files'], )\n if 'save_names' in dr: zip_urls += (dr['save_names'], )\n else: zip_urls += ([],)\n\n for file_list, save_list in zip_longest(*zip_urls, fillvalue=[]):\n for f, s in zip_longest(file_list, save_list, fillvalue=None):\n if s is not None: f=s # If there is a save_name given, use that one\n if not os.path.exists(os.path.join(data_path, dataset_name, f)):\n return False\n return True", "response": "Check if the data set is available on the local machine already."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ndownload a file from a url and save it to disk.", "response": "def download_url(url, store_directory, save_name=None, messages=True, suffix=''):\n \"\"\"Download a file from a url and save it to disk.\"\"\"\n i = url.rfind('/')\n file = url[i+1:]\n print(file)\n dir_name = os.path.join(data_path, store_directory)\n\n if save_name is None: save_name = os.path.join(dir_name, file)\n else: save_name = os.path.join(dir_name, save_name)\n\n if suffix is None: suffix=''\n\n print(\"Downloading \", url, \"->\", save_name)\n if not os.path.exists(dir_name):\n os.makedirs(dir_name)\n try:\n response = urlopen(url+suffix)\n except URLError as e:\n if not hasattr(e, \"code\"):\n raise\n response = e\n if response.code > 399 and response.code<500:\n raise ValueError('Tried url ' + url + suffix + ' and received client error ' + str(response.code))\n elif response.code > 499:\n raise ValueError('Tried url ' + url + suffix + ' and received server error ' + str(response.code))\n with open(save_name, 'wb') as f:\n meta = response.info()\n content_length_str = meta.get(\"Content-Length\")\n if content_length_str:\n file_size = int(content_length_str)\n else:\n file_size = None\n status = \"\"\n file_size_dl = 0\n block_sz = 8192\n line_length=30\n while True:\n buff = response.read(block_sz)\n if not buff:\n break\n file_size_dl += len(buff)\n f.write(buff)\n sys.stdout.write(\" \"*(len(status)) + \"\\r\")\n if file_size:\n status = r\"[{perc: <{ll}}] {dl:7.3f}/{full:.3f}MB\".format(dl=file_size_dl/(1048576.),\n full=file_size/(1048576.), ll=line_length,\n perc=\"=\"*int(line_length*float(file_size_dl)/file_size))\n else:\n status = r\"[{perc: <{ll}}] {dl:7.3f}MB\".format(dl=file_size_dl/(1048576.),\n ll=line_length,\n perc=\".\"*int(line_length*float(file_size_dl/(10*1048576.))))\n\n sys.stdout.write(status)\n sys.stdout.flush()\n sys.stdout.write(\" \"*(len(status)) + \"\\r\")\n print(status)"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nchecking with the user that the user has the ability to download the data set.", "response": "def authorize_download(dataset_name=None):\n \"\"\"Check with the user that the are happy with terms and conditions for the data set.\"\"\"\n print(('Acquiring resource: ' + dataset_name))\n # TODO, check resource is in dictionary!\n print('')\n dr = data_resources[dataset_name]\n print('Details of data: ')\n print((dr['details']))\n print('')\n if dr['citation']:\n print('Please cite:')\n print((dr['citation']))\n print('')\n if dr['size']:\n print(('After downloading the data will take up ' + str(dr['size']) + ' bytes of space.'))\n print('')\n print(('Data will be stored in ' + os.path.join(data_path, dataset_name) + '.'))\n print('')\n if overide_manual_authorize:\n if dr['license']:\n print('You have agreed to the following license:')\n print((dr['license']))\n print('')\n return True\n else:\n if dr['license']:\n print('You must also agree to the following license:')\n print((dr['license']))\n print('')\n return prompt_user('Do you wish to proceed with the download? [yes/no]')"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef download_data(dataset_name=None):\n try:\n from itertools import zip_longest\n except ImportError:\n from itertools import izip_longest as zip_longest\n\n dr = data_resources[dataset_name]\n if not authorize_download(dataset_name):\n raise Exception(\"Permission to download data set denied.\")\n\n zip_urls = (dr['urls'], dr['files'])\n\n if 'save_names' in dr: zip_urls += (dr['save_names'], )\n else: zip_urls += ([],)\n\n if 'suffices' in dr: zip_urls += (dr['suffices'], )\n else: zip_urls += ([],)\n\n for url, files, save_names, suffices in zip_longest(*zip_urls, fillvalue=[]):\n for f, save_name, suffix in zip_longest(files, save_names, suffices, fillvalue=None):\n download_url(os.path.join(url,f), dataset_name, save_name, suffix=suffix)\n\n return True", "response": "Download the data set."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef cmu_urls_files(subj_motions, messages = True):\n '''\n Find which resources are missing on the local disk for the requested CMU motion capture motions.\n '''\n dr = data_resources['cmu_mocap_full']\n cmu_url = dr['urls'][0]\n\n subjects_num = subj_motions[0]\n motions_num = subj_motions[1]\n\n resource = {'urls' : [], 'files' : []}\n # Convert numbers to strings\n subjects = []\n motions = [list() for _ in range(len(subjects_num))]\n for i in range(len(subjects_num)):\n curSubj = str(int(subjects_num[i]))\n if int(subjects_num[i]) < 10:\n curSubj = '0' + curSubj\n subjects.append(curSubj)\n for j in range(len(motions_num[i])):\n curMot = str(int(motions_num[i][j]))\n if int(motions_num[i][j]) < 10:\n curMot = '0' + curMot\n motions[i].append(curMot)\n\n all_skels = []\n\n assert len(subjects) == len(motions)\n\n all_motions = []\n\n for i in range(len(subjects)):\n skel_dir = os.path.join(data_path, 'cmu_mocap')\n cur_skel_file = os.path.join(skel_dir, subjects[i] + '.asf')\n\n url_required = False\n file_download = []\n if not os.path.exists(cur_skel_file):\n # Current skel file doesn't exist.\n if not os.path.isdir(skel_dir):\n os.makedirs(skel_dir)\n # Add skel file to list.\n url_required = True\n file_download.append(subjects[i] + '.asf')\n for j in range(len(motions[i])):\n file_name = subjects[i] + '_' + motions[i][j] + '.amc'\n cur_motion_file = os.path.join(skel_dir, file_name)\n if not os.path.exists(cur_motion_file):\n url_required = True\n file_download.append(subjects[i] + '_' + motions[i][j] + '.amc')\n if url_required:\n resource['urls'].append(cmu_url + '/' + subjects[i] + '/')\n resource['files'].append(file_download)\n return resource", "response": "Return a list of CMU URLs files for the requested CMU motion capture motions."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef football_data(season='1314', data_set='football_data'):\n def league2num(string):\n league_dict = {'E0':0, 'E1':1, 'E2': 2, 'E3': 3, 'EC':4}\n return league_dict[string]\n\n def football2num(string):\n if string in football_dict:\n return football_dict[string]\n else:\n football_dict[string] = len(football_dict)+1\n return len(football_dict)+1\n\n data_set_season = data_set + '_' + season\n data_resources[data_set_season] = copy.deepcopy(data_resources[data_set])\n data_resources[data_set_season]['urls'][0]+=season + '/'\n start_year = int(season[0:2])\n end_year = int(season[2:4])\n files = ['E0.csv', 'E1.csv', 'E2.csv', 'E3.csv']\n if start_year>4 and start_year < 93:\n files += ['EC.csv']\n data_resources[data_set_season]['files'] = [files]\n if not data_available(data_set_season):\n download_data(data_set_season)\n from matplotlib import pyplot as pb\n for file in reversed(files):\n filename = os.path.join(data_path, data_set_season, file)\n # rewrite files removing blank rows.\n writename = os.path.join(data_path, data_set_season, 'temp.csv')\n input = open(filename, 'rb')\n output = open(writename, 'wb')\n writer = csv.writer(output)\n for row in csv.reader(input):\n if any(field.strip() for field in row):\n writer.writerow(row)\n input.close()\n output.close()\n table = np.loadtxt(writename,skiprows=1, usecols=(0, 1, 2, 3, 4, 5), converters = {0: league2num, 1: pb.datestr2num, 2:football2num, 3:football2num}, delimiter=',')\n X = table[:, :4]\n Y = table[:, 4:]\n return data_details_return({'X': X, 'Y': Y}, data_set)", "response": "Download football data from English games since 1993."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef oil(data_set='three_phase_oil_flow'):\n if not data_available(data_set):\n download_data(data_set)\n oil_train_file = os.path.join(data_path, data_set, 'DataTrn.txt')\n oil_trainlbls_file = os.path.join(data_path, data_set, 'DataTrnLbls.txt')\n oil_test_file = os.path.join(data_path, data_set, 'DataTst.txt')\n oil_testlbls_file = os.path.join(data_path, data_set, 'DataTstLbls.txt')\n oil_valid_file = os.path.join(data_path, data_set, 'DataVdn.txt')\n oil_validlbls_file = os.path.join(data_path, data_set, 'DataVdnLbls.txt')\n fid = open(oil_train_file)\n X = np.fromfile(fid, sep='\\t').reshape((-1, 12))\n fid.close()\n fid = open(oil_test_file)\n Xtest = np.fromfile(fid, sep='\\t').reshape((-1, 12))\n fid.close()\n fid = open(oil_valid_file)\n Xvalid = np.fromfile(fid, sep='\\t').reshape((-1, 12))\n fid.close()\n fid = open(oil_trainlbls_file)\n Y = np.fromfile(fid, sep='\\t').reshape((-1, 3)) * 2. - 1.\n fid.close()\n fid = open(oil_testlbls_file)\n Ytest = np.fromfile(fid, sep='\\t').reshape((-1, 3)) * 2. - 1.\n fid.close()\n fid = open(oil_validlbls_file)\n Yvalid = np.fromfile(fid, sep='\\t').reshape((-1, 3)) * 2. - 1.\n fid.close()\n return data_details_return({'X': X, 'Y': Y, 'Xtest': Xtest, 'Ytest': Ytest, 'Xtest' : Xtest, 'Xvalid': Xvalid, 'Yvalid': Yvalid}, data_set)", "response": "The three phase oil data from Bishop and James 2003."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef hapmap3(data_set='hapmap3'):\n try:\n from pandas import read_pickle, DataFrame\n from sys import stdout\n import bz2\n except ImportError as i:\n raise i(\"Need pandas for hapmap dataset, make sure to install pandas (http://pandas.pydata.org/) before loading the hapmap dataset\")\n\n dir_path = os.path.join(data_path,'hapmap3')\n hapmap_file_name = 'hapmap3_r2_b36_fwd.consensus.qc.poly'\n unpacked_files = [os.path.join(dir_path, hapmap_file_name+ending) for ending in ['.ped', '.map']]\n unpacked_files_exist = reduce(lambda a, b:a and b, map(os.path.exists, unpacked_files))\n\n if not unpacked_files_exist and not data_available(data_set):\n download_data(data_set)\n\n preprocessed_data_paths = [os.path.join(dir_path,hapmap_file_name + file_name) for file_name in \\\n ['.snps.pickle',\n '.info.pickle',\n '.nan.pickle']]\n\n if not reduce(lambda a,b: a and b, map(os.path.exists, preprocessed_data_paths)):\n if not overide_manual_authorize and not prompt_user(\"Preprocessing requires ~25GB \"\n \"of memory and can take a (very) long time, continue? [Y/n]\"):\n print(\"Preprocessing required for further usage.\")\n return\n status = \"Preprocessing data, please be patient...\"\n print(status)\n def write_status(message, progress, status):\n stdout.write(\" \"*len(status)); stdout.write(\"\\r\"); stdout.flush()\n status = r\"[{perc: <{ll}}] {message: <13s}\".format(message=message, ll=20,\n perc=\"=\"*int(20.*progress/100.))\n stdout.write(status); stdout.flush()\n return status\n if not unpacked_files_exist:\n status=write_status('unpacking...', 0, '')\n curr = 0\n for newfilepath in unpacked_files:\n if not os.path.exists(newfilepath):\n filepath = newfilepath + '.bz2'\n file_size = os.path.getsize(filepath)\n with open(newfilepath, 'wb') as new_file, open(filepath, 'rb') as f:\n decomp = bz2.BZ2Decompressor()\n file_processed = 0\n buffsize = 100 * 1024\n for data in iter(lambda : f.read(buffsize), b''):\n new_file.write(decomp.decompress(data))\n file_processed += len(data)\n status=write_status('unpacking...', curr+12.*file_processed/(file_size), status)\n curr += 12\n status=write_status('unpacking...', curr, status)\n os.remove(filepath)\n status=write_status('reading .ped...', 25, status)\n # Preprocess data:\n snpstrnp = np.loadtxt(unpacked_files[0], dtype=str)\n status=write_status('reading .map...', 33, status)\n mapnp = np.loadtxt(unpacked_files[1], dtype=str)\n status=write_status('reading relationships.txt...', 42, status)\n # and metainfo:\n infodf = DataFrame.from_csv(os.path.join(dir_path,'./relationships_w_pops_121708.txt'), header=0, sep='\\t')\n infodf.set_index('IID', inplace=1)\n status=write_status('filtering nan...', 45, status)\n snpstr = snpstrnp[:,6:].astype('S1').reshape(snpstrnp.shape[0], -1, 2)\n inan = snpstr[:,:,0] == '0'\n status=write_status('filtering reference alleles...', 55, status)\n ref = np.array(map(lambda x: np.unique(x)[-2:], snpstr.swapaxes(0,1)[:,:,:]))\n status=write_status('encoding snps...', 70, status)\n # Encode the information for each gene in {-1,0,1}:\n status=write_status('encoding snps...', 73, status)\n snps = (snpstr==ref[None,:,:])\n status=write_status('encoding snps...', 76, status)\n snps = (snps*np.array([1,-1])[None,None,:])\n status=write_status('encoding snps...', 78, status)\n snps = snps.sum(-1)\n status=write_status('encoding snps...', 81, status)\n snps = snps.astype('i8')\n status=write_status('marking nan values...', 88, status)\n # put in nan values (masked as -128):\n snps[inan] = -128\n status=write_status('setting up meta...', 94, status)\n # get meta information:\n metaheader = np.r_[['family_id', 'iid', 'paternal_id', 'maternal_id', 'sex', 'phenotype']]\n metadf = DataFrame(columns=metaheader, data=snpstrnp[:,:6])\n metadf.set_index('iid', inplace=1)\n metadf = metadf.join(infodf.population)\n metadf.to_pickle(preprocessed_data_paths[1])\n # put everything together:\n status=write_status('setting up snps...', 96, status)\n snpsdf = DataFrame(index=metadf.index, data=snps, columns=mapnp[:,1])\n with open(preprocessed_data_paths[0], 'wb') as f:\n pickle.dump(f, snpsdf, protocoll=-1)\n status=write_status('setting up snps...', 98, status)\n inandf = DataFrame(index=metadf.index, data=inan, columns=mapnp[:,1])\n inandf.to_pickle(preprocessed_data_paths[2])\n status=write_status('done :)', 100, status)\n print('')\n else:\n print(\"loading snps...\")\n snpsdf = read_pickle(preprocessed_data_paths[0])\n print(\"loading metainfo...\")\n metadf = read_pickle(preprocessed_data_paths[1])\n print(\"loading nan entries...\")\n inandf = read_pickle(preprocessed_data_paths[2])\n snps = snpsdf.values\n populations = metadf.population.values.astype('S3')\n hapmap = dict(name=data_set,\n description='The HapMap phase three SNP dataset - '\n '1184 samples out of 11 populations. inan is a '\n 'boolean array, containing wheather or not the '\n 'given entry is nan (nans are masked as '\n '-128 in snps).',\n snpsdf=snpsdf,\n metadf=metadf,\n snps=snps,\n inan=inandf.values,\n inandf=inandf,\n populations=populations)\n return hapmap", "response": "This function loads the HapMap 3 SNP dataset and returns a dictionary containing the information for the base pair of SNPs and meta information."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef toy_rbf_1d(seed=default_seed, num_samples=500):\n np.random.seed(seed=seed)\n num_in = 1\n X = np.random.uniform(low= -1.0, high=1.0, size=(num_samples, num_in))\n X.sort(axis=0)\n rbf = GPy.kern.RBF(num_in, variance=1., lengthscale=np.array((0.25,)))\n white = GPy.kern.White(num_in, variance=1e-2)\n kernel = rbf + white\n K = kernel.K(X)\n y = np.reshape(np.random.multivariate_normal(np.zeros(num_samples), K), (num_samples, 1))\n return {'X':X, 'Y':y, 'info': \"Sampled \" + str(num_samples) + \" values of a function from an RBF covariance with very small noise for inputs uniformly distributed between -1 and 1.\"}", "response": "Generate a 1D toy RBF dataset."} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ngenerate a crescent data set from a mixture of four Gaussians.", "response": "def crescent_data(num_data=200, seed=default_seed):\n \"\"\"\nData set formed from a mixture of four Gaussians. In each class two of the Gaussians are elongated at right angles to each other and offset to form an approximation to the crescent data that is popular in semi-supervised learning as a toy problem.\n\n :param num_data_part: number of data to be sampled (default is 200).\n :type num_data: int\n :param seed: random seed to be used for data generation.\n :type seed: int\n\n \"\"\"\n np.random.seed(seed=seed)\n sqrt2 = np.sqrt(2)\n # Rotation matrix\n R = np.array([[sqrt2 / 2, -sqrt2 / 2], [sqrt2 / 2, sqrt2 / 2]])\n # Scaling matrices\n scales = []\n scales.append(np.array([[3, 0], [0, 1]]))\n scales.append(np.array([[3, 0], [0, 1]]))\n scales.append([[1, 0], [0, 3]])\n scales.append([[1, 0], [0, 3]])\n means = []\n means.append(np.array([4, 4]))\n means.append(np.array([0, 4]))\n means.append(np.array([-4, -4]))\n means.append(np.array([0, -4]))\n\n Xparts = []\n num_data_part = []\n num_data_total = 0\n for i in range(0, 4):\n num_data_part.append(round(((i + 1) * num_data) / 4.))\n num_data_part[i] -= num_data_total\n part = np.random.normal(size=(num_data_part[i], 2))\n part = np.dot(np.dot(part, scales[i]), R) + means[i]\n Xparts.append(part)\n num_data_total += num_data_part[i]\n X = np.vstack((Xparts[0], Xparts[1], Xparts[2], Xparts[3]))\n\n Y = np.vstack((np.ones((num_data_part[0] + num_data_part[1], 1)), -np.ones((num_data_part[2] + num_data_part[3], 1))))\n return {'X':X, 'Y':Y, 'info': \"Two separate classes of data formed approximately in the shape of two crescents.\"}"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef cmu_mocap_49_balance(data_set='cmu_mocap'):\n train_motions = ['18', '19']\n test_motions = ['20']\n data = cmu_mocap('49', train_motions, test_motions, sample_every=4, data_set=data_set)\n data['info'] = \"One legged balancing motions from CMU data base subject 49. As used in Alvarez, Luengo and Lawrence at AISTATS 2009. It consists of \" + data['info']\n return data", "response": "Load CMU subject 49 s one legged balancing motion that was used by Alvarez Luengo and Lawrence at AISTATS 2009."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef cmu_mocap_35_walk_jog(data_set='cmu_mocap'):\n train_motions = ['01', '02', '03', '04', '05', '06',\n '07', '08', '09', '10', '11', '12',\n '13', '14', '15', '16', '17', '19',\n '20', '21', '22', '23', '24', '25',\n '26', '28', '30', '31', '32', '33', '34']\n test_motions = ['18', '29']\n data = cmu_mocap('35', train_motions, test_motions, sample_every=4, data_set=data_set)\n data['info'] = \"Walk and jog data from CMU data base subject 35. As used in Tayor, Roweis and Hinton at NIPS 2007, but without their pre-processing (i.e. as used by Lawrence at AISTATS 2007). It consists of \" + data['info']\n return data", "response": "Load CMU subject 35 s walking and jogging motions the same data that was used by Taylor Roweis and Hinton at AISTATS 2007 but without their preprocessing. Also used by Lawrence at AISTATS 2007."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef cmu_mocap(subject, train_motions, test_motions=[], sample_every=4, data_set='cmu_mocap'):\n # Load in subject skeleton.\n subject_dir = os.path.join(data_path, data_set)\n\n # Make sure the data is downloaded.\n all_motions = train_motions + test_motions\n resource = cmu_urls_files(([subject], [all_motions]))\n data_resources[data_set] = data_resources['cmu_mocap_full'].copy()\n data_resources[data_set]['files'] = resource['files']\n data_resources[data_set]['urls'] = resource['urls']\n if resource['urls']:\n download_data(data_set)\n\n skel = GPy.util.mocap.acclaim_skeleton(os.path.join(subject_dir, subject + '.asf'))\n\n # Set up labels for each sequence\n exlbls = np.eye(len(train_motions))\n\n # Load sequences\n tot_length = 0\n temp_Y = []\n temp_lbls = []\n for i in range(len(train_motions)):\n temp_chan = skel.load_channels(os.path.join(subject_dir, subject + '_' + train_motions[i] + '.amc'))\n temp_Y.append(temp_chan[::sample_every, :])\n temp_lbls.append(np.tile(exlbls[i, :], (temp_Y[i].shape[0], 1)))\n tot_length += temp_Y[i].shape[0]\n\n Y = np.zeros((tot_length, temp_Y[0].shape[1]))\n lbls = np.zeros((tot_length, temp_lbls[0].shape[1]))\n\n end_ind = 0\n for i in range(len(temp_Y)):\n start_ind = end_ind\n end_ind += temp_Y[i].shape[0]\n Y[start_ind:end_ind, :] = temp_Y[i]\n lbls[start_ind:end_ind, :] = temp_lbls[i]\n if len(test_motions) > 0:\n temp_Ytest = []\n temp_lblstest = []\n\n testexlbls = np.eye(len(test_motions))\n tot_test_length = 0\n for i in range(len(test_motions)):\n temp_chan = skel.load_channels(os.path.join(subject_dir, subject + '_' + test_motions[i] + '.amc'))\n temp_Ytest.append(temp_chan[::sample_every, :])\n temp_lblstest.append(np.tile(testexlbls[i, :], (temp_Ytest[i].shape[0], 1)))\n tot_test_length += temp_Ytest[i].shape[0]\n\n # Load test data\n Ytest = np.zeros((tot_test_length, temp_Ytest[0].shape[1]))\n lblstest = np.zeros((tot_test_length, temp_lblstest[0].shape[1]))\n\n end_ind = 0\n for i in range(len(temp_Ytest)):\n start_ind = end_ind\n end_ind += temp_Ytest[i].shape[0]\n Ytest[start_ind:end_ind, :] = temp_Ytest[i]\n lblstest[start_ind:end_ind, :] = temp_lblstest[i]\n else:\n Ytest = None\n lblstest = None\n\n info = 'Subject: ' + subject + '. Training motions: '\n for motion in train_motions:\n info += motion + ', '\n info = info[:-2]\n if len(test_motions) > 0:\n info += '. Test motions: '\n for motion in test_motions:\n info += motion + ', '\n info = info[:-2] + '.'\n else:\n info += '.'\n if sample_every != 1:\n info += ' Data is sub-sampled to every ' + str(sample_every) + ' frames.'\n return data_details_return({'Y': Y, 'lbls' : lbls, 'Ytest': Ytest, 'lblstest' : lblstest, 'info': info, 'skel': skel}, data_set)", "response": "Load a given subject s training and test motions from the CMU motion capture data."} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nupdating the gradients of the kernel with respect to the parameters", "response": "def update_gradients_full(self, dL_dK, X, X2=None):\n \"\"\"derivative of the covariance matrix with respect to the parameters (shape is N x num_inducing x num_params)\"\"\"\n if X2 is None: X2 = X\n FX = self._cos(self.basis_alpha[None,:],self.basis_omega[None,:],self.basis_phi[None,:])(X)\n FX2 = self._cos(self.basis_alpha[None,:],self.basis_omega[None,:],self.basis_phi[None,:])(X2)\n\n La = np.column_stack((self.a[0]*np.ones((self.n_basis,1)),self.a[1]*self.basis_omega))\n Lo = np.column_stack((self.basis_omega,self.basis_omega))\n Lp = np.column_stack((self.basis_phi,self.basis_phi+np.pi/2))\n r,omega,phi = self._cos_factorization(La,Lo,Lp)\n Gint = self._int_computation( r,omega,phi, r,omega,phi)\n\n Flower = np.array(self._cos(self.basis_alpha,self.basis_omega,self.basis_phi)(self.lower))[:,None]\n\n #dK_dvar\n dK_dvar = 1./self.variance*mdot(FX,self.Gi,FX2.T)\n\n #dK_dlen\n da_dlen = [-1./self.lengthscale**2,0.]\n dLa_dlen = np.column_stack((da_dlen[0]*np.ones((self.n_basis,1)),da_dlen[1]*self.basis_omega))\n r1,omega1,phi1 = self._cos_factorization(dLa_dlen,Lo,Lp)\n dGint_dlen = self._int_computation(r1,omega1,phi1, r,omega,phi)\n dGint_dlen = dGint_dlen + dGint_dlen.T\n dG_dlen = 1./2*Gint + self.lengthscale/2*dGint_dlen\n dK_dlen = -mdot(FX,self.Gi,dG_dlen/self.variance,self.Gi,FX2.T)\n\n #dK_dper\n dFX_dper = self._cos(-self.basis_alpha[None,:]*self.basis_omega[None,:]/self.period*X ,self.basis_omega[None,:],self.basis_phi[None,:]+np.pi/2)(X)\n dFX2_dper = self._cos(-self.basis_alpha[None,:]*self.basis_omega[None,:]/self.period*X2,self.basis_omega[None,:],self.basis_phi[None,:]+np.pi/2)(X2)\n\n dLa_dper = np.column_stack((-self.a[0]*self.basis_omega/self.period, -self.a[1]*self.basis_omega**2/self.period))\n dLp_dper = np.column_stack((self.basis_phi+np.pi/2,self.basis_phi+np.pi))\n r1,omega1,phi1 = self._cos_factorization(dLa_dper,Lo,dLp_dper)\n\n IPPprim1 = self.upper*(1./(omega+omega1.T)*np.cos((omega+omega1.T)*self.upper+phi+phi1.T-np.pi/2) + 1./(omega-omega1.T)*np.cos((omega-omega1.T)*self.upper+phi-phi1.T-np.pi/2))\n IPPprim1 -= self.lower*(1./(omega+omega1.T)*np.cos((omega+omega1.T)*self.lower+phi+phi1.T-np.pi/2) + 1./(omega-omega1.T)*np.cos((omega-omega1.T)*self.lower+phi-phi1.T-np.pi/2))\n # SIMPLIFY!!! IPPprim1 = (self.upper - self.lower)*np.cos((omega+omega1.T)*self.upper+phi+phi1.T-np.pi/2) + 1./(omega-omega1.T)*np.cos((omega-omega1.T)*self.upper+phi-phi1.T-np.pi/2))\n IPPprim2 = self.upper*(1./(omega+omega1.T)*np.cos((omega+omega1.T)*self.upper+phi+phi1.T-np.pi/2) + self.upper*np.cos(phi-phi1.T))\n IPPprim2 -= self.lower*(1./(omega+omega1.T)*np.cos((omega+omega1.T)*self.lower+phi+phi1.T-np.pi/2) + self.lower*np.cos(phi-phi1.T))\n IPPprim = np.where(np.logical_or(np.isnan(IPPprim1), np.isinf(IPPprim1)), IPPprim2, IPPprim1)\n\n\n IPPint1 = 1./(omega+omega1.T)**2*np.cos((omega+omega1.T)*self.upper+phi+phi1.T-np.pi) + 1./(omega-omega1.T)**2*np.cos((omega-omega1.T)*self.upper+phi-phi1.T-np.pi)\n IPPint1 -= 1./(omega+omega1.T)**2*np.cos((omega+omega1.T)*self.lower+phi+phi1.T-np.pi) + 1./(omega-omega1.T)**2*np.cos((omega-omega1.T)*self.lower+phi-phi1.T-np.pi)\n IPPint2 = 1./(omega+omega1.T)**2*np.cos((omega+omega1.T)*self.upper+phi+phi1.T-np.pi) + 1./2*self.upper**2*np.cos(phi-phi1.T)\n IPPint2 -= 1./(omega+omega1.T)**2*np.cos((omega+omega1.T)*self.lower+phi+phi1.T-np.pi) + 1./2*self.lower**2*np.cos(phi-phi1.T)\n #IPPint2[0,0] = (self.upper**2 - self.lower**2)*np.cos(phi[0,0])*np.cos(phi1[0,0])\n IPPint = np.where(np.isnan(IPPint1),IPPint2,IPPint1)\n\n dLa_dper2 = np.column_stack((-self.a[1]*self.basis_omega/self.period))\n dLp_dper2 = np.column_stack((self.basis_phi+np.pi/2))\n r2,omega2,phi2 = dLa_dper2.T,Lo[:,0:1],dLp_dper2.T\n\n dGint_dper = np.dot(r,r1.T)/2 * (IPPprim - IPPint) + self._int_computation(r2,omega2,phi2, r,omega,phi)\n dGint_dper = dGint_dper + dGint_dper.T\n\n dFlower_dper = np.array(self._cos(-self.lower*self.basis_alpha*self.basis_omega/self.period,self.basis_omega,self.basis_phi+np.pi/2)(self.lower))[:,None]\n\n dG_dper = 1./self.variance*(self.lengthscale/2*dGint_dper + self.b[0]*(np.dot(dFlower_dper,Flower.T)+np.dot(Flower,dFlower_dper.T)))\n\n dK_dper = mdot(dFX_dper,self.Gi,FX2.T) - mdot(FX,self.Gi,dG_dper,self.Gi,FX2.T) + mdot(FX,self.Gi,dFX2_dper.T)\n\n self.variance.gradient = np.sum(dK_dvar*dL_dK)\n self.lengthscale.gradient = np.sum(dK_dlen*dL_dK)\n self.period.gradient = np.sum(dK_dper*dL_dK)"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef set_X_gradients(self, X, X_grad):\n X.mean.gradient, X.variance.gradient = X_grad", "response": "Set the gradients of the posterior distribution of X in its specific form."} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nlogs PDF for the point - by - point", "response": "def logpdf_link(self, link_f, y, Y_metadata=None):\n \"\"\"\n Log Likelihood Function given link(f)\n\n .. math::\n \\\\ln p(y_{i}|\\lambda(f_{i})) = \\\\alpha_{i}\\\\log \\\\beta - \\\\log \\\\Gamma(\\\\alpha_{i}) + (\\\\alpha_{i} - 1)\\\\log y_{i} - \\\\beta y_{i}\\\\\\\\\n \\\\alpha_{i} = \\\\beta y_{i}\n\n :param link_f: latent variables (link(f))\n :type link_f: Nx1 array\n :param y: data\n :type y: Nx1 array\n :param Y_metadata: Y_metadata which is not used in poisson distribution\n :returns: likelihood evaluated for this point\n :rtype: float\n\n \"\"\"\n #alpha = self.gp_link.transf(gp)*self.beta\n #return (1. - alpha)*np.log(obs) + self.beta*obs - alpha * np.log(self.beta) + np.log(special.gamma(alpha))\n alpha = link_f*self.beta\n log_objective = alpha*np.log(self.beta) - np.log(special.gamma(alpha)) + (alpha - 1)*np.log(y) - self.beta*y\n return log_objective"} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn the second derivative of logpdf at y given a given link and a given y.", "response": "def d2logpdf_dlink2(self, link_f, y, Y_metadata=None):\n \"\"\"\n Hessian at y, given link(f), w.r.t link(f)\n i.e. second derivative logpdf at y given link(f_i) and link(f_j) w.r.t link(f_i) and link(f_j)\n The hessian will be 0 unless i == j\n\n .. math::\n \\\\frac{d^{2} \\\\ln p(y_{i}|\\lambda(f_{i}))}{d^{2}\\\\lambda(f)} = -\\\\beta^{2}\\\\frac{d\\\\Psi(\\\\alpha_{i})}{d\\\\alpha_{i}}\\\\\\\\\n \\\\alpha_{i} = \\\\beta y_{i}\n\n :param link_f: latent variables link(f)\n :type link_f: Nx1 array\n :param y: data\n :type y: Nx1 array\n :param Y_metadata: Y_metadata which is not used in gamma distribution\n :returns: Diagonal of hessian matrix (second derivative of likelihood evaluated at points f)\n :rtype: Nx1 array\n\n .. Note::\n Will return diagonal of hessian, since every where else it is 0, as the likelihood factorizes over cases\n (the distribution for y_i depends only on link(f_i) not on link(f_(j!=i))\n \"\"\"\n hess = -special.polygamma(1, self.beta*link_f)*(self.beta**2)\n #old\n #return -self.gp_link.d2transf_df2(gp)*self.beta*np.log(obs) + special.polygamma(1,self.gp_link.transf(gp)*self.beta)*(self.gp_link.dtransf_df(gp)*self.beta)**2 + special.psi(self.gp_link.transf(gp)*self.beta)*self.gp_link.d2transf_df2(gp)*self.beta\n return hess"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef d3logpdf_dlink3(self, link_f, y, Y_metadata=None):\n d3lik_dlink3 = -special.polygamma(2, self.beta*link_f)*(self.beta**3)\n return d3lik_dlink3", "response": "third order derivative log - likelihood function at y given link f given link y given data Y_metadata"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef parameters_changed(self):\n # using the warped X to update\n self.X = self.transform_data(self.X_untransformed)\n super(InputWarpedGP, self).parameters_changed()\n # the gradient of log likelihood w.r.t. input AFTER warping is a product of dL_dK and dK_dX\n dL_dX = self.kern.gradients_X(self.grad_dict['dL_dK'], self.X)\n self.warping_function.update_grads(self.X_untransformed, dL_dX)", "response": "Update the gradients of the kernels and other parameters for warping function\n "} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef transform_data(self, X, test_data=False):\n return self.warping_function.f(X, test_data)", "response": "Apply warping_function to some Input data"} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\npredicts on the new data.", "response": "def predict(self, Xnew):\n \"\"\"Prediction on the new data\n\n Parameters\n ----------\n Xnew : array_like, shape = (n_samples, n_features)\n The test data.\n\n Returns\n -------\n mean : array_like, shape = (n_samples, output.dim)\n Posterior mean at the location of Xnew\n\n var : array_like, shape = (n_samples, 1)\n Posterior variance at the location of Xnew\n \"\"\"\n Xnew_warped = self.transform_data(Xnew, test_data=True)\n mean, var = super(InputWarpedGP, self).predict(Xnew_warped, kern=self.kernel, full_cov=False)\n return mean, var"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _set_namespace(self, namespaces):\n self.namespace = {}\n for m in namespaces[::-1]:\n buf = _get_namespace(m)\n self.namespace.update(buf)\n self.namespace.update(self.__dict__)", "response": "Set the name space for use when calling eval."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nextracting expressions and variables from the user provided expressions.", "response": "def _set_expressions(self, expressions):\n \"\"\"Extract expressions and variables from the user provided expressions.\"\"\"\n self.expressions = {}\n for key, item in expressions.items():\n self.expressions[key] = {'function': item}"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nset the variables that are only stored in the cache and the parameters that are not in the cache.", "response": "def _set_variables(self, cacheable):\n \"\"\"Pull the variable names out of the provided expressions and separate into cacheable expressions and normal parameters. Those that are only stored in the cache, the parameters are stored in this object.\"\"\"\n # pull the parameters and inputs out of the symbolic pdf\n def extract_vars(expr):\n return [e for e in expr.atoms() if e.is_Symbol and e not in vars]\n self.cacheable = cacheable\n self.variables = {}\n vars = []\n for expression in self.expressions.values():\n vars += extract_vars(expression['function'])\n # inputs are assumed to be those things that are\n # cacheable. I.e. those things that aren't stored within the\n # object except as cached. For covariance functions this is X\n # and Z, for likelihoods F and for mapping functions X.\n self.cacheable_vars = [] # list of everything that's cacheable\n for var in cacheable: \n self.variables[var] = [e for e in vars if e.name.split('_')[0]==var.lower()]\n self.cacheable_vars += self.variables[var]\n for var in cacheable:\n if not self.variables[var]:\n raise ValueError('Variable ' + var + ' was specified as cacheable but is not in expression. Expected to find symbols of the form ' + var.lower() + '_0 to represent ' + var)\n\n # things that aren't cacheable are assumed to be parameters.\n self.variables['theta'] = sorted([e for e in vars if not e in self.cacheable_vars],key=lambda e:e.name)"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _set_parameters(self, parameters):\n for theta in self.variables['theta']:\n val = 1.0\n # TODO: improve approach for initializing parameters.\n if parameters is not None:\n if theta.name in parameters:\n val = parameters[theta.name]\n # Add parameter.\n \n self.link_parameters(Param(theta.name, val, None))", "response": "Add parameters to the model and initialize with given values."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nsetting the attribute name to value.", "response": "def _set_attribute(self, name, value):\n \"\"\"Make sure namespace gets updated when setting attributes.\"\"\"\n setattr(self, name, value)\n self.namespace.update({name: getattr(self, name)})"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nextract a list of expressions from the dictionary of expressions.", "response": "def update_expression_list(self):\n \"\"\"Extract a list of expressions from the dictionary of expressions.\"\"\"\n self.expression_list = [] # code arrives in dictionary, but is passed in this list\n self.expression_keys = [] # Keep track of the dictionary keys.\n self.expression_order = [] # This may be unecessary. It's to give ordering for cse\n for fname, fexpressions in self.expressions.items():\n for type, texpressions in fexpressions.items():\n if type == 'function':\n self.expression_list.append(texpressions) \n self.expression_keys.append([fname, type])\n self.expression_order.append(1) \n elif type[-10:] == 'derivative':\n for dtype, expression in texpressions.items():\n self.expression_list.append(expression)\n self.expression_keys.append([fname, type, dtype])\n if type[:-10] == 'first_' or type[:-10] == '':\n self.expression_order.append(3) #sym.count_ops(self.expressions[type][dtype]))\n elif type[:-10] == 'second_':\n self.expression_order.append(4) #sym.count_ops(self.expressions[type][dtype]))\n elif type[:-10] == 'third_':\n self.expression_order.append(5) #sym.count_ops(self.expressions[type][dtype]))\n else:\n self.expression_list.append(fexpressions[type]) \n self.expression_keys.append([fname, type])\n self.expression_order.append(2) \n\n # This step may be unecessary.\n # Not 100% sure if the sub expression elimination is order sensitive. This step orders the list with the 'function' code first and derivatives after.\n self.expression_order, self.expression_list, self.expression_keys = zip(*sorted(zip(self.expression_order, self.expression_list, self.expression_keys)))"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ngenerating code for the list of expressions provided using the common sub - expression eliminator to separate out portions that are computed multiple times.", "response": "def _gen_code(self):\n \"\"\"Generate code for the list of expressions provided using the common sub-expression eliminator to separate out portions that are computed multiple times.\"\"\"\n # This is the dictionary that stores all the generated code.\n\n self.code = {}\n def match_key(expr):\n if type(expr) is dict:\n code = {}\n for key in expr.keys():\n code[key] = match_key(expr[key])\n else:\n arg_list = [e for e in expr.atoms() if e.is_Symbol]\n code = self._expr2code(arg_list, expr)\n return code\n\n self.code = match_key(self.expressions)"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _expr2code(self, arg_list, expr):\n code = lambdastr(arg_list, expr)\n function_code = code.split(':')[1].strip()\n #for arg in arg_list:\n # function_code = function_code.replace(arg.name, 'self.'+arg.name)\n\n return function_code", "response": "Convert the given symbolic expression into code."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _print_code(self, code):\n # This needs a rewrite --- it doesn't check for match clashes! So sub11 would be replaced by sub1 before being replaced with sub11!!\n for key in self.variables.keys():\n for arg in self.variables[key]:\n code = code.replace(arg.name, 'self.'+arg.name)\n return code", "response": "Prepare code for string writing."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _outer_values_update(self, full_values):\n super(BayesianGPLVMMiniBatch, self)._outer_values_update(full_values)\n if self.has_uncertain_inputs():\n meangrad_tmp, vargrad_tmp = self.kern.gradients_qX_expectations(\n variational_posterior=self.X,\n Z=self.Z, dL_dpsi0=full_values['dL_dpsi0'],\n dL_dpsi1=full_values['dL_dpsi1'],\n dL_dpsi2=full_values['dL_dpsi2'],\n psi0=self.psi0, psi1=self.psi1, psi2=self.psi2)\n\n self.X.mean.gradient = meangrad_tmp\n self.X.variance.gradient = vargrad_tmp\n else:\n self.X.gradient = self.kern.gradients_X(full_values['dL_dKnm'], self.X, self.Z)\n self.X.gradient += self.kern.gradients_X_diag(full_values['dL_dKdiag'], self.X)", "response": "Update the gradients of the values of the current instance of the class."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef K(self, X, X2=None):\n X,slices = X[:,:-1],index_to_slices(X[:,-1])\n if X2 is None:\n X2,slices2 = X,slices\n K = np.zeros((X.shape[0], X.shape[0]))\n else:\n X2,slices2 = X2[:,:-1],index_to_slices(X2[:,-1])\n K = np.zeros((X.shape[0], X2.shape[0]))\n\n tdist = (X[:,0][:,None] - X2[:,0][None,:])**2\n ttdist = (X[:,0][:,None] - X2[:,0][None,:])\n \n vyt = self.variance_Yt\n \n lyt=1/(2*self.lengthscale_Yt)\n\n a = -self.a\n c = self.c\n\n kyy = lambda tdist: np.exp(-lyt*(tdist))\n\n k1 = lambda tdist: (2*lyt - 4*lyt**2 *(tdist) )\n\n k4 = lambda tdist: 2*lyt*(tdist)\n\n for i, s1 in enumerate(slices):\n for j, s2 in enumerate(slices2):\n for ss1 in s1:\n for ss2 in s2:\n if i==0 and j==0:\n K[ss1,ss2] = vyt*kyy(tdist[ss1,ss2])\n elif i==0 and j==1:\n K[ss1,ss2] = (k4(ttdist[ss1,ss2])+1)*vyt*kyy(tdist[ss1,ss2])\n #K[ss1,ss2] = (2*lyt*(ttdist[ss1,ss2])+1)*vyt*kyy(tdist[ss1,ss2])\n elif i==1 and j==1:\n K[ss1,ss2] = ( k1(tdist[ss1,ss2]) + 1. )*vyt* kyy(tdist[ss1,ss2])+self.ubias\n else:\n K[ss1,ss2] = (-k4(ttdist[ss1,ss2])+1)*vyt*kyy(tdist[ss1,ss2])\n #K[ss1,ss2] = (-2*lyt*(ttdist[ss1,ss2])+1)*vyt*kyy(tdist[ss1,ss2])\n #stop\n return K", "response": "Compute the covariance matrix between X and X2."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef update_gradients_full(self, dL_dK, X, X2=None):\n X,slices = X[:,:-1],index_to_slices(X[:,-1])\n if X2 is None:\n X2,slices2 = X,slices\n K = np.zeros((X.shape[0], X.shape[0]))\n else:\n X2,slices2 = X2[:,:-1],index_to_slices(X2[:,-1])\n\n\n vyt = self.variance_Yt\n\n lyt = 1./(2*self.lengthscale_Yt)\n\n tdist = (X[:,0][:,None] - X2[:,0][None,:])**2\n ttdist = (X[:,0][:,None] - X2[:,0][None,:])\n #rdist = [tdist,xdist]\n \n rd=tdist.shape[0]\n\n dka = np.zeros([rd,rd])\n dkc = np.zeros([rd,rd])\n dkYdvart = np.zeros([rd,rd])\n dkYdlent = np.zeros([rd,rd])\n\n dkdubias = np.zeros([rd,rd])\n\n kyy = lambda tdist: np.exp(-lyt*(tdist))\n dkyydlyt = lambda tdist: kyy(tdist)*(-tdist)\n\n k1 = lambda tdist: (2*lyt - 4*lyt**2 * (tdist) )\n\n k4 = lambda ttdist: 2*lyt*(ttdist)\n\n dk1dlyt = lambda tdist: 2. - 4*2.*lyt*tdist\n\n dk4dlyt = lambda ttdist: 2*(ttdist)\n\n for i, s1 in enumerate(slices):\n for j, s2 in enumerate(slices2):\n for ss1 in s1:\n for ss2 in s2:\n if i==0 and j==0:\n dkYdvart[ss1,ss2] = kyy(tdist[ss1,ss2])\n dkYdlent[ss1,ss2] = vyt*dkyydlyt(tdist[ss1,ss2])\n dkdubias[ss1,ss2] = 0\n elif i==0 and j==1:\n dkYdvart[ss1,ss2] = (k4(ttdist[ss1,ss2])+1)*kyy(tdist[ss1,ss2])\n #dkYdvart[ss1,ss2] = ((2*lyt*ttdist[ss1,ss2])+1)*kyy(tdist[ss1,ss2])\n dkYdlent[ss1,ss2] = vyt*dkyydlyt(tdist[ss1,ss2])* (k4(ttdist[ss1,ss2])+1.)+\\\n vyt*kyy(tdist[ss1,ss2])*(dk4dlyt(ttdist[ss1,ss2]))\n #dkYdlent[ss1,ss2] = vyt*dkyydlyt(tdist[ss1,ss2])* (2*lyt*(ttdist[ss1,ss2])+1.)+\\\n #vyt*kyy(tdist[ss1,ss2])*(2*ttdist[ss1,ss2])\n dkdubias[ss1,ss2] = 0\n elif i==1 and j==1:\n dkYdvart[ss1,ss2] = (k1(tdist[ss1,ss2]) + 1. )* kyy(tdist[ss1,ss2])\n dkYdlent[ss1,ss2] = vyt*dkyydlyt(tdist[ss1,ss2])*( k1(tdist[ss1,ss2]) + 1. ) +\\\n \t\t\tvyt*kyy(tdist[ss1,ss2])*dk1dlyt(tdist[ss1,ss2])\n dkdubias[ss1,ss2] = 1\n else:\n dkYdvart[ss1,ss2] = (-k4(ttdist[ss1,ss2])+1)*kyy(tdist[ss1,ss2])\n #dkYdvart[ss1,ss2] = (-2*lyt*(ttdist[ss1,ss2])+1)*kyy(tdist[ss1,ss2])\n dkYdlent[ss1,ss2] = vyt*dkyydlyt(tdist[ss1,ss2])* (-k4(ttdist[ss1,ss2])+1.)+\\\n vyt*kyy(tdist[ss1,ss2])*(-dk4dlyt(ttdist[ss1,ss2]) )\n dkdubias[ss1,ss2] = 0\n #dkYdlent[ss1,ss2] = vyt*dkyydlyt(tdist[ss1,ss2])* (-2*lyt*(ttdist[ss1,ss2])+1.)+\\\n #vyt*kyy(tdist[ss1,ss2])*(-2)*(ttdist[ss1,ss2])\n \n\n self.variance_Yt.gradient = np.sum(dkYdvart * dL_dK)\n\n self.lengthscale_Yt.gradient = np.sum(dkYdlent*(-0.5*self.lengthscale_Yt**(-2)) * dL_dK)\n\n self.ubias.gradient = np.sum(dkdubias * dL_dK)", "response": "derivative of the covariance matrix with respect to the parameters."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _dK_ode_dtheta(self, target):\n t_ode = self._t[self._index>0]\n dL_dK_ode = self._dL_dK[self._index>0, :]\n index_ode = self._index[self._index>0]-1\n if self._t2 is None:\n if t_ode.size==0:\n return \n t2_ode = t_ode\n dL_dK_ode = dL_dK_ode[:, self._index>0]\n index2_ode = index_ode\n else:\n t2_ode = self._t2[self._index2>0]\n dL_dK_ode = dL_dK_ode[:, self._index2>0]\n if t_ode.size==0 or t2_ode.size==0:\n return\n index2_ode = self._index2[self._index2>0]-1\n\n h1 = self._compute_H(t_ode, index_ode, t2_ode, index2_ode, stationary=self.is_stationary, update_derivatives=True)\n #self._dK_ddelay = self._dh_ddelay\n self._dK_dsigma = self._dh_dsigma\n\n if self._t2 is None:\n h2 = h1\n else:\n h2 = self._compute_H(t2_ode, index2_ode, t_ode, index_ode, stationary=self.is_stationary, update_derivatives=True)\n\n #self._dK_ddelay += self._dh_ddelay.T\n self._dK_dsigma += self._dh_dsigma.T\n # C1 = self.sensitivity\n # C2 = self.sensitivity\n\n # K = 0.5 * (h1 + h2.T)\n # var2 = C1*C2\n # if self.is_normalized:\n # dk_dD1 = (sum(sum(dL_dK.*dh1_dD1)) + sum(sum(dL_dK.*dh2_dD1.T)))*0.5*var2\n # dk_dD2 = (sum(sum(dL_dK.*dh1_dD2)) + sum(sum(dL_dK.*dh2_dD2.T)))*0.5*var2\n # dk_dsigma = 0.5 * var2 * sum(sum(dL_dK.*dK_dsigma))\n # dk_dC1 = C2 * sum(sum(dL_dK.*K))\n # dk_dC2 = C1 * sum(sum(dL_dK.*K))\n # else:\n # K = np.sqrt(np.pi) * K\n # dk_dD1 = (sum(sum(dL_dK.*dh1_dD1)) + * sum(sum(dL_dK.*K))\n # dk_dC2 = self.sigma * C1 * sum(sum(dL_dK.*K))\n\n\n # dk_dSim1Variance = dk_dC1\n # Last element is the length scale.\n (dL_dK_ode[:, :, None]*self._dh_ddelay[:, None, :]).sum(2)\n\n target[-1] += (dL_dK_ode*self._dK_dsigma/np.sqrt(2)).sum()", "response": "Do all the computations for the ode parts of the covariance function."} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nextracting times and output indices from the input matrix X.", "response": "def _extract_t_indices(self, X, X2=None, dL_dK=None):\n \"\"\"Extract times and output indices from the input matrix X. Times are ordered according to their index for convenience of computation, this ordering is stored in self._order and self.order2. These orderings are then mapped back to the original ordering (in X) using self._rorder and self._rorder2. \"\"\"\n\n # TODO: some fast checking here to see if this needs recomputing?\n self._t = X[:, 0]\n if not X.shape[1] == 2:\n raise ValueError('Input matrix for ode1 covariance should have two columns, one containing times, the other output indices')\n self._index = np.asarray(X[:, 1],dtype=np.int)\n # Sort indices so that outputs are in blocks for computational\n # convenience.\n self._order = self._index.argsort()\n self._index = self._index[self._order]\n self._t = self._t[self._order]\n self._rorder = self._order.argsort() # rorder is for reversing the order\n \n if X2 is None:\n self._t2 = None\n self._index2 = None\n self._order2 = self._order\n self._rorder2 = self._rorder\n else:\n if not X2.shape[1] == 2:\n raise ValueError('Input matrix for ode1 covariance should have two columns, one containing times, the other output indices')\n self._t2 = X2[:, 0]\n self._index2 = np.asarray(X2[:, 1],dtype=np.int)\n self._order2 = self._index2.argsort()\n self._index2 = self._index2[self._order2]\n self._t2 = self._t2[self._order2]\n self._rorder2 = self._order2.argsort() # rorder2 is for reversing order\n\n if dL_dK is not None:\n self._dL_dK = dL_dK[self._order, :]\n self._dL_dK = self._dL_dK[:, self._order2]"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _K_computations(self, X, X2):\n # First extract times and indices.\n self._extract_t_indices(X, X2)\n\n self._K_compute_eq()\n self._K_compute_ode_eq()\n if X2 is None:\n self._K_eq_ode = self._K_ode_eq.T\n else:\n self._K_compute_ode_eq(transpose=True)\n self._K_compute_ode()\n\n if X2 is None:\n self._K_dvar = np.zeros((self._t.shape[0], self._t.shape[0]))\n else:\n self._K_dvar = np.zeros((self._t.shape[0], self._t2.shape[0]))\n\n # Reorder values of blocks for placing back into _K_dvar.\n self._K_dvar = np.vstack((np.hstack((self._K_eq, self._K_eq_ode)),\n np.hstack((self._K_ode_eq, self._K_ode))))\n self._K_dvar = self._K_dvar[self._rorder, :]\n self._K_dvar = self._K_dvar[:, self._rorder2]\n \n \n if X2 is None:\n # Matrix giving scales of each output\n self._scale = np.zeros((self._t.size, self._t.size))\n code=\"\"\"\n for(int i=0;i0]\n index_ode = self._index2[self._index2>0]-1\n else:\n t_eq = self._t2[self._index2==0]\n t_ode = self._t[self._index>0]\n index_ode = self._index[self._index>0]-1\n else:\n t_eq = self._t[self._index==0]\n t_ode = self._t[self._index>0]\n index_ode = self._index[self._index>0]-1\n\n if t_ode.size==0 or t_eq.size==0:\n if transpose:\n self._K_eq_ode = np.zeros((t_eq.shape[0], t_ode.shape[0]))\n else:\n self._K_ode_eq = np.zeros((t_ode.shape[0], t_eq.shape[0]))\n return\n\n t_ode_mat = t_ode[:, None]\n t_eq_mat = t_eq[None, :]\n if self.delay is not None:\n t_ode_mat -= self.delay[index_ode, None]\n diff_t = (t_ode_mat - t_eq_mat)\n\n inv_sigma_diff_t = 1./self.sigma*diff_t\n decay_vals = self.decay[index_ode][:, None]\n half_sigma_d_i = 0.5*self.sigma*decay_vals\n\n if self.is_stationary:\n ln_part, signs = ln_diff_erfs(inf, half_sigma_d_i - inv_sigma_diff_t, return_sign=True)\n else:\n ln_part, signs = ln_diff_erfs(half_sigma_d_i + t_eq_mat/self.sigma, half_sigma_d_i - inv_sigma_diff_t, return_sign=True)\n sK = signs*np.exp(half_sigma_d_i*half_sigma_d_i - decay_vals*diff_t + ln_part)\n\n sK *= 0.5\n\n if not self.is_normalized:\n sK *= np.sqrt(np.pi)*self.sigma\n\n\n if transpose:\n self._K_eq_ode = sK.T\n else:\n self._K_ode_eq = sK"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef from_dict(input_dict):\n import copy\n input_dict = copy.deepcopy(input_dict)\n mapping_class = input_dict.pop('class')\n input_dict[\"name\"] = str(input_dict[\"name\"])\n import GPy\n mapping_class = eval(mapping_class)\n return mapping_class._build_from_input_dict(mapping_class, input_dict)", "response": "Instantiate an object of a derived class from a dictionary."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef inference(self, kern_r, kern_c, Xr, Xc, Zr, Zc, likelihood, Y, qU_mean ,qU_var_r, qU_var_c, indexD, output_dim):\n\n N, D, Mr, Mc, Qr, Qc = Y.shape[0], output_dim,Zr.shape[0], Zc.shape[0], Zr.shape[1], Zc.shape[1]\n\n uncertain_inputs_r = isinstance(Xr, VariationalPosterior)\n uncertain_inputs_c = isinstance(Xc, VariationalPosterior)\n uncertain_outputs = isinstance(Y, VariationalPosterior)\n\n grad_dict = self._init_grad_dict(N,D,Mr,Mc)\n\n beta = 1./likelihood.variance\n if len(beta)==1:\n beta = np.zeros(D)+beta\n\n psi0_r, psi1_r, psi2_r = self.gatherPsiStat(kern_r, Xr, Zr, uncertain_inputs_r)\n psi0_c, psi1_c, psi2_c = self.gatherPsiStat(kern_c, Xc, Zc, uncertain_inputs_c)\n\n #======================================================================\n # Compute Common Components\n #======================================================================\n\n Kuu_r = kern_r.K(Zr).copy()\n diag.add(Kuu_r, self.const_jitter)\n Lr = jitchol(Kuu_r)\n\n Kuu_c = kern_c.K(Zc).copy()\n diag.add(Kuu_c, self.const_jitter)\n Lc = jitchol(Kuu_c)\n\n mu, Sr, Sc = qU_mean, qU_var_r, qU_var_c\n LSr = jitchol(Sr)\n LSc = jitchol(Sc)\n\n LcInvMLrInvT = dtrtrs(Lc,dtrtrs(Lr,mu.T)[0].T)[0]\n LcInvLSc = dtrtrs(Lc, LSc)[0]\n LrInvLSr = dtrtrs(Lr, LSr)[0]\n LcInvScLcInvT = tdot(LcInvLSc)\n LrInvSrLrInvT = tdot(LrInvLSr)\n tr_LrInvSrLrInvT = np.square(LrInvLSr).sum()\n tr_LcInvScLcInvT = np.square(LcInvLSc).sum()\n\n mid_res = {\n 'psi0_r': psi0_r,\n 'psi1_r': psi1_r,\n 'psi2_r': psi2_r,\n 'psi0_c': psi0_c,\n 'psi1_c': psi1_c,\n 'psi2_c': psi2_c,\n 'Lr':Lr,\n 'Lc':Lc,\n 'LcInvMLrInvT': LcInvMLrInvT,\n 'LcInvScLcInvT': LcInvScLcInvT,\n 'LrInvSrLrInvT': LrInvSrLrInvT,\n }\n\n #======================================================================\n # Compute log-likelihood\n #======================================================================\n\n logL = 0.\n for d in range(D):\n logL += self.inference_d(d, beta, Y, indexD, grad_dict, mid_res, uncertain_inputs_r, uncertain_inputs_c, Mr, Mc)\n\n logL += -Mc * (np.log(np.diag(Lr)).sum()-np.log(np.diag(LSr)).sum()) -Mr * (np.log(np.diag(Lc)).sum()-np.log(np.diag(LSc)).sum()) \\\n - np.square(LcInvMLrInvT).sum()/2. - tr_LrInvSrLrInvT * tr_LcInvScLcInvT/2. + Mr*Mc/2.\n\n #======================================================================\n # Compute dL_dKuu\n #======================================================================\n\n tmp = tdot(LcInvMLrInvT)/2. + tr_LrInvSrLrInvT/2. * LcInvScLcInvT - Mr/2.*np.eye(Mc)\n\n dL_dKuu_c = backsub_both_sides(Lc, tmp, 'left')\n dL_dKuu_c += dL_dKuu_c.T\n dL_dKuu_c *= 0.5\n\n tmp = tdot(LcInvMLrInvT.T)/2. + tr_LcInvScLcInvT/2. * LrInvSrLrInvT - Mc/2.*np.eye(Mr)\n\n dL_dKuu_r = backsub_both_sides(Lr, tmp, 'left')\n dL_dKuu_r += dL_dKuu_r.T\n dL_dKuu_r *= 0.5\n\n #======================================================================\n # Compute dL_dqU\n #======================================================================\n\n tmp = - LcInvMLrInvT\n dL_dqU_mean = dtrtrs(Lc, dtrtrs(Lr, tmp.T, trans=1)[0].T, trans=1)[0]\n\n LScInv = dtrtri(LSc)\n tmp = -tr_LrInvSrLrInvT/2.*np.eye(Mc)\n dL_dqU_var_c = backsub_both_sides(Lc, tmp, 'left') + tdot(LScInv.T) * Mr/2.\n\n LSrInv = dtrtri(LSr)\n tmp = -tr_LcInvScLcInvT/2.*np.eye(Mr)\n dL_dqU_var_r = backsub_both_sides(Lr, tmp, 'left') + tdot(LSrInv.T) * Mc/2.\n\n #======================================================================\n # Compute the Posterior distribution of inducing points p(u|Y)\n #======================================================================\n\n post = PosteriorMultioutput(LcInvMLrInvT=LcInvMLrInvT, LcInvScLcInvT=LcInvScLcInvT,\n LrInvSrLrInvT=LrInvSrLrInvT, Lr=Lr, Lc=Lc, kern_r=kern_r, Xr=Xr, Zr=Zr)\n\n #======================================================================\n # Compute dL_dpsi\n #======================================================================\n\n grad_dict['dL_dqU_mean'] += dL_dqU_mean\n grad_dict['dL_dqU_var_c'] += dL_dqU_var_c\n grad_dict['dL_dqU_var_r'] += dL_dqU_var_r\n grad_dict['dL_dKuu_c'] += dL_dKuu_c\n grad_dict['dL_dKuu_r'] += dL_dKuu_r\n\n if not uncertain_inputs_c:\n grad_dict['dL_dKdiag_c'] = grad_dict['dL_dpsi0_c']\n grad_dict['dL_dKfu_c'] = grad_dict['dL_dpsi1_c']\n\n if not uncertain_inputs_r:\n grad_dict['dL_dKdiag_r'] = grad_dict['dL_dpsi0_r']\n grad_dict['dL_dKfu_r'] = grad_dict['dL_dpsi1_r']\n\n return post, logL, grad_dict", "response": "This function computes the SVI - VarDTC inference for the current SVI - VarDTC."} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nfinds common subarrays of 2D arrays.", "response": "def common_subarrays(X, axis=0):\n \"\"\"\n Find common subarrays of 2 dimensional X, where axis is the axis to apply the search over.\n Common subarrays are returned as a dictionary of pairs, where\n the subarray is a tuple representing the subarray and the index is the index\n for the subarray in X, where index is the index to the remaining axis.\n\n :param :class:`np.ndarray` X: 2d array to check for common subarrays in\n :param int axis: axis to apply subarray detection over.\n When the index is 0, compare rows -- columns, otherwise.\n\n Examples:\n =========\n\n In a 2d array:\n >>> import numpy as np\n >>> X = np.zeros((3,6), dtype=bool)\n >>> X[[1,1,1],[0,4,5]] = 1; X[1:,[2,3]] = 1\n >>> X\n array([[False, False, False, False, False, False],\n [ True, False, True, True, True, True],\n [False, False, True, True, False, False]], dtype=bool)\n >>> d = common_subarrays(X,axis=1)\n >>> len(d)\n 3\n >>> X[:, d[tuple(X[:,0])]]\n array([[False, False, False],\n [ True, True, True],\n [False, False, False]], dtype=bool)\n >>> d[tuple(X[:,4])] == d[tuple(X[:,0])] == [0, 4, 5]\n True\n >>> d[tuple(X[:,1])]\n [1]\n \"\"\"\n from collections import defaultdict\n from itertools import count\n from operator import iadd\n assert X.ndim == 2 and axis in (0,1), \"Only implemented for 2D arrays\"\n subarrays = defaultdict(list)\n cnt = count()\n def accumulate(x, s, c):\n t = tuple(x)\n col = next(c)\n iadd(s[t], [col])\n return None\n if axis == 0: [accumulate(x, subarrays, cnt) for x in X]\n else: [accumulate(x, subarrays, cnt) for x in X.T]\n return subarrays"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef predict(self, Xnew, full_cov=False, Y_metadata=None, kern=None, Yindex=0):\n b = self.bgplvms[Yindex]\n self.posterior = b.posterior\n self.kern = b.kern\n self.likelihood = b.likelihood\n return super(MRD, self).predict(Xnew, full_cov, Y_metadata, kern)", "response": "Predict the output mean and variance for the dataset given in Xnew."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nplotting input sensitivity for all datasets to see which input dimensions are significant for which dataset.", "response": "def plot_scales(self, titles=None, fig_kwargs={}, **kwargs):\n \"\"\"\n Plot input sensitivity for all datasets, to see which input dimensions are\n significant for which dataset.\n\n :param titles: titles for axes of datasets\n\n kwargs go into plot_ARD for each kernel.\n \"\"\"\n from ..plotting import plotting_library as pl\n\n if titles is None:\n titles = [r'${}$'.format(name) for name in self.names]\n\n M = len(self.bgplvms)\n fig = pl().figure(rows=1, cols=M, **fig_kwargs)\n for c in range(M):\n canvas = self.bgplvms[c].kern.plot_ARD(title=titles[c], figure=fig, col=c+1, **kwargs)\n return canvas"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nplotting latent space for the current dataset.", "response": "def plot_latent(self, labels=None, which_indices=None,\n resolution=60, legend=True,\n plot_limits=None,\n updates=False,\n kern=None, marker='<>^vsd',\n num_samples=1000, projection='2d',\n predict_kwargs={},\n scatter_kwargs=None, **imshow_kwargs):\n \"\"\"\n see plotting.matplot_dep.dim_reduction_plots.plot_latent\n if predict_kwargs is None, will plot latent spaces for 0th dataset (and kernel), otherwise give\n predict_kwargs=dict(Yindex='index') for plotting only the latent space of dataset with 'index'.\n \"\"\"\n from ..plotting.gpy_plot.latent_plots import plot_latent\n\n if \"Yindex\" not in predict_kwargs:\n predict_kwargs['Yindex'] = 0\n\n Yindex = predict_kwargs['Yindex']\n\n self.kern = self.bgplvms[Yindex].kern\n self.likelihood = self.bgplvms[Yindex].likelihood\n\n return plot_latent(self, labels, which_indices, resolution, legend, plot_limits, updates, kern, marker, num_samples, projection, scatter_kwargs)"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef factorize_space(self, threshold=0.005, printOut=False, views=None):\n M = len(self.bgplvms)\n if views is None:\n # There are some small modifications needed to make this work for M > 2 (currently the code\n # takes account of this, but it's not right there)\n if M is not 2:\n raise NotImplementedError(\"Not implemented for M > 2\")\n obsMod = [0]\n infMod = 1\n else:\n obsMod = views[0]\n infMod = views[1]\n\n scObs = [None] * len(obsMod)\n for i in range(0,len(obsMod)):\n # WARNING: the [0] in the end assumes that the ARD kernel (if there's addition) is the 1st one\n scObs[i] = np.atleast_2d(self.bgplvms[obsMod[i]].kern.input_sensitivity(summarize=False))[0]\n # Normalise to have max 1\n scObs[i] /= np.max(scObs[i])\n scInf = np.atleast_2d(self.bgplvms[infMod].kern.input_sensitivity(summarize=False))[0]\n scInf /= np.max(scInf)\n\n retainedScales = [None]*(len(obsMod)+1)\n for i in range(0,len(obsMod)):\n retainedScales[obsMod[i]] = np.where(scObs[i] > threshold)[0]\n retainedScales[infMod] = np.where(scInf > threshold)[0]\n\n for i in range(len(retainedScales)):\n retainedScales[i] = [k for k in retainedScales[i]] # Transform array to list\n\n sharedDims = set(retainedScales[obsMod[0]]).intersection(set(retainedScales[infMod]))\n for i in range(1,len(obsMod)):\n sharedDims = sharedDims.intersection(set(retainedScales[obsMod[i]]))\n privateDims = [None]*M\n for i in range(0,len(retainedScales)):\n privateDims[i] = set(retainedScales[i]).difference(sharedDims)\n privateDims[i] = [k for k in privateDims[i]] # Transform set to list\n sharedDims = [k for k in sharedDims] # Transform set to list\n\n sharedDims.sort()\n for i in range(len(privateDims)):\n privateDims[i].sort()\n\n if printOut:\n print('# Shared dimensions: ' + str(sharedDims))\n for i in range(len(retainedScales)):\n print('# Private dimensions model ' + str(i) + ':' + str(privateDims[i]))\n\n return sharedDims, privateDims", "response": "This function factorizes the latent space by factorizing the latent space."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef plot_mean(self, plot_limits=None, fixed_inputs=None,\n resolution=None, plot_raw=False,\n apply_link=False, visible_dims=None,\n which_data_ycols='all',\n levels=20, projection='2d',\n label='gp mean',\n predict_kw=None,\n **kwargs):\n \"\"\"\n Plot the mean of the GP.\n\n You can deactivate the legend for this one plot by supplying None to label.\n\n Give the Y_metadata in the predict_kw if you need it.\n\n\n\n :param plot_limits: The limits of the plot. If 1D [xmin,xmax], if 2D [[xmin,ymin],[xmax,ymax]]. Defaluts to data limits\n :type plot_limits: np.array\n :param fixed_inputs: a list of tuple [(i,v), (i,v)...], specifying that input dimension i should be set to value v.\n :type fixed_inputs: a list of tuples\n :param int resolution: The resolution of the prediction [defaults are 1D:200, 2D:50]\n :param bool plot_raw: plot the latent function (usually denoted f) only?\n :param bool apply_link: whether to apply the link function of the GP to the raw prediction.\n :param array-like which_data_ycols: which columns of y to plot (array-like or list of ints)\n :param int levels: for 2D plotting, the number of contour levels to use is\n :param {'2d','3d'} projection: whether to plot in 2d or 3d. This only applies when plotting two dimensional inputs!\n :param str label: the label for the plot.\n :param dict predict_kw: the keyword arguments for the prediction. If you want to plot a specific kernel give dict(kern=) in here\n \"\"\"\n canvas, kwargs = pl().new_canvas(projection=projection, **kwargs)\n X = get_x_y_var(self)[0]\n helper_data = helper_for_plot_data(self, X, plot_limits, visible_dims, fixed_inputs, resolution)\n helper_prediction = helper_predict_with_model(self, helper_data[2], plot_raw,\n apply_link, None,\n get_which_data_ycols(self, which_data_ycols),\n predict_kw)\n plots = _plot_mean(self, canvas, helper_data, helper_prediction,\n levels, projection, label, **kwargs)\n return pl().add_to_canvas(canvas, plots)", "response": "Plot the mean of the GP."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef plot_confidence(self, lower=2.5, upper=97.5, plot_limits=None, fixed_inputs=None,\n resolution=None, plot_raw=False,\n apply_link=False, visible_dims=None,\n which_data_ycols='all', label='gp confidence',\n predict_kw=None,\n **kwargs):\n \"\"\"\n Plot the confidence interval between the percentiles lower and upper.\n E.g. the 95% confidence interval is $2.5, 97.5$.\n Note: Only implemented for one dimension!\n\n You can deactivate the legend for this one plot by supplying None to label.\n\n Give the Y_metadata in the predict_kw if you need it.\n\n\n :param float lower: the lower percentile to plot\n :param float upper: the upper percentile to plot\n :param plot_limits: The limits of the plot. If 1D [xmin,xmax], if 2D [[xmin,ymin],[xmax,ymax]]. Defaluts to data limits\n :type plot_limits: np.array\n :param fixed_inputs: a list of tuple [(i,v), (i,v)...], specifying that input dimension i should be set to value v.\n :type fixed_inputs: a list of tuples\n :param int resolution: The resolution of the prediction [default:200]\n :param bool plot_raw: plot the latent function (usually denoted f) only?\n :param bool apply_link: whether to apply the link function of the GP to the raw prediction.\n :param array-like visible_dims: which columns of the input X (!) to plot (array-like or list of ints)\n :param array-like which_data_ycols: which columns of the output y (!) to plot (array-like or list of ints)\n :param dict predict_kw: the keyword arguments for the prediction. If you want to plot a specific kernel give dict(kern=) in here\n \"\"\"\n canvas, kwargs = pl().new_canvas(**kwargs)\n ycols = get_which_data_ycols(self, which_data_ycols)\n X = get_x_y_var(self)[0]\n helper_data = helper_for_plot_data(self, X, plot_limits, visible_dims, fixed_inputs, resolution)\n helper_prediction = helper_predict_with_model(self, helper_data[2], plot_raw, apply_link,\n (lower, upper),\n ycols, predict_kw)\n plots = _plot_confidence(self, canvas, helper_data, helper_prediction, label, **kwargs)\n return pl().add_to_canvas(canvas, plots, legend=label is not None)", "response": "Plot the confidence interval between the percentiles lower and upper."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef plot_samples(self, plot_limits=None, fixed_inputs=None,\n resolution=None, plot_raw=True,\n apply_link=False, visible_dims=None,\n which_data_ycols='all',\n samples=3, projection='2d', label='gp_samples',\n predict_kw=None,\n **kwargs):\n \"\"\"\n Plot the mean of the GP.\n\n You can deactivate the legend for this one plot by supplying None to label.\n\n Give the Y_metadata in the predict_kw if you need it.\n\n\n\n :param plot_limits: The limits of the plot. If 1D [xmin,xmax], if 2D [[xmin,ymin],[xmax,ymax]]. Defaluts to data limits\n :type plot_limits: np.array\n :param fixed_inputs: a list of tuple [(i,v), (i,v)...], specifying that input dimension i should be set to value v.\n :type fixed_inputs: a list of tuples\n :param int resolution: The resolution of the prediction [defaults are 1D:200, 2D:50]\n :param bool plot_raw: plot the latent function (usually denoted f) only? This is usually what you want!\n :param bool apply_link: whether to apply the link function of the GP to the raw prediction.\n :param array-like visible_dims: which columns of the input X (!) to plot (array-like or list of ints)\n :param array-like which_data_ycols: which columns of y to plot (array-like or list of ints)\n :param dict predict_kw: the keyword arguments for the prediction. If you want to plot a specific kernel give dict(kern=) in here\n :param int levels: for 2D plotting, the number of contour levels to use is\n \"\"\"\n canvas, kwargs = pl().new_canvas(projection=projection, **kwargs)\n ycols = get_which_data_ycols(self, which_data_ycols)\n X = get_x_y_var(self)[0]\n helper_data = helper_for_plot_data(self, X, plot_limits, visible_dims, fixed_inputs, resolution)\n helper_prediction = helper_predict_with_model(self, helper_data[2], plot_raw, apply_link,\n None,\n ycols, predict_kw, samples)\n plots = _plot_samples(self, canvas, helper_data, helper_prediction,\n projection, label, **kwargs)\n return pl().add_to_canvas(canvas, plots)", "response": "Plot the mean of the GP."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nplotting the density of the GP.", "response": "def plot_density(self, plot_limits=None, fixed_inputs=None,\n resolution=None, plot_raw=False,\n apply_link=False, visible_dims=None,\n which_data_ycols='all',\n levels=35, label='gp density',\n predict_kw=None,\n **kwargs):\n \"\"\"\n Plot the confidence interval between the percentiles lower and upper.\n E.g. the 95% confidence interval is $2.5, 97.5$.\n Note: Only implemented for one dimension!\n\n You can deactivate the legend for this one plot by supplying None to label.\n\n Give the Y_metadata in the predict_kw if you need it.\n\n :param plot_limits: The limits of the plot. If 1D [xmin,xmax], if 2D [[xmin,ymin],[xmax,ymax]]. Defaluts to data limits\n :type plot_limits: np.array\n :param fixed_inputs: a list of tuple [(i,v), (i,v)...], specifying that input dimension i should be set to value v.\n :type fixed_inputs: a list of tuples\n :param int resolution: The resolution of the prediction [default:200]\n :param bool plot_raw: plot the latent function (usually denoted f) only?\n :param bool apply_link: whether to apply the link function of the GP to the raw prediction.\n :param array-like visible_dims: which columns of the input X (!) to plot (array-like or list of ints)\n :param array-like which_data_ycols: which columns of y to plot (array-like or list of ints)\n :param int levels: the number of levels in the density (number bigger then 1, where 35 is smooth and 1 is the same as plot_confidence). You can go higher then 50 if the result is not smooth enough for you.\n :param dict predict_kw: the keyword arguments for the prediction. If you want to plot a specific kernel give dict(kern=) in here\n \"\"\"\n canvas, kwargs = pl().new_canvas(**kwargs)\n X = get_x_y_var(self)[0]\n helper_data = helper_for_plot_data(self, X, plot_limits, visible_dims, fixed_inputs, resolution)\n helper_prediction = helper_predict_with_model(self, helper_data[2], plot_raw,\n apply_link, np.linspace(2.5, 97.5, levels*2),\n get_which_data_ycols(self, which_data_ycols),\n predict_kw)\n plots = _plot_density(self, canvas, helper_data, helper_prediction, label, **kwargs)\n return pl().add_to_canvas(canvas, plots)"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef plot_f(self, plot_limits=None, fixed_inputs=None,\n resolution=None,\n apply_link=False,\n which_data_ycols='all', which_data_rows='all',\n visible_dims=None,\n levels=20, samples=0, lower=2.5, upper=97.5,\n plot_density=False,\n plot_data=True, plot_inducing=True,\n projection='2d', legend=True,\n predict_kw=None,\n **kwargs):\n \"\"\"\n Convinience function for plotting the fit of a GP.\n This is the same as plot, except it plots the latent function fit of the GP!\n\n If you want fine graned control use the specific plotting functions supplied in the model.\n\n You can deactivate the legend for this one plot by supplying None to label.\n\n Give the Y_metadata in the predict_kw if you need it.\n\n\n :param plot_limits: The limits of the plot. If 1D [xmin,xmax], if 2D [[xmin,ymin],[xmax,ymax]]. Defaluts to data limits\n :type plot_limits: np.array\n :param fixed_inputs: a list of tuple [(i,v), (i,v)...], specifying that input dimension i should be set to value v.\n :type fixed_inputs: a list of tuples\n :param int resolution: The resolution of the prediction [default:200]\n :param bool apply_link: whether to apply the link function of the GP to the raw prediction.\n :param which_data_ycols: when the data has several columns (independant outputs), only plot these\n :type which_data_ycols: 'all' or a list of integers\n :param which_data_rows: which of the training data to plot (default all)\n :type which_data_rows: 'all' or a slice object to slice self.X, self.Y\n :param array-like visible_dims: an array specifying the input dimensions to plot (maximum two)\n :param int levels: the number of levels in the density (number bigger then 1, where 35 is smooth and 1 is the same as plot_confidence). You can go higher then 50 if the result is not smooth enough for you.\n :param int samples: the number of samples to draw from the GP and plot into the plot. This will allways be samples from the latent function.\n :param float lower: the lower percentile to plot\n :param float upper: the upper percentile to plot\n :param bool plot_data: plot the data into the plot?\n :param bool plot_inducing: plot inducing inputs?\n :param bool plot_density: plot density instead of the confidence interval?\n :param dict predict_kw: the keyword arguments for the prediction. If you want to plot a specific kernel give dict(kern=) in here\n :param dict error_kwargs: kwargs for the error plot for the plotting library you are using\n :param kwargs plot_kwargs: kwargs for the data plot for the plotting library you are using\n \"\"\"\n return plot(self, plot_limits, fixed_inputs, resolution, True,\n apply_link, which_data_ycols, which_data_rows,\n visible_dims, levels, samples, 0,\n lower, upper, plot_data, plot_inducing,\n plot_density, predict_kw, projection, legend, **kwargs)", "response": "This function is used to plot the latent function fit of a GP."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef from_dict(input_dict, data=None):\n import copy\n input_dict = copy.deepcopy(input_dict)\n model_class = input_dict.pop('class')\n input_dict[\"name\"] = str(input_dict[\"name\"])\n import GPy\n model_class = eval(model_class)\n return model_class._build_from_input_dict(input_dict, data)", "response": "Instantiate an object of a derived class from a dictionary."} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nconvert the object into a json serializable dictionary.", "response": "def to_dict(self):\n \"\"\"\n Convert the object into a json serializable dictionary.\n\n Note: It uses the private method _save_to_input_dict of the parent.\n\n :return dict: json serializable dictionary containing the needed information to instantiate the object\n \"\"\"\n\n input_dict = super(Linear, self)._save_to_input_dict()\n input_dict[\"class\"] = \"GPy.mappings.Linear\"\n input_dict[\"A\"] = self.A.values.tolist()\n return input_dict"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef ICM(input_dim, num_outputs, kernel, W_rank=1,W=None,kappa=None,name='ICM'):\n if kernel.input_dim != input_dim:\n kernel.input_dim = input_dim\n warnings.warn(\"kernel's input dimension overwritten to fit input_dim parameter.\")\n\n K = kernel.prod(GPy.kern.Coregionalize(1, num_outputs, active_dims=[input_dim], rank=W_rank,W=W,kappa=kappa,name='B'),name=name)\n return K", "response": "Builds a kernel for an Intrinsic Coregionalization Model"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef LCM(input_dim, num_outputs, kernels_list, W_rank=1,name='ICM'):\n Nk = len(kernels_list)\n K = ICM(input_dim,num_outputs,kernels_list[0],W_rank,name='%s%s' %(name,0))\n j = 1\n for kernel in kernels_list[1:]:\n K += ICM(input_dim,num_outputs,kernel,W_rank,name='%s%s' %(name,j))\n j += 1\n return K", "response": "Builds a kernel for a Linear Coregionalization Model"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef Private(input_dim, num_outputs, kernel, output, kappa=None,name='X'):\n K = ICM(input_dim,num_outputs,kernel,W_rank=1,kappa=kappa,name=name)\n K.B.W.fix(0)\n _range = range(num_outputs)\n _range.pop(output)\n for j in _range:\n K.B.kappa[j] = 0\n K.B.kappa[j].fix()\n return K", "response": "Private function for creating a Private ICM model."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef K(self, X, X2=None):\n r = self._scaled_dist(X, X2)\n return self.K_of_r(r)", "response": "Kernel function applied on inputs X and X2."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ncomputing the derivative of K wrt X going through X2", "response": "def dK_dr_via_X(self, X, X2):\n \"\"\"\n compute the derivative of K wrt X going through X\n \"\"\"\n #a convenience function, so we can cache dK_dr\n return self.dK_dr(self._scaled_dist(X, X2))"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _unscaled_dist(self, X, X2=None):\n #X, = self._slice_X(X)\n if X2 is None:\n Xsq = np.sum(np.square(X),1)\n r2 = -2.*tdot(X) + (Xsq[:,None] + Xsq[None,:])\n util.diag.view(r2)[:,]= 0. # force diagnoal to be zero: sometime numerically a little negative\n r2 = np.clip(r2, 0, np.inf)\n return np.sqrt(r2)\n else:\n #X2, = self._slice_X(X2)\n X1sq = np.sum(np.square(X),1)\n X2sq = np.sum(np.square(X2),1)\n r2 = -2.*np.dot(X, X2.T) + (X1sq[:,None] + X2sq[None,:])\n r2 = np.clip(r2, 0, np.inf)\n return np.sqrt(r2)", "response": "Compute the Euclidean distance between each row of X and X2 or between\n X2 or between\n X1 and X2."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _scaled_dist(self, X, X2=None):\n if self.ARD:\n if X2 is not None:\n X2 = X2 / self.lengthscale\n return self._unscaled_dist(X/self.lengthscale, X2)\n else:\n return self._unscaled_dist(X, X2)/self.lengthscale", "response": "Efficiently compute the scaled distance for a set of elements X and X2."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef update_gradients_direct(self, dL_dVar, dL_dLen):\n self.variance.gradient = dL_dVar\n self.lengthscale.gradient = dL_dLen", "response": "Update the gradients of the kernel and lengthscale for the log likelihood derivates."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _inv_dist(self, X, X2=None):\n dist = self._scaled_dist(X, X2).copy()\n return 1./np.where(dist != 0., dist, np.inf)", "response": "Compute the elementwise inverse of the distance matrix"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncompute the derivative of the objective wrt K X", "response": "def gradients_X(self, dL_dK, X, X2=None):\n \"\"\"\n Given the derivative of the objective wrt K (dL_dK), compute the derivative wrt X\n \"\"\"\n if use_stationary_cython:\n return self._gradients_X_cython(dL_dK, X, X2)\n else:\n return self._gradients_X_pure(dL_dK, X, X2)"} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncomputes the second derivative of K wrt X and X2.", "response": "def gradients_XX(self, dL_dK, X, X2=None):\n \"\"\"\n Given the derivative of the objective K(dL_dK), compute the second derivative of K wrt X and X2:\n\n returns the full covariance matrix [QxQ] of the input dimensionfor each pair or vectors, thus\n the returned array is of shape [NxNxQxQ].\n\n ..math:\n \\frac{\\partial^2 K}{\\partial X2 ^2} = - \\frac{\\partial^2 K}{\\partial X\\partial X2}\n\n ..returns:\n dL2_dXdX2: [NxMxQxQ] in the cov=True case, or [NxMxQ] in the cov=False case,\n for X [NxQ] and X2[MxQ] (X2 is X if, X2 is None)\n Thus, we return the second derivative in X2.\n \"\"\"\n # According to multivariable chain rule, we can chain the second derivative through r:\n # d2K_dXdX2 = dK_dr*d2r_dXdX2 + d2K_drdr * dr_dX * dr_dX2:\n invdist = self._inv_dist(X, X2)\n invdist2 = invdist**2\n dL_dr = self.dK_dr_via_X(X, X2) #* dL_dK # we perform this product later\n tmp1 = dL_dr * invdist\n dL_drdr = self.dK2_drdr_via_X(X, X2) #* dL_dK # we perofrm this product later\n tmp2 = dL_drdr*invdist2\n l2 = np.ones(X.shape[1])*self.lengthscale**2 #np.multiply(np.ones(X.shape[1]) ,self.lengthscale**2)\n\n if X2 is None:\n X2 = X\n tmp1 -= np.eye(X.shape[0])*self.variance\n else:\n tmp1[invdist2==0.] -= self.variance\n\n #grad = np.empty((X.shape[0], X2.shape[0], X2.shape[1], X.shape[1]), dtype=np.float64)\n dist = X[:,None,:] - X2[None,:,:]\n dist = (dist[:,:,:,None]*dist[:,:,None,:])\n I = np.ones((X.shape[0], X2.shape[0], X2.shape[1], X.shape[1]))*np.eye((X2.shape[1]))\n grad = (((dL_dK*(tmp1*invdist2 - tmp2))[:,:,None,None] * dist)/l2[None,None,:,None]\n - (dL_dK*tmp1)[:,:,None,None] * I)/l2[None,None,None,:]\n return grad"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ngive the derivative of the objective dL_dK compute the second derivative of K wrt X", "response": "def gradients_XX_diag(self, dL_dK_diag, X):\n \"\"\"\n Given the derivative of the objective dL_dK, compute the second derivative of K wrt X:\n\n ..math:\n \\frac{\\partial^2 K}{\\partial X\\partial X}\n\n ..returns:\n dL2_dXdX: [NxQxQ]\n \"\"\"\n dL_dK_diag = dL_dK_diag.copy().reshape(-1, 1, 1)\n assert (dL_dK_diag.size == X.shape[0]) or (dL_dK_diag.size == 1), \"dL_dK_diag has to be given as row [N] or column vector [Nx1]\"\n\n l4 = np.ones(X.shape[1])*self.lengthscale**2\n return dL_dK_diag * (np.eye(X.shape[1]) * -self.dK2_drdr_diag()/(l4))[None, :,:]"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns the state space representation of the covariance.", "response": "def sde(self):\n \"\"\"\n Return the state space representation of the covariance.\n \"\"\"\n variance = float(self.variance.values)\n lengthscale = float(self.lengthscale.values)\n foo = np.sqrt(3.)/lengthscale\n F = np.array([[0, 1], [-foo**2, -2*foo]])\n L = np.array([[0], [1]])\n Qc = np.array([[12.*np.sqrt(3) / lengthscale**3 * variance]])\n H = np.array([[1, 0]])\n Pinf = np.array([[variance, 0],\n [0, 3.*variance/(lengthscale**2)]])\n # Allocate space for the derivatives\n dF = np.empty([F.shape[0],F.shape[1],2])\n dQc = np.empty([Qc.shape[0],Qc.shape[1],2])\n dPinf = np.empty([Pinf.shape[0],Pinf.shape[1],2])\n # The partial derivatives\n dFvariance = np.zeros([2,2])\n dFlengthscale = np.array([[0,0],\n [6./lengthscale**3,2*np.sqrt(3)/lengthscale**2]])\n dQcvariance = np.array([12.*np.sqrt(3)/lengthscale**3])\n dQclengthscale = np.array([-3*12*np.sqrt(3)/lengthscale**4*variance])\n dPinfvariance = np.array([[1,0],[0,3./lengthscale**2]])\n dPinflengthscale = np.array([[0,0],\n [0,-6*variance/lengthscale**3]])\n # Combine the derivatives\n dF[:,:,0] = dFvariance\n dF[:,:,1] = dFlengthscale\n dQc[:,:,0] = dQcvariance\n dQc[:,:,1] = dQclengthscale\n dPinf[:,:,0] = dPinfvariance\n dPinf[:,:,1] = dPinflengthscale\n\n return (F, L, Qc, H, Pinf, dF, dQc, dPinf)"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef Gram_matrix(self, F, F1, F2, F3, lower, upper):\n assert self.input_dim == 1\n def L(x,i):\n return(5*np.sqrt(5)/self.lengthscale**3*F[i](x) + 15./self.lengthscale**2*F1[i](x)+ 3*np.sqrt(5)/self.lengthscale*F2[i](x) + F3[i](x))\n n = F.shape[0]\n G = np.zeros((n,n))\n for i in range(n):\n for j in range(i,n):\n G[i,j] = G[j,i] = integrate.quad(lambda x : L(x,i)*L(x,j),lower,upper)[0]\n G_coef = 3.*self.lengthscale**5/(400*np.sqrt(5))\n Flower = np.array([f(lower) for f in F])[:,None]\n F1lower = np.array([f(lower) for f in F1])[:,None]\n F2lower = np.array([f(lower) for f in F2])[:,None]\n orig = 9./8*np.dot(Flower,Flower.T) + 9.*self.lengthscale**4/200*np.dot(F2lower,F2lower.T)\n orig2 = 3./5*self.lengthscale**2 * ( np.dot(F1lower,F1lower.T) + 1./8*np.dot(Flower,F2lower.T) + 1./8*np.dot(F2lower,Flower.T))\n return(1./self.variance* (G_coef*G + orig + orig2))", "response": "This function returns the Gram matrix of the vector of functions F with respect to the RKHS norm."} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nconverts the object into a json serializable dictionary.", "response": "def to_dict(self):\n \"\"\"\n Convert the object into a json serializable dictionary.\n\n Note: It uses the private method _save_to_input_dict of the parent.\n\n :return dict: json serializable dictionary containing the needed information to instantiate the object\n \"\"\"\n\n input_dict = super(RatQuad, self)._save_to_input_dict()\n input_dict[\"class\"] = \"GPy.kern.RatQuad\"\n input_dict[\"power\"] = self.power.values.tolist()\n return input_dict"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef fixed_inputs(model, non_fixed_inputs, fix_routine='median', as_list=True, X_all=False):\n from ...inference.latent_function_inference.posterior import VariationalPosterior\n f_inputs = []\n if hasattr(model, 'has_uncertain_inputs') and model.has_uncertain_inputs():\n X = model.X.mean.values.copy()\n elif isinstance(model.X, VariationalPosterior):\n X = model.X.values.copy()\n else:\n if X_all:\n X = model.X_all.copy()\n else:\n X = model.X.copy()\n for i in range(X.shape[1]):\n if i not in non_fixed_inputs:\n if fix_routine == 'mean':\n f_inputs.append( (i, np.mean(X[:,i])) )\n if fix_routine == 'median':\n f_inputs.append( (i, np.median(X[:,i])) )\n else: # set to zero zero\n f_inputs.append( (i, 0) )\n if not as_list:\n X[:,i] = f_inputs[-1][1]\n if as_list:\n return f_inputs\n else:\n return X", "response": "This function returns back the fixed_inputs where the other inputs are fixed using fix_routine\n "} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef infer_newX(model, Y_new, optimize=True, init='L2'):\n infr_m = InferenceX(model, Y_new, init=init)\n\n if optimize:\n infr_m.optimize()\n\n return infr_m.X, infr_m", "response": "Infer the distribution of X for the new observed data Y_new."} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns the state space representation of the covariance matrix.", "response": "def sde(self):\n \"\"\"\n Return the state space representation of the covariance.\n \n Note! For Sparse GP inference too small or two high values of lengthscale\n lead to instabilities. This is because Qc are too high or too low\n and P_inf are not full rank. This effect depends on approximatio order.\n For N = 10. lengthscale must be in (0.8,8). For other N tests must be conducted.\n N=6: (0.06,31)\n Variance should be within reasonable bounds as well, but its dependence is linear.\n \n The above facts do not take into accout regularization.\n \"\"\"\n #import pdb; pdb.set_trace()\n if self.approx_order is not None:\n N = self.approx_order\n else:\n N = 10# approximation order ( number of terms in exponent series expansion)\n \n roots_rounding_decimals = 6\n\n fn = np.math.factorial(N)\n\n p_lengthscale = float( self.lengthscale )\n p_variance = float(self.variance)\n kappa = 1.0/2.0/p_lengthscale**2\n\n Qc = np.array( ((p_variance*np.sqrt(np.pi/kappa)*fn*(4*kappa)**N,),) )\n \n eps = 1e-12\n if (float(Qc) > 1.0/eps) or (float(Qc) < eps):\n warnings.warn(\"\"\"sde_RBF kernel: the noise variance Qc is either very large or very small. \n It influece conditioning of P_inf: {0:e}\"\"\".format(float(Qc)) )\n\n pp1 = np.zeros((2*N+1,)) # array of polynomial coefficients from higher power to lower\n\n for n in range(0, N+1): # (2N+1) - number of polynomial coefficients\n pp1[2*(N-n)] = fn*(4.0*kappa)**(N-n)/np.math.factorial(n)*(-1)**n\n \n pp = sp.poly1d(pp1)\n roots = sp.roots(pp)\n\n neg_real_part_roots = roots[np.round(np.real(roots) ,roots_rounding_decimals) < 0]\n aa = sp.poly1d(neg_real_part_roots, r=True).coeffs\n\n F = np.diag(np.ones((N-1,)),1)\n F[-1,:] = -aa[-1:0:-1]\n\n L= np.zeros((N,1))\n L[N-1,0] = 1\n\n H = np.zeros((1,N))\n H[0,0] = 1\n\n # Infinite covariance:\n Pinf = lyap(F, -np.dot(L,np.dot( Qc[0,0],L.T)))\n Pinf = 0.5*(Pinf + Pinf.T)\n # Allocating space for derivatives\n dF = np.empty([F.shape[0],F.shape[1],2])\n dQc = np.empty([Qc.shape[0],Qc.shape[1],2])\n dPinf = np.empty([Pinf.shape[0],Pinf.shape[1],2])\n\n # Derivatives:\n dFvariance = np.zeros(F.shape)\n dFlengthscale = np.zeros(F.shape)\n dFlengthscale[-1,:] = -aa[-1:0:-1]/p_lengthscale * np.arange(-N,0,1)\n\n dQcvariance = Qc/p_variance\n dQclengthscale = np.array(( (p_variance*np.sqrt(2*np.pi)*fn*2**N*p_lengthscale**(-2*N)*(1-2*N),),))\n \n dPinf_variance = Pinf/p_variance\n\n lp = Pinf.shape[0]\n coeff = np.arange(1,lp+1).reshape(lp,1) + np.arange(1,lp+1).reshape(1,lp) - 2\n coeff[np.mod(coeff,2) != 0] = 0\n dPinf_lengthscale = -1/p_lengthscale*Pinf*coeff\n\n dF[:,:,0] = dFvariance\n dF[:,:,1] = dFlengthscale\n dQc[:,:,0] = dQcvariance\n dQc[:,:,1] = dQclengthscale\n dPinf[:,:,0] = dPinf_variance\n dPinf[:,:,1] = dPinf_lengthscale\n\n P0 = Pinf.copy()\n dP0 = dPinf.copy()\n\n if self.balance:\n # Benefits of this are not very sound. Helps only in one case:\n # SVD Kalman + RBF kernel\n import GPy.models.state_space_main as ssm\n (F, L, Qc, H, Pinf, P0, dF, dQc, dPinf,dP0) = ssm.balance_ss_model(F, L, Qc, H, Pinf, P0, dF, dQc, dPinf, dP0 )\n\n return (F, L, Qc, H, Pinf, P0, dF, dQc, dPinf, dP0)"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning the state space representation of the covariance.", "response": "def sde(self):\n \"\"\"\n Return the state space representation of the covariance.\n \"\"\"\n variance = float(self.variance.values)\n lengthscale = float(self.lengthscale)\n\n F = np.array(((-1.0/lengthscale,),))\n L = np.array(((1.0,),))\n Qc = np.array( ((2.0*variance/lengthscale,),) )\n H = np.array(((1.0,),))\n Pinf = np.array(((variance,),))\n P0 = Pinf.copy()\n\n dF = np.zeros((1,1,2));\n dQc = np.zeros((1,1,2));\n dPinf = np.zeros((1,1,2));\n\n dF[:,:,0] = 0.0\n dF[:,:,1] = 1.0/lengthscale**2\n\n dQc[:,:,0] = 2.0/lengthscale\n dQc[:,:,1] = -2.0*variance/lengthscale**2\n\n dPinf[:,:,0] = 1.0\n dPinf[:,:,1] = 0.0\n\n dP0 = dPinf.copy()\n\n return (F, L, Qc, H, Pinf, P0, dF, dQc, dPinf, dP0)"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef pdf_link(self, link_f, y, Y_metadata=None):\n assert np.atleast_1d(link_f).shape == np.atleast_1d(y).shape\n c = np.zeros((link_f.shape[0],))\n\n # log_objective = np.log(self.r) + (self.r - 1) * np.log(y) - link_f - (np.exp(-link_f) * (y ** self.r))\n # log_objective = stats.weibull_min.pdf(y,c=self.beta,loc=link_f,scale=1.)\n log_objective = self.logpdf_link(link_f, y, Y_metadata)\n return np.exp(log_objective)", "response": "Evaluate the likelihood function given a link function and a latent variable y"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef logpdf_link(self, link_f, y, Y_metadata=None):\n # alpha = self.gp_link.transf(gp)*self.beta sum(log(a) + (a-1).*log(y)- f - exp(-f).*y.^a)\n # return (1. - alpha)*np.log(obs) + self.beta*obs - alpha * np.log(self.beta) + np.log(special.gamma(alpha))\n assert np.atleast_1d(link_f).shape == np.atleast_1d(y).shape\n c = np.zeros_like(y)\n if Y_metadata is not None and 'censored' in Y_metadata.keys():\n c = Y_metadata['censored']\n\n # uncensored = (1-c)* (np.log(self.r) + (self.r - 1) * np.log(y) - link_f - (np.exp(-link_f) * (y ** self.r)))\n # censored = (-c)*np.exp(-link_f)*(y**self.r)\n uncensored = (1-c)*( np.log(self.r)-np.log(link_f)+(self.r-1)*np.log(y) - y**self.r/link_f)\n censored = -c*y**self.r/link_f\n\n log_objective = uncensored + censored\n return log_objective", "response": "Log PDF for the point in the log distribution"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef d2logpdf_dlink2(self, link_f, y, Y_metadata=None):\n # hess = (self.beta - 1.) / (y - link_f)**2\n c = np.zeros_like(y)\n if Y_metadata is not None and 'censored' in Y_metadata.keys():\n c = Y_metadata['censored']\n\n # uncensored = (1-c)* (-(y ** self.r) * np.exp(-link_f))\n # censored = -c*np.exp(-link_f)*y**self.r\n uncensored = (1-c)*(1/link_f**2 -2*y**self.r/link_f**3)\n censored = -c*2*y**self.r/link_f**3\n hess = uncensored + censored\n # hess = -(y ** self.r) * np.exp(-link_f)\n return hess", "response": "d2logpdf_dlink2 - 2d logpdf at y given link f and y"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef d3logpdf_dlink3(self, link_f, y, Y_metadata=None):\n # d3lik_dlink3 = (1. - self.beta) / (y - link_f)**3\n\n c = np.zeros_like(y)\n if Y_metadata is not None and 'censored' in Y_metadata.keys():\n c = Y_metadata['censored']\n # uncensored = (1-c)* ((y ** self.r) * np.exp(-link_f))\n # censored = c*np.exp(-link_f)*y**self.r\n uncensored = (1-c)*(-2/link_f**3+ 6*y**self.r/link_f**4)\n censored = c*6*y**self.r/link_f**4\n\n d3lik_dlink3 = uncensored + censored\n # d3lik_dlink3 = (y ** self.r) * np.exp(-link_f)\n return d3lik_dlink3", "response": "Function to compute the third order derivative log - likelihood function at y given link f w. r. t link f y given link f w. r. t link f y given link f w. r. t link f y given link f w. r. t link f y given link f y given Y_metadata which is not used in gamma distribution"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nfunctions to compute the third derivative of loglikelihood wrt r", "response": "def dlogpdf_dlink_dr(self, inv_link_f, y, Y_metadata=None):\n \"\"\"\n First order derivative derivative of loglikelihood wrt r:shape parameter\n\n :param link_f: latent variables link(f)\n :type link_f: Nx1 array\n :param y: data\n :type y: Nx1 array\n :param Y_metadata: Y_metadata which is not used in gamma distribution\n :returns: third derivative of likelihood evaluated at points f\n :rtype: Nx1 array\n \"\"\"\n # dlogpdf_dlink_dr = self.beta * y**(self.beta - 1) * np.exp(-link_f)\n # dlogpdf_dlink_dr = np.exp(-link_f) * (y ** self.r) * np.log(y)\n c = np.zeros_like(y)\n if Y_metadata is not None and 'censored' in Y_metadata.keys():\n c = Y_metadata['censored']\n\n link_f = inv_link_f\n # uncensored = (1-c)*(np.exp(-link_f)* (y ** self.r) * np.log(y))\n # censored = c*np.exp(-link_f)*(y**self.r)*np.log(y)\n uncensored = (1-c)*(y**self.r*np.log(y)/link_f**2)\n censored = c*(y**self.r*np.log(y)/link_f**2)\n dlogpdf_dlink_dr = uncensored + censored\n return dlogpdf_dlink_dr"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning a set of samples conditioned on a given value of latent variable f.", "response": "def samples(self, gp, Y_metadata=None):\n \"\"\"\n Returns a set of samples of observations conditioned on a given value of latent variable f.\n\n :param gp: latent variable\n \"\"\"\n orig_shape = gp.shape\n gp = gp.flatten()\n weibull_samples = np.array([sp.stats.weibull_min.rvs(self.r, loc=0, scale=self.gp_link.transf(f)) for f in gp])\n return weibull_samples.reshape(orig_shape)"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef inference_likelihood(self, kern, X, Z, likelihood, Y):\n\n num_data, output_dim = Y.shape\n input_dim = Z.shape[0]\n if self.mpi_comm != None:\n from mpi4py import MPI\n num_data_all = np.array(num_data,dtype=np.int32)\n self.mpi_comm.Allreduce([np.int32(num_data), MPI.INT], [num_data_all, MPI.INT])\n num_data = num_data_all\n\n if isinstance(X, VariationalPosterior):\n uncertain_inputs = True\n else:\n uncertain_inputs = False\n\n #see whether we've got a different noise variance for each datum\n beta = 1./np.fmax(likelihood.variance, 1e-6)\n het_noise = beta.size > 1\n if het_noise:\n self.batchsize = 1\n\n psi0_full, psi1Y_full, psi2_full, YRY_full = self.gatherPsiStat(kern, X, Z, Y, beta, uncertain_inputs)\n\n #======================================================================\n # Compute Common Components\n #======================================================================\n\n Kmm = kern.K(Z).copy()\n diag.add(Kmm, self.const_jitter)\n if not np.isfinite(Kmm).all():\n print(Kmm)\n Lm = jitchol(Kmm)\n LmInv = dtrtri(Lm)\n\n LmInvPsi2LmInvT = LmInv.dot(psi2_full.dot(LmInv.T))\n Lambda = np.eye(Kmm.shape[0])+LmInvPsi2LmInvT\n LL = jitchol(Lambda)\n LLInv = dtrtri(LL)\n logdet_L = 2.*np.sum(np.log(np.diag(LL)))\n LmLLInv = LLInv.dot(LmInv)\n \n b = psi1Y_full.dot(LmLLInv.T)\n bbt = np.square(b).sum()\n v = b.dot(LmLLInv).T\n LLinvPsi1TYYTPsi1LLinvT = tdot(b.T)\n \n tmp = -LLInv.T.dot(LLinvPsi1TYYTPsi1LLinvT+output_dim*np.eye(input_dim)).dot(LLInv)\n dL_dpsi2R = LmInv.T.dot(tmp+output_dim*np.eye(input_dim)).dot(LmInv)/2.\n \n # Cache intermediate results\n self.midRes['dL_dpsi2R'] = dL_dpsi2R\n self.midRes['v'] = v\n\n #======================================================================\n # Compute log-likelihood\n #======================================================================\n if het_noise:\n logL_R = -np.log(beta).sum()\n else:\n logL_R = -num_data*np.log(beta)\n logL = -(output_dim*(num_data*log_2_pi+logL_R+psi0_full-np.trace(LmInvPsi2LmInvT))+YRY_full-bbt)/2.-output_dim*logdet_L/2.\n\n #======================================================================\n # Compute dL_dKmm\n #======================================================================\n\n dL_dKmm = dL_dpsi2R - output_dim*LmInv.T.dot(LmInvPsi2LmInvT).dot(LmInv)/2.\n\n #======================================================================\n # Compute the Posterior distribution of inducing points p(u|Y)\n #======================================================================\n\n if not self.Y_speedup or het_noise:\n wd_inv = backsub_both_sides(Lm, np.eye(input_dim)- backsub_both_sides(LL, np.identity(input_dim), transpose='left'), transpose='left')\n post = Posterior(woodbury_inv=wd_inv, woodbury_vector=v, K=Kmm, mean=None, cov=None, K_chol=Lm)\n else:\n post = None\n\n #======================================================================\n # Compute dL_dthetaL for uncertian input and non-heter noise\n #======================================================================\n\n if not het_noise:\n dL_dthetaL = (YRY_full*beta + beta*output_dim*psi0_full - num_data*output_dim*beta)/2. - beta*(dL_dpsi2R*psi2_full).sum() - beta*(v.T*psi1Y_full).sum()\n self.midRes['dL_dthetaL'] = dL_dthetaL\n\n return logL, dL_dKmm, post", "response": "Compute log - likelihood of a set of data points."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nconverting the object into a json serializable dictionary.", "response": "def to_dict(self):\n \"\"\"\n Convert the object into a json serializable dictionary.\n\n Note: It uses the private method _save_to_input_dict of the parent.\n\n :return dict: json serializable dictionary containing the needed information to instantiate the object\n \"\"\"\n\n input_dict = super(RBF, self)._save_to_input_dict()\n input_dict[\"class\"] = \"GPy.kern.RBF\"\n input_dict[\"inv_l\"] = self.use_invLengthscale\n if input_dict[\"inv_l\"] == True:\n input_dict[\"lengthscale\"] = np.sqrt(1 / float(self.inv_l))\n return input_dict"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef get_one_dimensional_kernel(self, dim):\n oneDkernel = GridRBF(input_dim=1, variance=self.variance.copy(), originalDimensions=dim)\n return oneDkernel", "response": "Returns a one - dimensional kernel that is the one - dimensional grid."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef load_text_data(dataset, directory, centre=True):\n\n points, point_names = parse_text(os.path.join(directory, dataset + '.txt'))[0:2]\n # Remove markers where there is a NaN\n present_index = [i for i in range(points[0].shape[1]) if not (np.any(np.isnan(points[0][:, i])) or np.any(np.isnan(points[0][:, i])) or np.any(np.isnan(points[0][:, i])))]\n\n point_names = point_names[present_index]\n for i in range(3):\n points[i] = points[i][:, present_index]\n if centre:\n points[i] = (points[i].T - points[i].mean(axis=1)).T \n\n # Concatanate the X, Y and Z markers together\n Y = np.concatenate((points[0], points[1], points[2]), axis=1)\n Y = Y/400.\n connect = read_connections(os.path.join(directory, 'connections.txt'), point_names)\n return Y, connect", "response": "Load in a data set of marker points from the Ohio State University C3D motion capture files."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef parse_text(file_name):\n\n # Read the header\n fid = open(file_name, 'r')\n point_names = np.array(fid.readline().split())[2:-1:3]\n fid.close()\n for i in range(len(point_names)):\n point_names[i] = point_names[i][0:-2]\n\n # Read the matrix data\n S = np.loadtxt(file_name, skiprows=1)\n field = np.uint(S[:, 0])\n times = S[:, 1]\n S = S[:, 2:]\n\n # Set the -9999.99 markers to be not present\n S[S==-9999.99] = np.NaN\n\n # Store x, y and z in different arrays\n points = []\n points.append(S[:, 0:-1:3])\n points.append(S[:, 1:-1:3])\n points.append(S[:, 2:-1:3])\n\n return points, point_names, times", "response": "Parse data from Ohio State University text mocap files ( http://accad. osu.edu / research / mocap_data. htm )"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef read_connections(file_name, point_names):\n\n connections = []\n fid = open(file_name, 'r')\n line=fid.readline()\n while(line):\n connections.append(np.array(line.split(',')))\n connections[-1][0] = connections[-1][0].strip()\n connections[-1][1] = connections[-1][1].strip()\n line = fid.readline()\n connect = np.zeros((len(point_names), len(point_names)),dtype=bool)\n for i in range(len(point_names)):\n for j in range(len(point_names)):\n for k in range(len(connections)):\n if connections[k][0] == point_names[i] and connections[k][1] == point_names[j]:\n \n connect[i,j]=True\n connect[j,i]=True\n break\n \n return connect", "response": "Read a file detailing which markers should be connected to which motion capture data should be connected to which for motion capture data."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef find_children(self):\n for i in range(len(self.vertices)):\n self.vertices[i].children = []\n for i in range(len(self.vertices)):\n for parent in self.vertices[i].parents:\n if i not in self.vertices[parent].children:\n self.vertices[parent].children.append(i)", "response": "Take a tree and set the children according to the parents."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef find_parents(self):\n for i in range(len(self.vertices)):\n self.vertices[i].parents = []\n for i in range(len(self.vertices)):\n for child in self.vertices[i].children:\n if i not in self.vertices[child].parents:\n self.vertices[child].parents.append(i)", "response": "Take a tree and set the parents according to the children\n\n "} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef find_root(self):\n self.find_parents()\n index = 0\n while len(self.vertices[index].parents)>0:\n index = self.vertices[index].parents[0]\n return index", "response": "Finds the index of the root node of the tree."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ngiving the index associated with a given vertex id.", "response": "def get_index_by_id(self, id):\n \"\"\"Give the index associated with a given vertex id.\"\"\"\n for i in range(len(self.vertices)):\n if self.vertices[i].id == id:\n return i\n raise ValueError('Reverse look up of id failed.')"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ngive the index associated with a given vertex name.", "response": "def get_index_by_name(self, name):\n \"\"\"Give the index associated with a given vertex name.\"\"\"\n for i in range(len(self.vertices)):\n if self.vertices[i].name == name:\n return i\n raise ValueError('Reverse look up of name failed.')"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef order_vertices(self):\n \n ordered = False\n while ordered == False:\n for i in range(len(self.vertices)):\n ordered = True\n for parent in self.vertices[i].parents:\n if parent>i:\n ordered = False\n self.swap_vertices(i, parent)", "response": "Order the vertices in the graph such that parents always have a lower index than children."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nswap two vertices in the tree structure array. swap_vertex swaps the location of two vertices in a tree structure array. :param tree: the tree for which two vertices are to be swapped. :param i: the index of the first vertex to be swapped. :param j: the index of the second vertex to be swapped. :rval tree: the tree structure with the two vertex locations swapped.", "response": "def swap_vertices(self, i, j):\n \"\"\"\n Swap two vertices in the tree structure array.\n swap_vertex swaps the location of two vertices in a tree structure array. \n\n :param tree: the tree for which two vertices are to be swapped.\n :param i: the index of the first vertex to be swapped.\n :param j: the index of the second vertex to be swapped.\n :rval tree: the tree structure with the two vertex locations swapped.\n\n \"\"\"\n store_vertex_i = self.vertices[i]\n store_vertex_j = self.vertices[j]\n self.vertices[j] = store_vertex_i\n self.vertices[i] = store_vertex_j\n for k in range(len(self.vertices)):\n for swap_list in [self.vertices[k].children, self.vertices[k].parents]:\n if i in swap_list:\n swap_list[swap_list.index(i)] = -1\n if j in swap_list:\n swap_list[swap_list.index(j)] = i\n if -1 in swap_list:\n swap_list[swap_list.index(-1)] = j"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef smooth_angle_channels(self, channels):\n for vertex in self.vertices:\n for col in vertex.meta['rot_ind']:\n if col:\n for k in range(1, channels.shape[0]):\n diff=channels[k, col]-channels[k-1, col]\n if abs(diff+360.)0: \n start_val = end_val\n end_val = end_val + len(vertex.meta['channels'])\n for j in range(num_frames):\n channels[j, start_val:end_val] = bones[i][j]\n self.resolve_indices(i, start_val)\n\n self.smooth_angle_channels(channels)\n return channels"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef read_documentation(self, fid):\n\n lin = self.read_line(fid)\n while lin[0] != ':':\n self.documentation.append(lin)\n lin = self.read_line(fid)\n return lin", "response": "Read documentation from an acclaim skeleton file stream."} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nread hierarchy information from acclaim skeleton file stream.", "response": "def read_hierarchy(self, fid):\n \"\"\"Read hierarchy information from acclaim skeleton file stream.\"\"\"\n\n lin = self.read_line(fid)\n \n while lin != 'end':\n parts = lin.split()\n if lin != 'begin':\n ind = self.get_index_by_name(parts[0])\n for i in range(1, len(parts)):\n self.vertices[ind].children.append(self.get_index_by_name(parts[i]))\n lin = self.read_line(fid)\n lin = self.read_line(fid)\n return lin"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nread a line from a file string and check it isn t commented before returning.", "response": "def read_line(self, fid):\n \"\"\"Read a line from a file string and check it isn't either empty or commented before returning.\"\"\"\n lin = '#'\n while lin[0] == '#':\n lin = fid.readline().strip()\n if lin == '':\n return lin\n return lin"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreads the root node from an acclaim skeleton file stream.", "response": "def read_root(self, fid):\n \"\"\"Read the root node from an acclaim skeleton file stream.\"\"\"\n lin = self.read_line(fid) \n while lin[0] != ':':\n parts = lin.split()\n if parts[0]=='order':\n order = []\n for i in range(1, len(parts)):\n if parts[i].lower()=='rx':\n chan = 'Xrotation'\n order.append('x')\n elif parts[i].lower()=='ry':\n chan = 'Yrotation'\n order.append('y')\n elif parts[i].lower()=='rz':\n chan = 'Zrotation'\n order.append('z')\n elif parts[i].lower()=='tx':\n chan = 'Xposition'\n elif parts[i].lower()=='ty':\n chan = 'Yposition'\n elif parts[i].lower()=='tz':\n chan = 'Zposition'\n elif parts[i].lower()=='l':\n chan = 'length'\n self.vertices[0].meta['channels'].append(chan)\n # order is reversed compared to bvh\n self.vertices[0].meta['order'] = order[::-1]\n\n elif parts[0]=='axis':\n # order is reversed compared to bvh\n self.vertices[0].meta['axis_order'] = parts[1][::-1].lower()\n elif parts[0]=='position':\n self.vertices[0].meta['offset'] = [float(parts[1]),\n float(parts[2]),\n float(parts[3])]\n elif parts[0]=='orientation':\n self.vertices[0].meta['orientation'] = [float(parts[1]),\n float(parts[2]),\n float(parts[3])]\n lin = self.read_line(fid)\n return lin"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef read_skel(self, fid):\n lin = self.read_line(fid)\n while lin:\n if lin[0]==':':\n if lin[1:]== 'name':\n lin = self.read_line(fid)\n self.name = lin\n elif lin[1:]=='units':\n lin = self.read_units(fid)\n elif lin[1:]=='documentation':\n lin = self.read_documentation(fid)\n elif lin[1:]=='root':\n lin = self.read_root(fid)\n elif lin[1:]=='bonedata':\n lin = self.read_bonedata(fid)\n elif lin[1:]=='hierarchy':\n lin = self.read_hierarchy(fid)\n elif lin[1:8]=='version':\n lin = self.read_line(fid)\n continue\n else: \n if not lin:\n self.finalize()\n return\n lin = self.read_line(fid)\n else:\n raise ValueError('Unrecognised file format')\n self.finalize()", "response": "Loads an acclaim skeleton format from a file stream."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nread units from an acclaim skeleton file stream.", "response": "def read_units(self, fid):\n \"\"\"Read units from an acclaim skeleton file stream.\"\"\"\n lin = self.read_line(fid) \n while lin[0] != ':':\n parts = lin.split()\n if parts[0]=='mass':\n self.mass = float(parts[1])\n elif parts[0]=='length':\n self.length = float(parts[1])\n elif parts[0]=='angle':\n self.angle = parts[1]\n lin = self.read_line(fid)\n return lin"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nresolve the indices for the skeleton from the channels when loading in channel data.", "response": "def resolve_indices(self, index, start_val):\n \"\"\"Get indices for the skeleton from the channels when loading in channel data.\"\"\"\n\n channels = self.vertices[index].meta['channels']\n base_channel = start_val \n rot_ind = -np.ones(3, dtype=int)\n pos_ind = -np.ones(3, dtype=int)\n for i in range(len(channels)):\n if channels[i]== 'Xrotation':\n rot_ind[0] = base_channel + i\n elif channels[i]=='Yrotation':\n rot_ind[1] = base_channel + i\n elif channels[i]=='Zrotation':\n rot_ind[2] = base_channel + i\n elif channels[i]=='Xposition':\n pos_ind[0] = base_channel + i\n elif channels[i]=='Yposition':\n pos_ind[1] = base_channel + i\n elif channels[i]=='Zposition':\n pos_ind[2] = base_channel + i\n self.vertices[index].meta['rot_ind'] = list(rot_ind)\n self.vertices[index].meta['pos_ind'] = list(pos_ind)"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef set_rotation_matrices(self):\n for i in range(len(self.vertices)):\n self.vertices[i].meta['C'] = rotation_matrix(self.vertices[i].meta['axis'][0], \n self.vertices[i].meta['axis'][1], \n self.vertices[i].meta['axis'][2], \n self.vertices[i].meta['axis_order'],\n degrees=True)\n # Todo: invert this by applying angle operations in reverse order\n self.vertices[i].meta['Cinv'] = np.linalg.inv(self.vertices[i].meta['C'])", "response": "Set the meta information at each vertex to contain the correct matrices C and Cinv as prescribed by the rotations and rotation orders."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef Kdiag(self, X):\n X_prod = self._comp_prod(X)\n return self.variance*four_over_tau*np.arcsin(X_prod/(X_prod+1.))", "response": "Compute the diagonal of the covariance matrix for X."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef update_gradients_full(self, dL_dK, X, X2=None):\n dvar, dw, db = self._comp_grads(dL_dK, X, X2)[:3]\n self.variance.gradient = dvar\n self.weight_variance.gradient = dw\n self.bias_variance.gradient = db", "response": "Update the gradients of the weighted covariance matrix with respect to the parameters."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef gradients_X(self, dL_dK, X, X2):\n return self._comp_grads(dL_dK, X, X2)[3]", "response": "Derivative of the covariance matrix with respect to X"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn the value of the parameters.", "response": "def _get_params(self):\n \"\"\"return the value of the parameters.\"\"\"\n return np.hstack((self.varianceU,self.varianceY, self.lengthscaleU,self.lengthscaleY))"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _set_params(self, x):\n assert x.size == self.num_params\n self.varianceU = x[0]\n self.varianceY = x[1]\n self.lengthscaleU = x[2]\n self.lengthscaleY = x[3]", "response": "set the value of the parameters."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef K(self, X, X2, target):\n if X2 is None: X2 = X\n # i1 = X[:,1]\n # i2 = X2[:,1]\n # X = X[:,0].reshape(-1,1)\n # X2 = X2[:,0].reshape(-1,1)\n dist = np.abs(X - X2.T)\n \n ly=1/self.lengthscaleY\n lu=np.sqrt(3)/self.lengthscaleU\n #ly=self.lengthscaleY\n #lu=self.lengthscaleU\n\n k1 = np.exp(-ly*dist)*(2*lu+ly)/(lu+ly)**2\n k2 = (np.exp(-lu*dist)*(ly-2*lu+lu*ly*dist-lu**2*dist) + np.exp(-ly*dist)*(2*lu-ly) ) / (ly-lu)**2 \n k3 = np.exp(-lu*dist) * ( (1+lu*dist)/(lu+ly) + (lu)/(lu+ly)**2 )\n\n np.add(self.varianceU*self.varianceY*(k1+k2+k3), target, target)", "response": "Compute the covariance matrix between X and X2."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef Kdiag(self, X, target):\n ly=1/self.lengthscaleY\n lu=np.sqrt(3)/self.lengthscaleU\n #ly=self.lengthscaleY\n #lu=self.lengthscaleU\n \n k1 = (2*lu+ly)/(lu+ly)**2\n k2 = (ly-2*lu + 2*lu-ly ) / (ly-lu)**2 \n k3 = 1/(lu+ly) + (lu)/(lu+ly)**2 \n\n np.add(self.varianceU*self.varianceY*(k1+k2+k3), target, target)", "response": "Compute the diagonal of the covariance matrix associated to X."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _param_grad_helper(self, dL_dK, X, X2, target):\n if X2 is None: X2 = X\n dist = np.abs(X - X2.T)\n\n ly=1/self.lengthscaleY\n lu=np.sqrt(3)/self.lengthscaleU\n #ly=self.lengthscaleY\n #lu=self.lengthscaleU\n\n dk1theta1 = np.exp(-ly*dist)*2*(-lu)/(lu+ly)**3\n #c=np.sqrt(3)\n #t1=c/lu\n #t2=1/ly\n #dk1theta1=np.exp(-dist*ly)*t2*( (2*c*t2+2*t1)/(c*t2+t1)**2 -2*(2*c*t2*t1+t1**2)/(c*t2+t1)**3 )\n \n dk2theta1 = 1*( \n np.exp(-lu*dist)*dist*(-ly+2*lu-lu*ly*dist+dist*lu**2)*(ly-lu)**(-2) + np.exp(-lu*dist)*(-2+ly*dist-2*dist*lu)*(ly-lu)**(-2) \n +np.exp(-dist*lu)*(ly-2*lu+ly*lu*dist-dist*lu**2)*2*(ly-lu)**(-3) \n +np.exp(-dist*ly)*2*(ly-lu)**(-2)\n +np.exp(-dist*ly)*2*(2*lu-ly)*(ly-lu)**(-3)\n )\n \n dk3theta1 = np.exp(-dist*lu)*(lu+ly)**(-2)*((2*lu+ly+dist*lu**2+lu*ly*dist)*(-dist-2/(lu+ly))+2+2*lu*dist+ly*dist)\n\n dktheta1 = self.varianceU*self.varianceY*(dk1theta1+dk2theta1+dk3theta1)\n\n\n\n\n dk1theta2 = np.exp(-ly*dist) * ((lu+ly)**(-2)) * ( (-dist)*(2*lu+ly) + 1 + (-2)*(2*lu+ly)/(lu+ly) )\n\n dk2theta2 = 1*(\n np.exp(-dist*lu)*(ly-lu)**(-2) * ( 1+lu*dist+(-2)*(ly-2*lu+lu*ly*dist-dist*lu**2)*(ly-lu)**(-1) )\n +np.exp(-dist*ly)*(ly-lu)**(-2) * ( (-dist)*(2*lu-ly) -1+(2*lu-ly)*(-2)*(ly-lu)**(-1) )\n )\n\n dk3theta2 = np.exp(-dist*lu) * (-3*lu-ly-dist*lu**2-lu*ly*dist)/(lu+ly)**3\n\n dktheta2 = self.varianceU*self.varianceY*(dk1theta2 + dk2theta2 +dk3theta2)\n\n\n\n k1 = np.exp(-ly*dist)*(2*lu+ly)/(lu+ly)**2\n k2 = (np.exp(-lu*dist)*(ly-2*lu+lu*ly*dist-lu**2*dist) + np.exp(-ly*dist)*(2*lu-ly) ) / (ly-lu)**2 \n k3 = np.exp(-lu*dist) * ( (1+lu*dist)/(lu+ly) + (lu)/(lu+ly)**2 )\n dkdvar = k1+k2+k3\n\n target[0] += np.sum(self.varianceY*dkdvar * dL_dK)\n target[1] += np.sum(self.varianceU*dkdvar * dL_dK)\n target[2] += np.sum(dktheta1*(-np.sqrt(3)*self.lengthscaleU**(-2)) * dL_dK)\n target[3] += np.sum(dktheta2*(-self.lengthscaleY**(-2)) * dL_dK)", "response": "Private function to calculate the derivative of the covariance matrix with respect to the parameters."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef inference(self, kern, X, likelihood, Y, Y_metadata=None):\n\n \"\"\"\n Returns a GridPosterior class containing essential quantities of the posterior\n \"\"\"\n N = X.shape[0] #number of training points\n D = X.shape[1] #number of dimensions\n\n Kds = np.zeros(D, dtype=object) #vector for holding covariance per dimension\n Qs = np.zeros(D, dtype=object) #vector for holding eigenvectors of covariance per dimension\n QTs = np.zeros(D, dtype=object) #vector for holding transposed eigenvectors of covariance per dimension\n V_kron = 1 # kronecker product of eigenvalues\n\n # retrieve the one-dimensional variation of the designated kernel\n oneDkernel = kern.get_one_dimensional_kernel(D)\n\n for d in range(D):\n xg = list(set(X[:,d])) #extract unique values for a dimension\n xg = np.reshape(xg, (len(xg), 1))\n oneDkernel.lengthscale = kern.lengthscale[d]\n Kds[d] = oneDkernel.K(xg)\n [V, Q] = np.linalg.eig(Kds[d])\n V_kron = np.kron(V_kron, V)\n Qs[d] = Q\n QTs[d] = Q.T\n\n noise = likelihood.variance + 1e-8\n\n alpha_kron = self.kron_mvprod(QTs, Y)\n V_kron = V_kron.reshape(-1, 1)\n alpha_kron = alpha_kron / (V_kron + noise)\n alpha_kron = self.kron_mvprod(Qs, alpha_kron)\n\n log_likelihood = -0.5 * (np.dot(Y.T, alpha_kron) + np.sum((np.log(V_kron + noise))) + N*log_2_pi)\n\n # compute derivatives wrt parameters Thete\n derivs = np.zeros(D+2, dtype='object')\n for t in range(len(derivs)):\n dKd_dTheta = np.zeros(D, dtype='object')\n gamma = np.zeros(D, dtype='object')\n gam = 1\n for d in range(D):\n xg = list(set(X[:,d]))\n xg = np.reshape(xg, (len(xg), 1))\n oneDkernel.lengthscale = kern.lengthscale[d]\n if t < D:\n dKd_dTheta[d] = oneDkernel.dKd_dLen(xg, (t==d), lengthscale=kern.lengthscale[t]) #derivative wrt lengthscale\n elif (t == D):\n dKd_dTheta[d] = oneDkernel.dKd_dVar(xg) #derivative wrt variance\n else:\n dKd_dTheta[d] = np.identity(len(xg)) #derivative wrt noise\n gamma[d] = np.diag(np.dot(np.dot(QTs[d], dKd_dTheta[d].T), Qs[d]))\n gam = np.kron(gam, gamma[d])\n \n gam = gam.reshape(-1,1)\n kappa = self.kron_mvprod(dKd_dTheta, alpha_kron)\n derivs[t] = 0.5*np.dot(alpha_kron.T,kappa) - 0.5*np.sum(gam / (V_kron + noise))\n\n # separate derivatives\n dL_dLen = derivs[:D]\n dL_dVar = derivs[D]\n dL_dThetaL = derivs[D+1]\n\n return GridPosterior(alpha_kron=alpha_kron, QTs=QTs, Qs=Qs, V_kron=V_kron), \\\n log_likelihood, {'dL_dLen':dL_dLen, 'dL_dVar':dL_dVar, 'dL_dthetaL':dL_dThetaL}", "response": "Inference method for the grid posterior."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nconvert the object into a json serializable dictionary.", "response": "def to_dict(self):\n \"\"\"\n Convert the object into a json serializable dictionary.\n\n Note: It uses the private method _save_to_input_dict of the parent.\n\n :return dict: json serializable dictionary containing the needed information to instantiate the object\n \"\"\"\n input_dict = super(Constant, self)._save_to_input_dict()\n input_dict[\"class\"] = \"GPy.mappings.Constant\"\n input_dict[\"value\"] = self.C.values[0]\n return input_dict"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef update_gradients(self, grads):\n self.sigma2.gradient = grads[0]\n self.v.gradient = grads[1]", "response": "Update the gradients of the sigma2 and v attributes of the current object."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef pdf_link(self, inv_link_f, y, Y_metadata=None):\n assert np.atleast_1d(inv_link_f).shape == np.atleast_1d(y).shape\n e = y - inv_link_f\n #Careful gamma(big_number) is infinity!\n objective = ((np.exp(gammaln((self.v + 1)*0.5) - gammaln(self.v * 0.5))\n / (np.sqrt(self.v * np.pi * self.sigma2)))\n * ((1 + (1./float(self.v))*((e**2)/float(self.sigma2)))**(-0.5*(self.v + 1)))\n )\n return np.prod(objective)", "response": "Evaluate the likelihood function given a link and a data set y"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef logpdf_link(self, inv_link_f, y, Y_metadata=None):\n e = y - inv_link_f\n #FIXME:\n #Why does np.log(1 + (1/self.v)*((y-inv_link_f)**2)/self.sigma2) suppress the divide by zero?!\n #But np.log(1 + (1/float(self.v))*((y-inv_link_f)**2)/self.sigma2) throws it correctly\n #print - 0.5*(self.v + 1)*np.log(1 + (1/np.float(self.v))*((e**2)/self.sigma2))\n objective = (+ gammaln((self.v + 1) * 0.5)\n - gammaln(self.v * 0.5)\n - 0.5*np.log(self.sigma2 * self.v * np.pi)\n - 0.5*(self.v + 1)*np.log(1 + (1/np.float(self.v))*((e**2)/self.sigma2))\n )\n return objective", "response": "Log PDF for the given link"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef dlogpdf_dlink(self, inv_link_f, y, Y_metadata=None):\n e = y - inv_link_f\n grad = ((self.v + 1) * e) / (self.v * self.sigma2 + (e**2))\n return grad", "response": "Gradient of the log likelihood function at y given link f"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef d3logpdf_dlink3(self, inv_link_f, y, Y_metadata=None):\n e = y - inv_link_f\n d3lik_dlink3 = ( -(2*(self.v + 1)*(-e)*(e**2 - 3*self.v*self.sigma2)) /\n ((e**2 + self.sigma2*self.v)**3)\n )\n return d3lik_dlink3", "response": "Function to compute the third order derivative log - likelihood function at y given link f"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ncalculating the derivative of the log - likelihood function at y given f given y given t_noise", "response": "def dlogpdf_link_dvar(self, inv_link_f, y, Y_metadata=None):\n \"\"\"\n Gradient of the log-likelihood function at y given f, w.r.t variance parameter (t_noise)\n\n .. math::\n \\\\frac{d \\\\ln p(y_{i}|\\lambda(f_{i}))}{d\\\\sigma^{2}} = \\\\frac{v((y_{i} - \\lambda(f_{i}))^{2} - \\\\sigma^{2})}{2\\\\sigma^{2}(\\\\sigma^{2}v + (y_{i} - \\lambda(f_{i}))^{2})}\n\n :param inv_link_f: latent variables link(f)\n :type inv_link_f: Nx1 array\n :param y: data\n :type y: Nx1 array\n :param Y_metadata: Y_metadata which is not used in student t distribution\n :returns: derivative of likelihood evaluated at points f w.r.t variance parameter\n :rtype: float\n \"\"\"\n e = y - inv_link_f\n e2 = np.square(e)\n dlogpdf_dvar = self.v*(e2 - self.sigma2)/(2*self.sigma2*(self.sigma2*self.v + e2))\n return dlogpdf_dvar"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef dlogpdf_dlink_dvar(self, inv_link_f, y, Y_metadata=None):\n e = y - inv_link_f\n dlogpdf_dlink_dvar = (self.v*(self.v+1)*(-e))/((self.sigma2*self.v + e**2)**2)\n return dlogpdf_dlink_dvar", "response": "Derivative of the dlogpdf_dlink w. r. t variance parameter"} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef d2logpdf_dlink2_dvar(self, inv_link_f, y, Y_metadata=None):\n e = y - inv_link_f\n d2logpdf_dlink2_dvar = ( (self.v*(self.v+1)*(self.sigma2*self.v - 3*(e**2)))\n / ((self.sigma2*self.v + (e**2))**3)\n )\n return d2logpdf_dlink2_dvar", "response": "Function to compute the derivative of the hessian evaluated at points f and f_j w. r. t variance parameter"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a set of samples of observations based on a given value of the latent variable.", "response": "def samples(self, gp, Y_metadata=None):\n \"\"\"\n Returns a set of samples of observations based on a given value of the latent variable.\n\n :param gp: latent variable\n \"\"\"\n orig_shape = gp.shape\n gp = gp.flatten()\n #FIXME: Very slow as we are computing a new random variable per input!\n #Can't get it to sample all at the same time\n #student_t_samples = np.array([stats.t.rvs(self.v, self.gp_link.transf(gpj),scale=np.sqrt(self.sigma2), size=1) for gpj in gp])\n dfs = np.ones_like(gp)*self.v\n scales = np.ones_like(gp)*np.sqrt(self.sigma2)\n student_t_samples = stats.t.rvs(dfs, loc=self.gp_link.transf(gp),\n scale=scales)\n return student_t_samples.reshape(orig_shape)"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _inner_parameters_changed(self, kern, X, Z, likelihood, Y, Y_metadata, Lm=None, dL_dKmm=None, psi0=None, psi1=None, psi2=None, **kwargs):\n return self.inference_method.inference(kern, X, Z, likelihood, Y, Y_metadata, Lm=Lm,\n dL_dKmm=dL_dKmm, psi0=psi0, psi1=psi1, psi2=psi2, **kwargs)", "response": "This method is used to compute the inner parameters of a kernel with respect to the parameters of the parameters_changed algorithm."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _inner_take_over_or_update(self, full_values=None, current_values=None, value_indices=None):\n for key in current_values.keys():\n if value_indices is not None and key in value_indices:\n index = value_indices[key]\n else:\n index = slice(None)\n if key in full_values:\n try:\n full_values[key][index] += current_values[key]\n except:\n full_values[key] += current_values[key]\n else:\n full_values[key] = current_values[key]", "response": "This function is used to take over or update the values in the inner loop of the data handling."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _outer_values_update(self, full_values):\n if self.has_uncertain_inputs():\n #gradients wrt kernel\n dL_dKmm = full_values['dL_dKmm']\n self.kern.update_gradients_full(dL_dKmm, self.Z, None)\n kgrad = self.kern.gradient.copy()\n self.kern.update_gradients_expectations(\n variational_posterior=self.X,\n Z=self.Z, dL_dpsi0=full_values['dL_dpsi0'],\n dL_dpsi1=full_values['dL_dpsi1'],\n dL_dpsi2=full_values['dL_dpsi2'])\n self.kern.gradient += kgrad\n\n\n #gradients wrt Z\n self.Z.gradient = self.kern.gradients_X(dL_dKmm, self.Z)\n self.Z.gradient += self.kern.gradients_Z_expectations(\n variational_posterior=self.X,\n Z=self.Z, dL_dpsi0=full_values['dL_dpsi0'],\n dL_dpsi1=full_values['dL_dpsi1'],\n dL_dpsi2=full_values['dL_dpsi2'])\n else:\n #gradients wrt kernel\n self.kern.update_gradients_diag(full_values['dL_dKdiag'], self.X)\n kgrad = self.kern.gradient.copy()\n self.kern.update_gradients_full(full_values['dL_dKnm'], self.X, self.Z)\n kgrad += self.kern.gradient\n self.kern.update_gradients_full(full_values['dL_dKmm'], self.Z, None)\n self.kern.gradient += kgrad\n #kgrad += self.kern.gradient\n\n #gradients wrt Z\n self.Z.gradient = self.kern.gradients_X(full_values['dL_dKmm'], self.Z)\n self.Z.gradient += self.kern.gradients_X(full_values['dL_dKnm'].T, self.Z, self.X)\n\n self.likelihood.update_gradients(full_values['dL_dthetaL'])", "response": "Update the gradients of the kernel and the likelihoods of the current object."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _outer_init_full_values(self):\n retd = dict(dL_dKmm=np.zeros((self.Z.shape[0], self.Z.shape[0])))\n if self.has_uncertain_inputs():\n retd.update(dict(dL_dpsi0=np.zeros(self.X.shape[0]),\n dL_dpsi1=np.zeros((self.X.shape[0], self.Z.shape[0])),\n dL_dpsi2=np.zeros((self.X.shape[0], self.Z.shape[0], self.Z.shape[0]))))\n else:\n retd.update({'dL_dKdiag': np.zeros(self.X.shape[0]),\n 'dL_dKnm': np.zeros((self.X.shape[0], self.Z.shape[0]))})\n return retd", "response": "This function initializes the full_values of the key - level entries for the current entry."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nstore the object into a json serializable dictionary", "response": "def to_dict(self, save_data=True):\n \"\"\"\n Store the object into a json serializable dictionary\n\n :param boolean save_data: if true, it adds the data self.X and self.Y to the dictionary\n :return dict: json serializable dictionary containing the needed information to instantiate the object\n \"\"\"\n model_dict = super(SparseGPClassification,self).to_dict(save_data)\n model_dict[\"class\"] = \"GPy.models.SparseGPClassification\"\n return model_dict"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nconvert the object into a json serializable dictionary.", "response": "def to_dict(self):\n \"\"\"\n Convert the object into a json serializable dictionary.\n\n Note: It uses the private method _save_to_input_dict of the parent.\n\n :return dict: json serializable dictionary containing the needed information to instantiate the object\n \"\"\"\n\n input_dict = super(Gaussian, self)._save_to_input_dict()\n input_dict[\"class\"] = \"GPy.likelihoods.Gaussian\"\n input_dict[\"variance\"] = self.variance.values.tolist()\n return input_dict"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef pdf_link(self, link_f, y, Y_metadata=None):\n #Assumes no covariance, exp, sum, log for numerical stability\n return np.exp(np.sum(np.log(stats.norm.pdf(y, link_f, np.sqrt(self.variance)))))", "response": "Evaluate the likelihood function given a link function and a data set y"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef logpdf_link(self, link_f, y, Y_metadata=None):\n ln_det_cov = np.log(self.variance)\n return -(1.0/(2*self.variance))*((y-link_f)**2) - 0.5*ln_det_cov - 0.5*np.log(2.*np.pi)", "response": "Log likelihood function given link f and data y"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef dlogpdf_dlink(self, link_f, y, Y_metadata=None):\n s2_i = 1.0/self.variance\n grad = s2_i*y - s2_i*link_f\n return grad", "response": "Gradient of the pdf at y given a latent variable link_f and a given y."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef d3logpdf_dlink3(self, link_f, y, Y_metadata=None):\n N = y.shape[0]\n D = link_f.shape[1]\n d3logpdf_dlink3 = np.zeros((N,D))\n return d3logpdf_dlink3", "response": "Function to compute the third order derivative log - likelihood function at y given link f and y given link f w. r. t y given link f and Y_metadata."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef dlogpdf_dlink_dvar(self, link_f, y, Y_metadata=None):\n s_4 = 1.0/(self.variance**2)\n dlik_grad_dsigma = -s_4*y + s_4*link_f\n return dlik_grad_dsigma", "response": "Derivative of the dlogpdf_dlink w. r. t variance parameter"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nfunctioning to compute the derivative of the log - likelihood of the hessian at points link_f and y.", "response": "def d2logpdf_dlink2_dvar(self, link_f, y, Y_metadata=None):\n \"\"\"\n Gradient of the hessian (d2logpdf_dlink2) w.r.t variance parameter (noise_variance)\n\n .. math::\n \\\\frac{d}{d\\\\sigma^{2}}(\\\\frac{d^{2} \\\\ln p(y_{i}|\\\\lambda(f_{i}))}{d^{2}\\\\lambda(f)}) = \\\\frac{1}{\\\\sigma^{4}}\n\n :param link_f: latent variables link(f)\n :type link_f: Nx1 array\n :param y: data\n :type y: Nx1 array\n :param Y_metadata: Y_metadata not used in gaussian\n :returns: derivative of log hessian evaluated at points link(f_i) and link(f_j) w.r.t variance parameter\n :rtype: Nx1 array\n \"\"\"\n s_4 = 1.0/(self.variance**2)\n N = y.shape[0]\n D = link_f.shape[1]\n d2logpdf_dlink2_dvar = np.ones((N, D))*s_4\n return d2logpdf_dlink2_dvar"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn a set of samples based on a given value of the latent variable.", "response": "def samples(self, gp, Y_metadata=None):\n \"\"\"\n Returns a set of samples of observations based on a given value of the latent variable.\n\n :param gp: latent variable\n \"\"\"\n orig_shape = gp.shape\n gp = gp.flatten()\n #orig_shape = gp.shape\n gp = gp.flatten()\n Ysim = np.array([np.random.normal(self.gp_link.transf(gpj), scale=np.sqrt(self.variance), size=1) for gpj in gp])\n return Ysim.reshape(orig_shape)"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nlogs predictive density of the log - predictive distribution.", "response": "def log_predictive_density(self, y_test, mu_star, var_star, Y_metadata=None):\n \"\"\"\n assumes independence\n \"\"\"\n v = var_star + self.variance\n return -0.5*np.log(2*np.pi) -0.5*np.log(v) - 0.5*np.square(y_test - mu_star)/v"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef to_dict(self, save_data=True):\n input_dict = super(GP, self)._save_to_input_dict()\n input_dict[\"class\"] = \"GPy.core.GP\"\n if not save_data:\n input_dict[\"X\"] = None\n input_dict[\"Y\"] = None\n else:\n try:\n input_dict[\"X\"] = self.X.values.tolist()\n except:\n input_dict[\"X\"] = self.X.tolist()\n try:\n input_dict[\"Y\"] = self.Y.values.tolist()\n except:\n input_dict[\"Y\"] = self.Y.tolist()\n input_dict[\"kernel\"] = self.kern.to_dict()\n input_dict[\"likelihood\"] = self.likelihood.to_dict()\n if self.mean_function is not None:\n input_dict[\"mean_function\"] = self.mean_function.to_dict()\n input_dict[\"inference_method\"] = self.inference_method.to_dict()\n #FIXME: Assumes the Y_metadata is serializable. We should create a Metadata class\n if self.Y_metadata is not None:\n input_dict[\"Y_metadata\"] = self.Y_metadata\n if self.normalizer is not None:\n input_dict[\"normalizer\"] = self.normalizer.to_dict()\n return input_dict", "response": "Convert the GP object into a json serializable dictionary."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef set_XY(self, X=None, Y=None):\n self.update_model(False)\n if Y is not None:\n if self.normalizer is not None:\n self.normalizer.scale_by(Y)\n self.Y_normalized = ObsAr(self.normalizer.normalize(Y))\n self.Y = Y\n else:\n self.Y = ObsAr(Y)\n self.Y_normalized = self.Y\n if X is not None:\n if self.X in self.parameters:\n # LVM models\n if isinstance(self.X, VariationalPosterior):\n assert isinstance(X, type(self.X)), \"The given X must have the same type as the X in the model!\"\n index = self.X._parent_index_\n self.unlink_parameter(self.X)\n self.X = X\n self.link_parameter(self.X, index=index)\n else:\n index = self.X._parent_index_\n self.unlink_parameter(self.X)\n from ..core import Param\n self.X = Param('latent mean', X)\n self.link_parameter(self.X, index=index)\n else:\n self.X = ObsAr(X)\n self.update_model(True)", "response": "Sets the input and output observations of the model to be the given data."} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\npredicts the latent function values at the new point Xnew.", "response": "def predict(self, Xnew, full_cov=False, Y_metadata=None, kern=None,\n likelihood=None, include_likelihood=True):\n \"\"\"\n Predict the function(s) at the new point(s) Xnew. This includes the\n likelihood variance added to the predicted underlying function\n (usually referred to as f).\n\n In order to predict without adding in the likelihood give\n `include_likelihood=False`, or refer to self.predict_noiseless().\n\n :param Xnew: The points at which to make a prediction\n :type Xnew: np.ndarray (Nnew x self.input_dim)\n :param full_cov: whether to return the full covariance matrix, or just\n the diagonal\n :type full_cov: bool\n :param Y_metadata: metadata about the predicting point to pass to the\n likelihood\n :param kern: The kernel to use for prediction (defaults to the model\n kern). this is useful for examining e.g. subprocesses.\n :param include_likelihood: Whether or not to add likelihood noise to\n the predicted underlying latent function f.\n :type include_likelihood: bool\n\n :returns: (mean, var):\n mean: posterior mean, a Numpy array, Nnew x self.input_dim\n var: posterior variance, a Numpy array, Nnew x 1 if full_cov=False,\n Nnew x Nnew otherwise\n\n If full_cov and self.input_dim > 1, the return shape of var is\n Nnew x Nnew x self.input_dim. If self.input_dim == 1, the return\n shape is Nnew x Nnew. This is to allow for different normalizations\n of the output dimensions.\n\n Note: If you want the predictive quantiles (e.g. 95% confidence\n interval) use :py:func:\"~GPy.core.gp.GP.predict_quantiles\".\n \"\"\"\n\n # Predict the latent function values\n mean, var = self._raw_predict(Xnew, full_cov=full_cov, kern=kern)\n\n if include_likelihood:\n # now push through likelihood\n if likelihood is None:\n likelihood = self.likelihood\n mean, var = likelihood.predictive_values(mean, var, full_cov,\n Y_metadata=Y_metadata)\n\n if self.normalizer is not None:\n mean = self.normalizer.inverse_mean(mean)\n\n # We need to create 3d array for the full covariance matrix with\n # multiple outputs.\n if full_cov & (mean.shape[1] > 1):\n var = self.normalizer.inverse_covariance(var)\n else:\n var = self.normalizer.inverse_variance(var)\n\n return mean, var"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef predict_noiseless(self, Xnew, full_cov=False, Y_metadata=None, kern=None):\n return self.predict(Xnew, full_cov, Y_metadata, kern, None, False)", "response": "This function is used to predict the underlying function of the GP with respect to the likelihood variance of the model."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef predict_quantiles(self, X, quantiles=(2.5, 97.5), Y_metadata=None, kern=None, likelihood=None):\n m, v = self._raw_predict(X, full_cov=False, kern=kern)\n if likelihood is None:\n likelihood = self.likelihood\n\n quantiles = likelihood.predictive_quantiles(m, v, quantiles, Y_metadata=Y_metadata)\n\n if self.normalizer is not None:\n quantiles = [self.normalizer.inverse_mean(q) for q in quantiles]\n return quantiles", "response": "Predict the quantiles around the prediction at X."} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncomputing the derivatives of the predicted latent function with respect to X", "response": "def predictive_gradients(self, Xnew, kern=None):\n \"\"\"\n Compute the derivatives of the predicted latent function with respect\n to X*\n\n Given a set of points at which to predict X* (size [N*,Q]), compute the\n derivatives of the mean and variance. Resulting arrays are sized:\n dmu_dX* -- [N*, Q ,D], where D is the number of output in this GP\n (usually one).\n\n Note that this is not the same as computing the mean and variance of\n the derivative of the function!\n\n dv_dX* -- [N*, Q], (since all outputs have the same variance)\n :param X: The points at which to get the predictive gradients\n :type X: np.ndarray (Xnew x self.input_dim)\n :returns: dmu_dX, dv_dX\n :rtype: [np.ndarray (N*, Q ,D), np.ndarray (N*,Q) ]\n\n \"\"\"\n if kern is None:\n kern = self.kern\n mean_jac = np.empty((Xnew.shape[0], Xnew.shape[1], self.output_dim))\n\n for i in range(self.output_dim):\n mean_jac[:, :, i] = kern.gradients_X(\n self.posterior.woodbury_vector[:, i:i+1].T, Xnew,\n self._predictive_variable)\n\n # Gradients wrt the diagonal part k_{xx}\n dv_dX = kern.gradients_X_diag(np.ones(Xnew.shape[0]), Xnew)\n\n # Grads wrt 'Schur' part K_{xf}K_{ff}^{-1}K_{fx}\n if self.posterior.woodbury_inv.ndim == 3:\n var_jac = np.empty(dv_dX.shape +\n (self.posterior.woodbury_inv.shape[2],))\n var_jac[:] = dv_dX[:, :, None]\n for i in range(self.posterior.woodbury_inv.shape[2]):\n alpha = -2.*np.dot(kern.K(Xnew, self._predictive_variable),\n self.posterior.woodbury_inv[:, :, i])\n var_jac[:, :, i] += kern.gradients_X(alpha, Xnew,\n self._predictive_variable)\n else:\n var_jac = dv_dX\n alpha = -2.*np.dot(kern.K(Xnew, self._predictive_variable),\n self.posterior.woodbury_inv)\n var_jac += kern.gradients_X(alpha, Xnew, self._predictive_variable)\n return mean_jac, var_jac"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef predict_jacobian(self, Xnew, kern=None, full_cov=False):\n if kern is None:\n kern = self.kern\n\n mean_jac = np.empty((Xnew.shape[0],Xnew.shape[1],self.output_dim))\n\n for i in range(self.output_dim):\n mean_jac[:,:,i] = kern.gradients_X(self.posterior.woodbury_vector[:,i:i+1].T, Xnew, self._predictive_variable)\n\n dK_dXnew_full = np.empty((self._predictive_variable.shape[0], Xnew.shape[0], Xnew.shape[1]))\n one = np.ones((1,1))\n for i in range(self._predictive_variable.shape[0]):\n dK_dXnew_full[i] = kern.gradients_X(one, Xnew, self._predictive_variable[[i]])\n\n if full_cov:\n dK2_dXdX = kern.gradients_XX(one, Xnew)\n else:\n dK2_dXdX = kern.gradients_XX_diag(one, Xnew)\n #dK2_dXdX = np.zeros((Xnew.shape[0], Xnew.shape[1], Xnew.shape[1]))\n #for i in range(Xnew.shape[0]):\n # dK2_dXdX[i:i+1,:,:] = kern.gradients_XX(one, Xnew[i:i+1,:])\n\n def compute_cov_inner(wi):\n if full_cov:\n var_jac = dK2_dXdX - np.einsum('qnm,msr->nsqr', dK_dXnew_full.T.dot(wi), dK_dXnew_full) # n,s = Xnew.shape[0], m = pred_var.shape[0]\n else:\n var_jac = dK2_dXdX - np.einsum('qnm,mnr->nqr', dK_dXnew_full.T.dot(wi), dK_dXnew_full)\n return var_jac\n\n if self.posterior.woodbury_inv.ndim == 3: # Missing data:\n if full_cov:\n var_jac = np.empty((Xnew.shape[0],Xnew.shape[0],Xnew.shape[1],Xnew.shape[1],self.output_dim))\n for d in range(self.posterior.woodbury_inv.shape[2]):\n var_jac[:, :, :, :, d] = compute_cov_inner(self.posterior.woodbury_inv[:, :, d])\n else:\n var_jac = np.empty((Xnew.shape[0],Xnew.shape[1],Xnew.shape[1],self.output_dim))\n for d in range(self.posterior.woodbury_inv.shape[2]):\n var_jac[:, :, :, d] = compute_cov_inner(self.posterior.woodbury_inv[:, :, d])\n else:\n var_jac = compute_cov_inner(self.posterior.woodbury_inv)\n return mean_jac, var_jac", "response": "Compute the jacobian of the posterior of the GP."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef predict_wishart_embedding(self, Xnew, kern=None, mean=True, covariance=True):\n if kern is None:\n kern = self.kern\n\n mu_jac, var_jac = self.predict_jacobian(Xnew, kern, full_cov=False)\n mumuT = np.einsum('iqd,ipd->iqp', mu_jac, mu_jac)\n Sigma = np.zeros(mumuT.shape)\n if var_jac.ndim == 4: # Missing data\n Sigma = var_jac.sum(-1)\n else:\n Sigma = self.output_dim*var_jac\n\n G = 0.\n if mean:\n G += mumuT\n if covariance:\n G += Sigma\n return G", "response": "Predict the wishart embedding G of the GP."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef predict_magnification(self, Xnew, kern=None, mean=True, covariance=True, dimensions=None):\n G = self.predict_wishart_embedding(Xnew, kern, mean, covariance)\n if dimensions is None:\n dimensions = self.get_most_significant_input_dimensions()[:2]\n G = G[:, dimensions][:,:,dimensions]\n from ..util.linalg import jitchol\n mag = np.empty(Xnew.shape[0])\n for n in range(Xnew.shape[0]):\n try:\n mag[n] = np.sqrt(np.exp(2*np.sum(np.log(np.diag(jitchol(G[n, :, :]))))))\n except:\n mag[n] = np.sqrt(np.linalg.det(G[n, :, :]))\n return mag", "response": "Predict the magnification factor for each point N in Xnew."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef posterior_samples_f(self,X, size=10, **predict_kwargs):\n predict_kwargs[\"full_cov\"] = True # Always use the full covariance for posterior samples. \n m, v = self._raw_predict(X, **predict_kwargs)\n if self.normalizer is not None:\n m, v = self.normalizer.inverse_mean(m), self.normalizer.inverse_variance(v)\n\n def sim_one_dim(m, v):\n return np.random.multivariate_normal(m, v, size).T\n\n if self.output_dim == 1:\n return sim_one_dim(m.flatten(), v)[:, np.newaxis, :]\n else:\n fsim = np.empty((X.shape[0], self.output_dim, size))\n for d in range(self.output_dim):\n if v.ndim == 3:\n fsim[:, d, :] = sim_one_dim(m[:, d], v[:, :, d])\n else:\n fsim[:, d, :] = sim_one_dim(m[:, d], v)\n return fsim", "response": "Samples the posterior GP at the points X."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef posterior_samples(self, X, size=10, Y_metadata=None, likelihood=None, **predict_kwargs):\n fsim = self.posterior_samples_f(X, size, **predict_kwargs)\n if likelihood is None:\n likelihood = self.likelihood\n if fsim.ndim == 3:\n for d in range(fsim.shape[1]):\n fsim[:, d] = likelihood.samples(fsim[:, d], Y_metadata=Y_metadata)\n else:\n fsim = likelihood.samples(fsim, Y_metadata=Y_metadata)\n return fsim", "response": "Samples the posterior GP at the points X."} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\noptimize the model using the given optimizer.", "response": "def optimize(self, optimizer=None, start=None, messages=False, max_iters=1000, ipython_notebook=True, clear_after_finish=False, **kwargs):\n \"\"\"\n Optimize the model using self.log_likelihood and self.log_likelihood_gradient, as well as self.priors.\n kwargs are passed to the optimizer. They can be:\n\n :param max_iters: maximum number of function evaluations\n :type max_iters: int\n :param messages: whether to display during optimisation\n :type messages: bool\n :param optimizer: which optimizer to use (defaults to self.preferred optimizer), a range of optimisers can be found in :module:`~GPy.inference.optimization`, they include 'scg', 'lbfgs', 'tnc'.\n :type optimizer: string\n :param bool ipython_notebook: whether to use ipython notebook widgets or not.\n :param bool clear_after_finish: if in ipython notebook, we can clear the widgets after optimization.\n \"\"\"\n self.inference_method.on_optimization_start()\n try:\n ret = super(GP, self).optimize(optimizer, start, messages, max_iters, ipython_notebook, clear_after_finish, **kwargs)\n except KeyboardInterrupt:\n print(\"KeyboardInterrupt caught, calling on_optimization_end() to round things up\")\n self.inference_method.on_optimization_end()\n raise\n return ret"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ninferring the new observed data X for the new observed data Y_new.", "response": "def infer_newX(self, Y_new, optimize=True):\n \"\"\"\n Infer X for the new observed data *Y_new*.\n\n :param Y_new: the new observed data for inference\n :type Y_new: numpy.ndarray\n :param optimize: whether to optimize the location of new X (True by default)\n :type optimize: boolean\n :return: a tuple containing the posterior estimation of X and the model that optimize X\n :rtype: (:class:`~GPy.core.parameterization.variational.VariationalPosterior` and numpy.ndarray, :class:`~GPy.core.model.Model`)\n \"\"\"\n from ..inference.latent_function_inference.inferenceX import infer_newX\n return infer_newX(self, Y_new, optimize=optimize)"} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncalculates the log predictive density of the log predictive distribution of the given test points.", "response": "def log_predictive_density(self, x_test, y_test, Y_metadata=None):\n \"\"\"\n Calculation of the log predictive density\n\n .. math:\n p(y_{*}|D) = p(y_{*}|f_{*})p(f_{*}|\\mu_{*}\\\\sigma^{2}_{*})\n\n :param x_test: test locations (x_{*})\n :type x_test: (Nx1) array\n :param y_test: test observations (y_{*})\n :type y_test: (Nx1) array\n :param Y_metadata: metadata associated with the test points\n \"\"\"\n mu_star, var_star = self._raw_predict(x_test)\n return self.likelihood.log_predictive_density(y_test, mu_star, var_star, Y_metadata=Y_metadata)"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef log_predictive_density_sampling(self, x_test, y_test, Y_metadata=None, num_samples=1000):\n mu_star, var_star = self._raw_predict(x_test)\n return self.likelihood.log_predictive_density_sampling(y_test, mu_star, var_star, Y_metadata=Y_metadata, num_samples=num_samples)", "response": "Calculates the log predictive density by sampling the log predictive density of the test points."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef posterior_covariance_between_points(self, X1, X2):\n return self.posterior.covariance_between_points(self.kern, self.X, X1, X2)", "response": "Computes the posterior covariance between points X1 and X2."} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ngenerate default dimensions and coordinates for a single variable.", "response": "def generate_dims_coords(shape, var_name, dims=None, coords=None, default_dims=None):\n \"\"\"Generate default dimensions and coordinates for a variable.\n\n Parameters\n ----------\n shape : tuple[int]\n Shape of the variable\n var_name : str\n Name of the variable. Used in the default name, if necessary\n dims : list\n List of dimensions for the variable\n coords : dict[str] -> list[str]\n Map of dimensions to coordinates\n default_dims : list[str]\n Dimensions that do not apply to the variable's shape\n\n Returns\n -------\n list[str]\n Default dims\n dict[str] -> list[str]\n Default coords\n \"\"\"\n if default_dims is None:\n default_dims = []\n if dims is None:\n dims = []\n if len([dim for dim in dims if dim not in default_dims]) > len(shape):\n warnings.warn(\n \"More dims ({dims_len}) given than exists ({shape_len}). \"\n \"Passed array should have shape (chains, draws, *shape)\".format(\n dims_len=len(dims), shape_len=len(shape)\n ),\n SyntaxWarning,\n )\n if coords is None:\n coords = {}\n\n coords = deepcopy(coords)\n dims = deepcopy(dims)\n\n for idx, dim_len in enumerate(shape):\n if (len(dims) < idx + 1) or (dims[idx] is None):\n dim_name = \"{var_name}_dim_{idx}\".format(var_name=var_name, idx=idx)\n if len(dims) < idx + 1:\n dims.append(dim_name)\n else:\n dims[idx] = dim_name\n dim_name = dims[idx]\n if dim_name not in coords:\n coords[dim_name] = np.arange(dim_len)\n coords = {key: coord for key, coord in coords.items() if any(key == dim for dim in dims)}\n return dims, coords"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef numpy_to_data_array(ary, *, var_name=\"data\", coords=None, dims=None):\n # manage and transform copies\n default_dims = [\"chain\", \"draw\"]\n ary = np.atleast_2d(ary)\n n_chains, n_samples, *shape = ary.shape\n if n_chains > n_samples:\n warnings.warn(\n \"More chains ({n_chains}) than draws ({n_samples}). \"\n \"Passed array should have shape (chains, draws, *shape)\".format(\n n_chains=n_chains, n_samples=n_samples\n ),\n SyntaxWarning,\n )\n\n dims, coords = generate_dims_coords(\n shape, var_name, dims=dims, coords=coords, default_dims=default_dims\n )\n\n # reversed order for default dims: 'chain', 'draw'\n if \"draw\" not in dims:\n dims = [\"draw\"] + dims\n if \"chain\" not in dims:\n dims = [\"chain\"] + dims\n\n if \"chain\" not in coords:\n coords[\"chain\"] = np.arange(n_chains)\n if \"draw\" not in coords:\n coords[\"draw\"] = np.arange(n_samples)\n\n # filter coords based on the dims\n coords = {key: xr.IndexVariable((key,), data=coords[key]) for key in dims}\n return xr.DataArray(ary, coords=coords, dims=dims)", "response": "Convert a numpy array to an xarray. DataArray."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef dict_to_dataset(data, *, attrs=None, library=None, coords=None, dims=None):\n if dims is None:\n dims = {}\n\n data_vars = {}\n for key, values in data.items():\n data_vars[key] = numpy_to_data_array(\n values, var_name=key, coords=coords, dims=dims.get(key)\n )\n return xr.Dataset(data_vars=data_vars, attrs=make_attrs(attrs=attrs, library=library))", "response": "Convert a dictionary of numpy arrays to an xarray. Dataset."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef make_attrs(attrs=None, library=None):\n default_attrs = {\"created_at\": datetime.datetime.utcnow().isoformat()}\n if library is not None:\n library_name = library.__name__\n default_attrs[\"inference_library\"] = library_name\n try:\n version = pkg_resources.get_distribution(library_name).version\n default_attrs[\"inference_library_version\"] = version\n except pkg_resources.DistributionNotFound:\n if hasattr(library, \"__version__\"):\n version = library.__version__\n default_attrs[\"inference_library_version\"] = version\n\n if attrs is not None:\n default_attrs.update(attrs)\n return default_attrs", "response": "Make standard attributes to attach to xarray datasets."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_data_home(data_home=None):\n if data_home is None:\n data_home = os.environ.get(\"ARVIZ_DATA\", os.path.join(\"~\", \"arviz_data\"))\n data_home = os.path.expanduser(data_home)\n if not os.path.exists(data_home):\n os.makedirs(data_home)\n return data_home", "response": "Return the path to the arviz data dir."} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncalculate the sha256 hash of the file at path.", "response": "def _sha256(path):\n \"\"\"Calculate the sha256 hash of the file at path.\"\"\"\n sha256hash = hashlib.sha256()\n chunk_size = 8192\n with open(path, \"rb\") as buff:\n while True:\n buffer = buff.read(chunk_size)\n if not buffer:\n break\n sha256hash.update(buffer)\n return sha256hash.hexdigest()"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nloading a local or remote pre - made dataset.", "response": "def load_arviz_data(dataset=None, data_home=None):\n \"\"\"Load a local or remote pre-made dataset.\n\n Run with no parameters to get a list of all available models.\n\n The directory to save to can also be set with the environement\n variable `ARVIZ_HOME`. The checksum of the dataset is checked against a\n hardcoded value to watch for data corruption.\n\n Run `az.clear_data_home` to clear the data directory.\n\n Parameters\n ----------\n dataset : str\n Name of dataset to load.\n\n data_home : str, optional\n Where to save remote datasets\n\n Returns\n -------\n xarray.Dataset\n \"\"\"\n if dataset in LOCAL_DATASETS:\n resource = LOCAL_DATASETS[dataset]\n return from_netcdf(resource.filename)\n\n elif dataset in REMOTE_DATASETS:\n remote = REMOTE_DATASETS[dataset]\n home_dir = get_data_home(data_home=data_home)\n file_path = os.path.join(home_dir, remote.filename)\n if not os.path.exists(file_path):\n urlretrieve(remote.url, file_path)\n checksum = _sha256(file_path)\n if remote.checksum != checksum:\n raise IOError(\n \"{} has an SHA256 checksum ({}) differing from expected ({}), \"\n \"file may be corrupted. Run `arviz.clear_data_home()` and try \"\n \"again, or please open an issue.\".format(file_path, checksum, remote.checksum)\n )\n return from_netcdf(file_path)\n else:\n raise ValueError(\n \"Dataset {} not found! The following are available:\\n{}\".format(\n dataset, list_datasets()\n )\n )"} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ngets a string representation of all available datasets with descriptions.", "response": "def list_datasets():\n \"\"\"Get a string representation of all available datasets with descriptions.\"\"\"\n lines = []\n for name, resource in itertools.chain(LOCAL_DATASETS.items(), REMOTE_DATASETS.items()):\n\n if isinstance(resource, LocalFileMetadata):\n location = \"local: {}\".format(resource.filename)\n elif isinstance(resource, RemoteFileMetadata):\n location = \"remote: {}\".format(resource.url)\n else:\n location = \"unknown\"\n lines.append(\"{}\\n{}\\n{}\\n{}\".format(name, \"=\" * len(name), resource.description, location))\n\n return \"\\n\\n{}\\n\\n\".format(10 * \"-\").join(lines)"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nmaking sure var_names and arg_names are assigned reasonably. This is meant to run before loading emcee objects into InferenceData. In case var_names or arg_names is None, will provide defaults. If they are not None, it verifies there are the right number of them. Throws a ValueError in case validation fails. Parameters ---------- sampler : emcee.EnsembleSampler Fitted emcee sampler var_names : list[str] or None Names for the emcee parameters arg_names : list[str] or None Names for the args/observations provided to emcee Returns ------- list[str], list[str] Defaults for var_names and arg_names", "response": "def _verify_names(sampler, var_names, arg_names):\n \"\"\"Make sure var_names and arg_names are assigned reasonably.\n\n This is meant to run before loading emcee objects into InferenceData.\n In case var_names or arg_names is None, will provide defaults. If they are\n not None, it verifies there are the right number of them.\n\n Throws a ValueError in case validation fails.\n\n Parameters\n ----------\n sampler : emcee.EnsembleSampler\n Fitted emcee sampler\n var_names : list[str] or None\n Names for the emcee parameters\n arg_names : list[str] or None\n Names for the args/observations provided to emcee\n\n Returns\n -------\n list[str], list[str]\n Defaults for var_names and arg_names\n \"\"\"\n # There are 3 possible cases: emcee2, emcee3 and sampler read from h5 file (emcee3 only)\n if hasattr(sampler, \"args\"):\n num_vars = sampler.chain.shape[-1]\n num_args = len(sampler.args)\n elif hasattr(sampler, \"log_prob_fn\"):\n num_vars = sampler.get_chain().shape[-1]\n num_args = len(sampler.log_prob_fn.args)\n else:\n num_vars = sampler.get_chain().shape[-1]\n num_args = 0 # emcee only stores the posterior samples\n\n if var_names is None:\n var_names = [\"var_{}\".format(idx) for idx in range(num_vars)]\n if arg_names is None:\n arg_names = [\"arg_{}\".format(idx) for idx in range(num_args)]\n\n if len(var_names) != num_vars:\n raise ValueError(\n \"The sampler has {} variables, but only {} var_names were provided!\".format(\n num_vars, len(var_names)\n )\n )\n\n if len(arg_names) != num_args:\n raise ValueError(\n \"The sampler has {} args, but only {} arg_names were provided!\".format(\n num_args, len(arg_names)\n )\n )\n return var_names, arg_names"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef from_emcee(sampler=None, *, var_names=None, arg_names=None, coords=None, dims=None):\n return EmceeConverter(\n sampler=sampler, var_names=var_names, arg_names=arg_names, coords=coords, dims=dims\n ).to_inference_data()", "response": "Convert emcee data into an InferenceData object."} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nconvert the posterior to an xarray dataset.", "response": "def posterior_to_xarray(self):\n \"\"\"Convert the posterior to an xarray dataset.\"\"\"\n data = {}\n for idx, var_name in enumerate(self.var_names):\n # Use emcee3 syntax, else use emcee2\n data[var_name] = (\n self.sampler.get_chain()[(..., idx)].T\n if hasattr(self.sampler, \"get_chain\")\n else self.sampler.chain[(..., idx)]\n )\n return dict_to_dataset(data, library=self.emcee, coords=self.coords, dims=self.dims)"} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nconverting observed data to xarray.", "response": "def observed_data_to_xarray(self):\n \"\"\"Convert observed data to xarray.\"\"\"\n data = {}\n for idx, var_name in enumerate(self.arg_names):\n # Use emcee3 syntax, else use emcee2\n data[var_name] = (\n self.sampler.log_prob_fn.args[idx]\n if hasattr(self.sampler, \"log_prob_fn\")\n else self.sampler.args[idx]\n )\n return dict_to_dataset(data, library=self.emcee, coords=self.coords, dims=self.dims)"} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nconvert a TFP data into an InferenceData object.", "response": "def from_tfp(\r\n posterior=None,\r\n *,\r\n var_names=None,\r\n model_fn=None,\r\n feed_dict=None,\r\n posterior_predictive_samples=100,\r\n posterior_predictive_size=1,\r\n observed=None,\r\n coords=None,\r\n dims=None\r\n):\r\n \"\"\"Convert tfp data into an InferenceData object.\"\"\"\r\n return TfpConverter(\r\n posterior=posterior,\r\n var_names=var_names,\r\n model_fn=model_fn,\r\n feed_dict=feed_dict,\r\n posterior_predictive_samples=posterior_predictive_samples,\r\n posterior_predictive_size=posterior_predictive_size,\r\n observed=observed,\r\n coords=coords,\r\n dims=dims,\r\n ).to_inference_data()"} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef posterior_to_xarray(self):\r\n data = {}\r\n for i, var_name in enumerate(self.var_names):\r\n data[var_name] = np.expand_dims(self.posterior[i], axis=0)\r\n return dict_to_dataset(data, library=self.tfp, coords=self.coords, dims=self.dims)", "response": "Convert the posterior to an xarray dataset."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef observed_data_to_xarray(self):\r\n if self.observed is None:\r\n return None\r\n\r\n observed_data = {}\r\n if isinstance(self.observed, self.tf.Tensor):\r\n with self.tf.Session() as sess:\r\n vals = sess.run(self.observed, feed_dict=self.feed_dict)\r\n else:\r\n vals = self.observed\r\n\r\n if self.dims is None:\r\n dims = {}\r\n else:\r\n dims = self.dims\r\n\r\n name = \"obs\"\r\n val_dims = dims.get(name)\r\n vals = np.atleast_1d(vals)\r\n val_dims, coords = generate_dims_coords(vals.shape, name, dims=val_dims, coords=self.coords)\r\n # coords = {key: xr.IndexVariable((key,), data=coords[key]) for key in val_dims}\r\n\r\n observed_data[name] = xr.DataArray(vals, dims=val_dims, coords=coords)\r\n return xr.Dataset(data_vars=observed_data, attrs=make_attrs(library=self.tfp))", "response": "Convert observed data to xarray."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef posterior_predictive_to_xarray(self):\r\n if self.model_fn is None:\r\n return None\r\n\r\n posterior_preds = []\r\n sample_size = self.posterior[0].shape[0]\r\n\r\n for i in np.arange(0, sample_size, int(sample_size / self.posterior_predictive_samples)):\r\n variables = {}\r\n for var_i, var_name in enumerate(self.var_names):\r\n variables[var_name] = self.posterior[var_i][i]\r\n\r\n with self.ed.interception(self._value_setter(variables)):\r\n if self.posterior_predictive_size > 1:\r\n posterior_preds.append(\r\n [self.model_fn() for _ in range(self.posterior_predictive_size)]\r\n )\r\n else:\r\n posterior_preds.append(self.model_fn())\r\n\r\n data = {}\r\n with self.tf.Session() as sess:\r\n data[\"obs\"] = np.expand_dims(\r\n sess.run(posterior_preds, feed_dict=self.feed_dict), axis=0\r\n )\r\n return dict_to_dataset(data, library=self.tfp, coords=self.coords, dims=self.dims)", "response": "Convert posterior predictive samples to xarray."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nextract sample_stats from tfp trace.", "response": "def sample_stats_to_xarray(self):\r\n \"\"\"Extract sample_stats from tfp trace.\"\"\"\r\n if self.model_fn is None or self.observed is None:\r\n return None\r\n\r\n log_likelihood = []\r\n sample_size = self.posterior[0].shape[0]\r\n\r\n for i in range(sample_size):\r\n variables = {}\r\n for var_i, var_name in enumerate(self.var_names):\r\n variables[var_name] = self.posterior[var_i][i]\r\n\r\n with self.ed.interception(self._value_setter(variables)):\r\n log_likelihood.append((self.model_fn().distribution.log_prob(self.observed)))\r\n\r\n data = {}\r\n if self.dims is not None:\r\n coord_name = self.dims.get(\"obs\")\r\n else:\r\n coord_name = None\r\n dims = {\"log_likelihood\": coord_name}\r\n\r\n with self.tf.Session() as sess:\r\n data[\"log_likelihood\"] = np.expand_dims(\r\n sess.run(log_likelihood, feed_dict=self.feed_dict), axis=0\r\n )\r\n return dict_to_dataset(data, library=self.tfp, coords=self.coords, dims=dims)"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef to_inference_data(self):\r\n return InferenceData(\r\n **{\r\n \"posterior\": self.posterior_to_xarray(),\r\n \"sample_stats\": self.sample_stats_to_xarray(),\r\n \"posterior_predictive\": self.posterior_predictive_to_xarray(),\r\n \"observed_data\": self.observed_data_to_xarray(),\r\n }\r\n )", "response": "Convert all available data to an InferenceData object."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nconverts pymc3 data into an InferenceData object.", "response": "def from_pymc3(trace=None, *, prior=None, posterior_predictive=None, coords=None, dims=None):\n \"\"\"Convert pymc3 data into an InferenceData object.\"\"\"\n return PyMC3Converter(\n trace=trace,\n prior=prior,\n posterior_predictive=posterior_predictive,\n coords=coords,\n dims=dims,\n ).to_inference_data()"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _extract_log_likelihood(self):\n # This next line is brittle and may not work forever, but is a secret\n # way to access the model from the trace.\n model = self.trace._straces[0].model # pylint: disable=protected-access\n if len(model.observed_RVs) != 1:\n return None, None\n else:\n if self.dims is not None:\n coord_name = self.dims.get(model.observed_RVs[0].name)\n else:\n coord_name = None\n\n cached = [(var, var.logp_elemwise) for var in model.observed_RVs]\n\n def log_likelihood_vals_point(point):\n \"\"\"Compute log likelihood for each observed point.\"\"\"\n log_like_vals = []\n for var, log_like in cached:\n log_like_val = log_like(point)\n if var.missing_values:\n log_like_val = log_like_val[~var.observations.mask]\n log_like_vals.append(log_like_val.ravel())\n return np.concatenate(log_like_vals)\n\n chain_likelihoods = []\n for chain in self.trace.chains:\n log_like = (log_likelihood_vals_point(point) for point in self.trace.points([chain]))\n chain_likelihoods.append(np.stack(log_like))\n return np.stack(chain_likelihoods), coord_name", "response": "Extract the log likelihood of each observation."} +{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef posterior_to_xarray(self):\n var_names = self.pymc3.util.get_default_varnames( # pylint: disable=no-member\n self.trace.varnames, include_transformed=False\n )\n data = {}\n for var_name in var_names:\n data[var_name] = np.array(self.trace.get_values(var_name, combine=False, squeeze=False))\n return dict_to_dataset(data, library=self.pymc3, coords=self.coords, dims=self.dims)", "response": "Convert the posterior to an xarray dataset."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef sample_stats_to_xarray(self):\n rename_key = {\"model_logp\": \"lp\"}\n data = {}\n for stat in self.trace.stat_names:\n name = rename_key.get(stat, stat)\n data[name] = np.array(self.trace.get_sampler_stats(stat, combine=False))\n log_likelihood, dims = self._extract_log_likelihood()\n if log_likelihood is not None:\n data[\"log_likelihood\"] = log_likelihood\n dims = {\"log_likelihood\": dims}\n else:\n dims = None\n\n return dict_to_dataset(data, library=self.pymc3, dims=dims, coords=self.coords)", "response": "Extract sample_stats from PyMC3 trace."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef posterior_predictive_to_xarray(self):\n data = {k: np.expand_dims(v, 0) for k, v in self.posterior_predictive.items()}\n return dict_to_dataset(data, library=self.pymc3, coords=self.coords, dims=self.dims)", "response": "Convert posterior_predictive samples to xarray."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nconverts prior samples to xarray.", "response": "def prior_to_xarray(self):\n \"\"\"Convert prior samples to xarray.\"\"\"\n return dict_to_dataset(\n {k: np.expand_dims(v, 0) for k, v in self.prior.items()},\n library=self.pymc3,\n coords=self.coords,\n dims=self.dims,\n )"} +{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nconverts observed data to xarray.", "response": "def observed_data_to_xarray(self):\n \"\"\"Convert observed data to xarray.\"\"\"\n # This next line is brittle and may not work forever, but is a secret\n # way to access the model from the trace.\n model = self.trace._straces[0].model # pylint: disable=protected-access\n\n observations = {obs.name: obs.observations for obs in model.observed_RVs}\n if self.dims is None:\n dims = {}\n else:\n dims = self.dims\n observed_data = {}\n for name, vals in observations.items():\n vals = np.atleast_1d(vals)\n val_dims = dims.get(name)\n val_dims, coords = generate_dims_coords(\n vals.shape, name, dims=val_dims, coords=self.coords\n )\n # filter coords based on the dims\n coords = {key: xr.IndexVariable((key,), data=coords[key]) for key in val_dims}\n observed_data[name] = xr.DataArray(vals, dims=val_dims, coords=coords)\n return xr.Dataset(data_vars=observed_data, attrs=make_attrs(library=self.pymc3))"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef plot_posterior(\n data,\n var_names=None,\n coords=None,\n figsize=None,\n textsize=None,\n credible_interval=0.94,\n round_to=1,\n point_estimate=\"mean\",\n rope=None,\n ref_val=None,\n kind=\"kde\",\n bw=4.5,\n bins=None,\n ax=None,\n **kwargs\n):\n \"\"\"Plot Posterior densities in the style of John K. Kruschke's book.\n\n Parameters\n ----------\n data : obj\n Any object that can be converted to an az.InferenceData object\n Refer to documentation of az.convert_to_dataset for details\n var_names : list of variable names\n Variables to be plotted, two variables are required.\n coords : mapping, optional\n Coordinates of var_names to be plotted. Passed to `Dataset.sel`\n figsize : tuple\n Figure size. If None it will be defined automatically.\n textsize: float\n Text size scaling factor for labels, titles and lines. If None it will be autoscaled based\n on figsize.\n credible_interval : float, optional\n Credible intervals. Defaults to 0.94.\n round_to : int\n Controls formatting for floating point numbers\n point_estimate: str\n Must be in ('mode', 'mean', 'median')\n rope: tuple or dictionary of tuples\n Lower and upper values of the Region Of Practical Equivalence. If a list is provided, its\n length should match the number of variables.\n ref_val: float or dictionary of floats\n display the percentage below and above the values in ref_val. If a list is provided, its\n length should match the number of variables.\n kind: str\n Type of plot to display (kde or hist) For discrete variables this argument is ignored and\n a histogram is always used.\n bw : float\n Bandwidth scaling factor for the KDE. Should be larger than 0. The higher this number the\n smoother the KDE will be. Defaults to 4.5 which is essentially the same as the Scott's rule\n of thumb (the default rule used by SciPy). Only works if `kind == kde`.\n bins : integer or sequence or 'auto', optional\n Controls the number of bins, accepts the same keywords `matplotlib.hist()` does. Only works\n if `kind == hist`. If None (default) it will use `auto` for continuous variables and\n `range(xmin, xmax + 1)` for discrete variables.\n ax : axes\n Matplotlib axes. Defaults to None.\n **kwargs\n Passed as-is to plt.hist() or plt.plot() function depending on the value of `kind`.\n\n Returns\n -------\n ax : matplotlib axes\n\n Examples\n --------\n Show a default kernel density plot following style of John Kruschke\n\n .. plot::\n :context: close-figs\n\n >>> import arviz as az\n >>> data = az.load_arviz_data('centered_eight')\n >>> az.plot_posterior(data)\n\n Plot subset variables by specifying variable name exactly\n\n .. plot::\n :context: close-figs\n\n >>> az.plot_posterior(data, var_names=['mu'])\n\n Plot Region of Practical Equivalence (rope) for all distributions\n\n .. plot::\n :context: close-figs\n\n >>> az.plot_posterior(data, var_names=['mu', 'theta'], rope=(-1, 1))\n\n Plot Region of Practical Equivalence for selected distributions\n\n .. plot::\n :context: close-figs\n\n >>> rope = {'mu': [{'rope': (-2, 2)}], 'theta': [{'school': 'Choate', 'rope': (2, 4)}]}\n >>> az.plot_posterior(data, var_names=['mu', 'theta'], rope=rope)\n\n\n Add reference lines\n\n .. plot::\n :context: close-figs\n\n >>> az.plot_posterior(data, var_names=['mu', 'theta_tilde'], ref_val=0)\n\n Show point estimate of distribution\n\n .. plot::\n :context: close-figs\n\n >>> az.plot_posterior(data, var_names=['mu', 'theta_tilde'], point_estimate='mode')\n\n Plot posterior as a histogram\n\n .. plot::\n :context: close-figs\n\n >>> az.plot_posterior(data, var_names=['mu'], kind='hist')\n\n Change size of credible interval\n\n .. plot::\n :context: close-figs\n\n >>> az.plot_posterior(data, var_names=['mu'], credible_interval=.75)\n \"\"\"\n data = convert_to_dataset(data, group=\"posterior\")\n var_names = _var_names(var_names, data)\n\n if coords is None:\n coords = {}\n\n plotters = list(xarray_var_iter(get_coords(data, coords), var_names=var_names, combined=True))\n length_plotters = len(plotters)\n rows, cols = default_grid(length_plotters)\n\n (figsize, ax_labelsize, titlesize, xt_labelsize, _linewidth, _) = _scale_fig_size(\n figsize, textsize, rows, cols\n )\n kwargs.setdefault(\"linewidth\", _linewidth)\n\n if ax is None:\n _, ax = _create_axes_grid(\n length_plotters, rows, cols, figsize=figsize, squeeze=False, constrained_layout=True\n )\n\n for (var_name, selection, x), ax_ in zip(plotters, np.ravel(ax)):\n _plot_posterior_op(\n x.flatten(),\n var_name,\n selection,\n ax=ax_,\n bw=bw,\n bins=bins,\n kind=kind,\n point_estimate=point_estimate,\n round_to=round_to,\n credible_interval=credible_interval,\n ref_val=ref_val,\n rope=rope,\n ax_labelsize=ax_labelsize,\n xt_labelsize=xt_labelsize,\n **kwargs\n )\n\n ax_.set_title(make_label(var_name, selection), fontsize=titlesize, wrap=True)\n\n return ax", "response": "Plots the posterior densities of a set of variables in a set of variables."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _plot_posterior_op(\n values,\n var_name,\n selection,\n ax,\n bw,\n linewidth,\n bins,\n kind,\n point_estimate,\n round_to,\n credible_interval,\n ref_val,\n rope,\n ax_labelsize,\n xt_labelsize,\n **kwargs\n): # noqa: D202\n \"\"\"Artist to draw posterior.\"\"\"\n\n def format_as_percent(x, round_to=0):\n return \"{0:.{1:d}f}%\".format(100 * x, round_to)\n\n def display_ref_val():\n if ref_val is None:\n return\n elif isinstance(ref_val, dict):\n val = None\n for sel in ref_val.get(var_name, []):\n if all(\n k in selection and selection[k] == v for k, v in sel.items() if k != \"ref_val\"\n ):\n val = sel[\"ref_val\"]\n break\n if val is None:\n return\n elif isinstance(ref_val, Number):\n val = ref_val\n else:\n raise ValueError(\n \"Argument `ref_val` must be None, a constant, or a \"\n 'dictionary like {\"var_name\": {\"ref_val\": (lo, hi)}}'\n )\n less_than_ref_probability = (values < val).mean()\n greater_than_ref_probability = (values >= val).mean()\n ref_in_posterior = \"{} <{:g}< {}\".format(\n format_as_percent(less_than_ref_probability, 1),\n val,\n format_as_percent(greater_than_ref_probability, 1),\n )\n ax.axvline(val, ymin=0.05, ymax=0.75, color=\"C1\", lw=linewidth, alpha=0.65)\n ax.text(\n values.mean(),\n plot_height * 0.6,\n ref_in_posterior,\n size=ax_labelsize,\n color=\"C1\",\n weight=\"semibold\",\n horizontalalignment=\"center\",\n )\n\n def display_rope():\n if rope is None:\n return\n elif isinstance(rope, dict):\n vals = None\n for sel in rope.get(var_name, []):\n if all(k in selection and selection[k] == v for k, v in sel.items() if k != \"rope\"):\n vals = sel[\"rope\"]\n break\n if vals is None:\n return\n elif len(rope) == 2:\n vals = rope\n else:\n raise ValueError(\n \"Argument `rope` must be None, a dictionary like\"\n '{\"var_name\": {\"rope\": (lo, hi)}}, or an'\n \"iterable of length 2\"\n )\n\n ax.plot(\n vals,\n (plot_height * 0.02, plot_height * 0.02),\n lw=linewidth * 5,\n color=\"C2\",\n solid_capstyle=\"round\",\n zorder=0,\n alpha=0.7,\n )\n text_props = {\"size\": ax_labelsize, \"horizontalalignment\": \"center\", \"color\": \"C2\"}\n ax.text(vals[0], plot_height * 0.2, vals[0], weight=\"semibold\", **text_props)\n ax.text(vals[1], plot_height * 0.2, vals[1], weight=\"semibold\", **text_props)\n\n def display_point_estimate():\n if not point_estimate:\n return\n if point_estimate not in (\"mode\", \"mean\", \"median\"):\n raise ValueError(\"Point Estimate should be in ('mode','mean','median')\")\n if point_estimate == \"mean\":\n point_value = values.mean()\n elif point_estimate == \"mode\":\n if isinstance(values[0], float):\n density, lower, upper = _fast_kde(values, bw=bw)\n x = np.linspace(lower, upper, len(density))\n point_value = x[np.argmax(density)]\n else:\n point_value = mode(values.round(round_to))[0][0]\n elif point_estimate == \"median\":\n point_value = np.median(values)\n point_text = \"{}={:.{}f}\".format(point_estimate, point_value, round_to)\n\n ax.text(\n point_value,\n plot_height * 0.8,\n point_text,\n size=ax_labelsize,\n horizontalalignment=\"center\",\n )\n\n def display_hpd():\n hpd_intervals = hpd(values, credible_interval=credible_interval)\n ax.plot(\n hpd_intervals,\n (plot_height * 0.02, plot_height * 0.02),\n lw=linewidth * 2,\n color=\"k\",\n solid_capstyle=\"round\",\n )\n ax.text(\n hpd_intervals[0],\n plot_height * 0.07,\n hpd_intervals[0].round(round_to),\n size=ax_labelsize,\n horizontalalignment=\"center\",\n )\n ax.text(\n hpd_intervals[1],\n plot_height * 0.07,\n hpd_intervals[1].round(round_to),\n size=ax_labelsize,\n horizontalalignment=\"center\",\n )\n ax.text(\n (hpd_intervals[0] + hpd_intervals[1]) / 2,\n plot_height * 0.3,\n format_as_percent(credible_interval) + \" HPD\",\n size=ax_labelsize,\n horizontalalignment=\"center\",\n )\n\n def format_axes():\n ax.yaxis.set_ticks([])\n ax.spines[\"top\"].set_visible(False)\n ax.spines[\"right\"].set_visible(False)\n ax.spines[\"left\"].set_visible(False)\n ax.spines[\"bottom\"].set_visible(True)\n ax.xaxis.set_ticks_position(\"bottom\")\n ax.tick_params(\n axis=\"x\", direction=\"out\", width=1, length=3, color=\"0.5\", labelsize=xt_labelsize\n )\n ax.spines[\"bottom\"].set_color(\"0.5\")\n\n if kind == \"kde\" and values.dtype.kind == \"f\":\n plot_kde(\n values,\n bw=bw,\n fill_kwargs={\"alpha\": kwargs.pop(\"fill_alpha\", 0)},\n plot_kwargs={\"linewidth\": linewidth},\n ax=ax,\n rug=False,\n )\n else:\n if bins is None:\n if values.dtype.kind == \"i\":\n xmin = values.min()\n xmax = values.max()\n bins = range(xmin, xmax + 2)\n ax.set_xlim(xmin - 0.5, xmax + 0.5)\n else:\n bins = \"auto\"\n kwargs.setdefault(\"align\", \"left\")\n kwargs.setdefault(\"color\", \"C0\")\n ax.hist(values, bins=bins, alpha=0.35, **kwargs)\n\n plot_height = ax.get_ylim()[1]\n\n format_axes()\n display_hpd()\n display_point_estimate()\n display_ref_val()\n display_rope()", "response": "Plots the posterior of the current object."} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef bfmi(energy):\n energy_mat = np.atleast_2d(energy)\n num = np.square(np.diff(energy_mat, axis=1)).mean(axis=1) # pylint: disable=no-member\n den = np.var(energy_mat, axis=1)\n return num / den", "response": "r Calculates the estimated Bayesian fraction of missing information of the model and trace."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef compare(\n dataset_dict,\n ic=\"waic\",\n method=\"BB-pseudo-BMA\",\n b_samples=1000,\n alpha=1,\n seed=None,\n scale=\"deviance\",\n):\n r\"\"\"Compare models based on WAIC or LOO cross validation.\n\n WAIC is Widely applicable information criterion, and LOO is leave-one-out\n (LOO) cross-validation. Read more theory here - in a paper by some of the\n leading authorities on model selection - dx.doi.org/10.1111/1467-9868.00353\n\n Parameters\n ----------\n dataset_dict : dict[str] -> InferenceData\n A dictionary of model names and InferenceData objects\n ic : str\n Information Criterion (WAIC or LOO) used to compare models. Default WAIC.\n method : str\n Method used to estimate the weights for each model. Available options are:\n\n - 'stacking' : stacking of predictive distributions.\n - 'BB-pseudo-BMA' : (default) pseudo-Bayesian Model averaging using Akaike-type\n weighting. The weights are stabilized using the Bayesian bootstrap\n - 'pseudo-BMA': pseudo-Bayesian Model averaging using Akaike-type\n weighting, without Bootstrap stabilization (not recommended)\n\n For more information read https://arxiv.org/abs/1704.02030\n b_samples: int\n Number of samples taken by the Bayesian bootstrap estimation.\n Only useful when method = 'BB-pseudo-BMA'.\n alpha : float\n The shape parameter in the Dirichlet distribution used for the Bayesian bootstrap. Only\n useful when method = 'BB-pseudo-BMA'. When alpha=1 (default), the distribution is uniform\n on the simplex. A smaller alpha will keeps the final weights more away from 0 and 1.\n seed : int or np.random.RandomState instance\n If int or RandomState, use it for seeding Bayesian bootstrap. Only\n useful when method = 'BB-pseudo-BMA'. Default None the global\n np.random state is used.\n scale : str\n Output scale for IC. Available options are:\n\n - `deviance` : (default) -2 * (log-score)\n - `log` : 1 * log-score (after Vehtari et al. (2017))\n - `negative_log` : -1 * (log-score)\n\n Returns\n -------\n A DataFrame, ordered from lowest to highest IC. The index reflects the order in which the\n models are passed to this function. The columns are:\n IC : Information Criteria (WAIC or LOO).\n Smaller IC indicates higher out-of-sample predictive fit (\"better\" model). Default WAIC.\n If `scale == log` higher IC indicates higher out-of-sample predictive fit (\"better\" model).\n pIC : Estimated effective number of parameters.\n dIC : Relative difference between each IC (WAIC or LOO)\n and the lowest IC (WAIC or LOO).\n It's always 0 for the top-ranked model.\n weight: Relative weight for each model.\n This can be loosely interpreted as the probability of each model (among the compared model)\n given the data. By default the uncertainty in the weights estimation is considered using\n Bayesian bootstrap.\n SE : Standard error of the IC estimate.\n If method = BB-pseudo-BMA these values are estimated using Bayesian bootstrap.\n dSE : Standard error of the difference in IC between each model and\n the top-ranked model.\n It's always 0 for the top-ranked model.\n warning : A value of 1 indicates that the computation of the IC may not be reliable. This could\n be indication of WAIC/LOO starting to fail see http://arxiv.org/abs/1507.04544 for details.\n scale : Scale used for the IC.\n \"\"\"\n names = list(dataset_dict.keys())\n scale = scale.lower()\n if scale == \"log\":\n scale_value = 1\n ascending = False\n else:\n if scale == \"negative_log\":\n scale_value = -1\n else:\n scale_value = -2\n ascending = True\n\n if ic == \"waic\":\n ic_func = waic\n df_comp = pd.DataFrame(\n index=names,\n columns=[\"waic\", \"p_waic\", \"d_waic\", \"weight\", \"se\", \"dse\", \"warning\", \"waic_scale\"],\n )\n scale_col = \"waic_scale\"\n\n elif ic == \"loo\":\n ic_func = loo\n df_comp = pd.DataFrame(\n index=names,\n columns=[\"loo\", \"p_loo\", \"d_loo\", \"weight\", \"se\", \"dse\", \"warning\", \"loo_scale\"],\n )\n scale_col = \"loo_scale\"\n\n else:\n raise NotImplementedError(\"The information criterion {} is not supported.\".format(ic))\n\n if method.lower() not in [\"stacking\", \"bb-pseudo-bma\", \"pseudo-bma\"]:\n raise ValueError(\"The method {}, to compute weights, is not supported.\".format(method))\n\n ic_se = \"{}_se\".format(ic)\n p_ic = \"p_{}\".format(ic)\n ic_i = \"{}_i\".format(ic)\n\n ics = pd.DataFrame()\n names = []\n for name, dataset in dataset_dict.items():\n names.append(name)\n ics = ics.append([ic_func(dataset, pointwise=True, scale=scale)])\n ics.index = names\n ics.sort_values(by=ic, inplace=True, ascending=ascending)\n\n if method.lower() == \"stacking\":\n rows, cols, ic_i_val = _ic_matrix(ics, ic_i)\n exp_ic_i = np.exp(ic_i_val / scale_value)\n last_col = cols - 1\n\n def w_fuller(weights):\n return np.concatenate((weights, [max(1.0 - np.sum(weights), 0.0)]))\n\n def log_score(weights):\n w_full = w_fuller(weights)\n score = 0.0\n for i in range(rows):\n score += np.log(np.dot(exp_ic_i[i], w_full))\n return -score\n\n def gradient(weights):\n w_full = w_fuller(weights)\n grad = np.zeros(last_col)\n for k in range(last_col - 1):\n for i in range(rows):\n grad[k] += (exp_ic_i[i, k] - exp_ic_i[i, last_col]) / np.dot(\n exp_ic_i[i], w_full\n )\n return -grad\n\n theta = np.full(last_col, 1.0 / cols)\n bounds = [(0.0, 1.0) for _ in range(last_col)]\n constraints = [\n {\"type\": \"ineq\", \"fun\": lambda x: 1.0 - np.sum(x)},\n {\"type\": \"ineq\", \"fun\": np.sum},\n ]\n\n weights = minimize(\n fun=log_score, x0=theta, jac=gradient, bounds=bounds, constraints=constraints\n )\n\n weights = w_fuller(weights[\"x\"])\n ses = ics[ic_se]\n\n elif method.lower() == \"bb-pseudo-bma\":\n rows, cols, ic_i_val = _ic_matrix(ics, ic_i)\n ic_i_val = ic_i_val * rows\n\n b_weighting = st.dirichlet.rvs(alpha=[alpha] * rows, size=b_samples, random_state=seed)\n weights = np.zeros((b_samples, cols))\n z_bs = np.zeros_like(weights)\n for i in range(b_samples):\n z_b = np.dot(b_weighting[i], ic_i_val)\n u_weights = np.exp((z_b - np.min(z_b)) / scale_value)\n z_bs[i] = z_b # pylint: disable=unsupported-assignment-operation\n weights[i] = u_weights / np.sum(u_weights)\n\n weights = weights.mean(axis=0)\n ses = pd.Series(z_bs.std(axis=0), index=names) # pylint: disable=no-member\n\n elif method.lower() == \"pseudo-bma\":\n min_ic = ics.iloc[0][ic]\n z_rv = np.exp((ics[ic] - min_ic) / scale_value)\n weights = z_rv / np.sum(z_rv)\n ses = ics[ic_se]\n\n if np.any(weights):\n min_ic_i_val = ics[ic_i].iloc[0]\n for idx, val in enumerate(ics.index):\n res = ics.loc[val]\n if scale_value < 0:\n diff = res[ic_i] - min_ic_i_val\n else:\n diff = min_ic_i_val - res[ic_i]\n d_ic = np.sum(diff)\n d_std_err = np.sqrt(len(diff) * np.var(diff))\n std_err = ses.loc[val]\n weight = weights[idx]\n df_comp.at[val] = (\n res[ic],\n res[p_ic],\n d_ic,\n weight,\n std_err,\n d_std_err,\n res[\"warning\"],\n res[scale_col],\n )\n\n return df_comp.sort_values(by=ic, ascending=ascending)", "response": "r Compare models based on WAIC or LOO cross - validation."} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _ic_matrix(ics, ic_i):\n cols, _ = ics.shape\n rows = len(ics[ic_i].iloc[0])\n ic_i_val = np.zeros((rows, cols))\n\n for idx, val in enumerate(ics.index):\n ic = ics.loc[val][ic_i]\n\n if len(ic) != rows:\n raise ValueError(\"The number of observations should be the same across all models\")\n\n ic_i_val[:, idx] = ic\n\n return rows, cols, ic_i_val", "response": "Store the previously computed pointwise predictive accuracy values ( ics ) in a 2D matrix."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncalculating the HPD of array for given credible interval.", "response": "def hpd(x, credible_interval=0.94, circular=False):\n \"\"\"\n Calculate highest posterior density (HPD) of array for given credible_interval.\n\n The HPD is the minimum width Bayesian credible interval (BCI). This implementation works only\n for unimodal distributions.\n\n Parameters\n ----------\n x : Numpy array\n An array containing posterior samples\n credible_interval : float, optional\n Credible interval to compute. Defaults to 0.94.\n circular : bool, optional\n Whether to compute the hpd taking into account `x` is a circular variable\n (in the range [-np.pi, np.pi]) or not. Defaults to False (i.e non-circular variables).\n\n Returns\n -------\n np.ndarray\n lower and upper value of the interval.\n \"\"\"\n if x.ndim > 1:\n hpd_array = np.array(\n [hpd(row, credible_interval=credible_interval, circular=circular) for row in x.T]\n )\n return hpd_array\n # Make a copy of trace\n x = x.copy()\n len_x = len(x)\n\n if circular:\n mean = st.circmean(x, high=np.pi, low=-np.pi)\n x = x - mean\n x = np.arctan2(np.sin(x), np.cos(x))\n\n x = np.sort(x)\n interval_idx_inc = int(np.floor(credible_interval * len_x))\n n_intervals = len_x - interval_idx_inc\n interval_width = x[interval_idx_inc:] - x[:n_intervals]\n\n if len(interval_width) == 0:\n raise ValueError(\n \"Too few elements for interval calculation. \"\n \"Check that credible_interval meets condition 0 =< credible_interval < 1\"\n )\n\n min_idx = np.argmin(interval_width)\n hdi_min = x[min_idx]\n hdi_max = x[min_idx + interval_idx_inc]\n\n if circular:\n hdi_min = hdi_min + mean\n hdi_max = hdi_max + mean\n hdi_min = np.arctan2(np.sin(hdi_min), np.cos(hdi_min))\n hdi_max = np.arctan2(np.sin(hdi_max), np.cos(hdi_max))\n\n return np.array([hdi_min, hdi_max])"} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _logsumexp(ary, *, b=None, b_inv=None, axis=None, keepdims=False, out=None, copy=True):\n # check dimensions for result arrays\n ary = np.asarray(ary)\n if ary.dtype.kind == \"i\":\n ary = ary.astype(np.float64)\n dtype = ary.dtype.type\n shape = ary.shape\n shape_len = len(shape)\n if isinstance(axis, Sequence):\n axis = tuple(axis_i if axis_i >= 0 else shape_len + axis_i for axis_i in axis)\n agroup = axis\n else:\n axis = axis if (axis is None) or (axis >= 0) else shape_len + axis\n agroup = (axis,)\n shape_max = (\n tuple(1 for _ in shape)\n if axis is None\n else tuple(1 if i in agroup else d for i, d in enumerate(shape))\n )\n # create result arrays\n if out is None:\n if not keepdims:\n out_shape = (\n tuple()\n if axis is None\n else tuple(d for i, d in enumerate(shape) if i not in agroup)\n )\n else:\n out_shape = shape_max\n out = np.empty(out_shape, dtype=dtype)\n if b_inv == 0:\n return np.full_like(out, np.inf, dtype=dtype) if out.shape else np.inf\n if b_inv is None and b == 0:\n return np.full_like(out, -np.inf) if out.shape else -np.inf\n ary_max = np.empty(shape_max, dtype=dtype)\n # calculations\n ary.max(axis=axis, keepdims=True, out=ary_max)\n if copy:\n ary = ary.copy()\n ary -= ary_max\n np.exp(ary, out=ary)\n ary.sum(axis=axis, keepdims=keepdims, out=out)\n np.log(out, out=out)\n if b_inv is not None:\n ary_max -= np.log(b_inv)\n elif b:\n ary_max += np.log(b)\n out += ary_max.squeeze() if not keepdims else ary_max\n # transform to scalar if possible\n return out if out.shape else dtype(out)", "response": "Stable logsumexp when b < 0 and b is scalar."} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncompute the leave - one - out cross - validation for an unweighted model fit.", "response": "def loo(data, pointwise=False, reff=None, scale=\"deviance\"):\n \"\"\"Pareto-smoothed importance sampling leave-one-out cross-validation.\n\n Calculates leave-one-out (LOO) cross-validation for out of sample predictive model fit,\n following Vehtari et al. (2017). Cross-validation is computed using Pareto-smoothed\n importance sampling (PSIS).\n\n Parameters\n ----------\n data : result of MCMC run\n pointwise: bool, optional\n if True the pointwise predictive accuracy will be returned. Defaults to False\n reff : float, optional\n Relative MCMC efficiency, `effective_n / n` i.e. number of effective samples divided by\n the number of actual samples. Computed from trace by default.\n scale : str\n Output scale for loo. Available options are:\n\n - `deviance` : (default) -2 * (log-score)\n - `log` : 1 * log-score (after Vehtari et al. (2017))\n - `negative_log` : -1 * (log-score)\n\n Returns\n -------\n pandas.Series with the following columns:\n loo: approximated Leave-one-out cross-validation\n loo_se: standard error of loo\n p_loo: effective number of parameters\n shape_warn: 1 if the estimated shape parameter of\n Pareto distribution is greater than 0.7 for one or more samples\n loo_i: array of pointwise predictive accuracy, only if pointwise True\n pareto_k: array of Pareto shape values, only if pointwise True\n loo_scale: scale of the loo results\n \"\"\"\n inference_data = convert_to_inference_data(data)\n for group in (\"posterior\", \"sample_stats\"):\n if not hasattr(inference_data, group):\n raise TypeError(\n \"Must be able to extract a {group}\" \"group from data!\".format(group=group)\n )\n if \"log_likelihood\" not in inference_data.sample_stats:\n raise TypeError(\"Data must include log_likelihood in sample_stats\")\n posterior = inference_data.posterior\n log_likelihood = inference_data.sample_stats.log_likelihood\n n_samples = log_likelihood.chain.size * log_likelihood.draw.size\n new_shape = (n_samples,) + log_likelihood.shape[2:]\n log_likelihood = log_likelihood.values.reshape(*new_shape)\n\n if scale.lower() == \"deviance\":\n scale_value = -2\n elif scale.lower() == \"log\":\n scale_value = 1\n elif scale.lower() == \"negative_log\":\n scale_value = -1\n else:\n raise TypeError('Valid scale values are \"deviance\", \"log\", \"negative_log\"')\n\n if reff is None:\n n_chains = len(posterior.chain)\n if n_chains == 1:\n reff = 1.0\n else:\n ess = effective_sample_size(posterior)\n # this mean is over all data variables\n reff = np.hstack([ess[v].values.flatten() for v in ess.data_vars]).mean() / n_samples\n\n log_weights, pareto_shape = psislw(-log_likelihood, reff)\n log_weights += log_likelihood\n\n warn_mg = 0\n if np.any(pareto_shape > 0.7):\n warnings.warn(\n \"\"\"Estimated shape parameter of Pareto distribution is greater than 0.7 for\n one or more samples. You should consider using a more robust model, this is because\n importance sampling is less likely to work well if the marginal posterior and LOO posterior\n are very different. This is more likely to happen with a non-robust model and highly\n influential observations.\"\"\"\n )\n warn_mg = 1\n\n loo_lppd_i = scale_value * _logsumexp(log_weights, axis=0)\n loo_lppd = loo_lppd_i.sum()\n loo_lppd_se = (len(loo_lppd_i) * np.var(loo_lppd_i)) ** 0.5\n\n lppd = np.sum(_logsumexp(log_likelihood, axis=0, b_inv=log_likelihood.shape[0]))\n p_loo = lppd - loo_lppd / scale_value\n\n if pointwise:\n if np.equal(loo_lppd, loo_lppd_i).all(): # pylint: disable=no-member\n warnings.warn(\n \"\"\"The point-wise LOO is the same with the sum LOO, please double check\n the Observed RV in your model to make sure it returns element-wise logp.\n \"\"\"\n )\n return pd.Series(\n data=[loo_lppd, loo_lppd_se, p_loo, warn_mg, loo_lppd_i, pareto_shape, scale],\n index=[\"loo\", \"loo_se\", \"p_loo\", \"warning\", \"loo_i\", \"pareto_k\", \"loo_scale\"],\n )\n\n else:\n return pd.Series(\n data=[loo_lppd, loo_lppd_se, p_loo, warn_mg, scale],\n index=[\"loo\", \"loo_se\", \"p_loo\", \"warning\", \"loo_scale\"],\n )"} +{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nestimate the parameters of the Generalized Pareto Distribution ( GPD ).", "response": "def _gpdfit(x):\n \"\"\"Estimate the parameters for the Generalized Pareto Distribution (GPD).\n\n Empirical Bayes estimate for the parameters of the generalized Pareto\n distribution given the data.\n\n Parameters\n ----------\n x : array\n sorted 1D data array\n\n Returns\n -------\n k : float\n estimated shape parameter\n sigma : float\n estimated scale parameter\n \"\"\"\n prior_bs = 3\n prior_k = 10\n len_x = len(x)\n m_est = 30 + int(len_x ** 0.5)\n\n b_ary = 1 - np.sqrt(m_est / (np.arange(1, m_est + 1, dtype=float) - 0.5))\n b_ary /= prior_bs * x[int(len_x / 4 + 0.5) - 1]\n b_ary += 1 / x[-1]\n\n k_ary = np.log1p(-b_ary[:, None] * x).mean(axis=1) # pylint: disable=no-member\n len_scale = len_x * (np.log(-(b_ary / k_ary)) - k_ary - 1)\n weights = 1 / np.exp(len_scale - len_scale[:, None]).sum(axis=1)\n\n # remove negligible weights\n real_idxs = weights >= 10 * np.finfo(float).eps\n if not np.all(real_idxs):\n weights = weights[real_idxs]\n b_ary = b_ary[real_idxs]\n # normalise weights\n weights /= weights.sum()\n\n # posterior mean for b\n b_post = np.sum(b_ary * weights)\n # estimate for k\n k_post = np.log1p(-b_post * x).mean() # pylint: disable=invalid-unary-operand-type,no-member\n # add prior for k_post\n k_post = (len_x * k_post + prior_k * 0.5) / (len_x + prior_k)\n sigma = -k_post / b_post\n\n return k_post, sigma"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _gpinv(probs, kappa, sigma):\n x = np.full_like(probs, np.nan)\n if sigma <= 0:\n return x\n ok = (probs > 0) & (probs < 1)\n if np.all(ok):\n if np.abs(kappa) < np.finfo(float).eps:\n x = -np.log1p(-probs) # pylint: disable=invalid-unary-operand-type\n else:\n x = np.expm1(-kappa * np.log1p(-probs)) / kappa\n x *= sigma\n else:\n if np.abs(kappa) < np.finfo(float).eps:\n x[ok] = -np.log1p(-probs[ok]) # pylint: disable=unsupported-assignment-operation, E1130\n else:\n x[ok] = ( # pylint: disable=unsupported-assignment-operation\n np.expm1(-kappa * np.log1p(-probs[ok])) / kappa\n )\n x *= sigma\n x[probs == 0] = 0\n if kappa >= 0:\n x[probs == 1] = np.inf # pylint: disable=unsupported-assignment-operation\n else:\n x[probs == 1] = -sigma / kappa # pylint: disable=unsupported-assignment-operation\n\n return x", "response": "Inverse Generalized Pareto distribution function."} +{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef r2_score(y_true, y_pred):\n if y_pred.ndim == 1:\n var_y_est = np.var(y_pred)\n var_e = np.var(y_true - y_pred)\n else:\n var_y_est = np.var(y_pred.mean(0))\n var_e = np.var(y_true - y_pred, 0)\n\n r_squared = var_y_est / (var_y_est + var_e)\n\n return pd.Series([np.mean(r_squared), np.std(r_squared)], index=[\"r2\", \"r2_std\"])", "response": "R\u00b2 for Bayesian regression models only valid for linear models. Only valid for linear models."} +{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncreate a data frame with summary statistics.", "response": "def summary(\n data,\n var_names=None,\n fmt=\"wide\",\n round_to=2,\n include_circ=None,\n stat_funcs=None,\n extend=True,\n credible_interval=0.94,\n order=\"C\",\n):\n \"\"\"Create a data frame with summary statistics.\n\n Parameters\n ----------\n data : obj\n Any object that can be converted to an az.InferenceData object\n Refer to documentation of az.convert_to_dataset for details\n var_names : list\n Names of variables to include in summary\n include_circ : bool\n Whether to include circular statistics\n fmt : {'wide', 'long', 'xarray'}\n Return format is either pandas.DataFrame {'wide', 'long'} or xarray.Dataset {'xarray'}.\n round_to : int\n Number of decimals used to round results. Defaults to 2.\n stat_funcs : dict\n A list of functions or a dict of functions with function names as keys used to calculate\n statistics. By default, the mean, standard deviation, simulation standard error, and\n highest posterior density intervals are included.\n\n The functions will be given one argument, the samples for a variable as an nD array,\n The functions should be in the style of a ufunc and return a single number. For example,\n `np.mean`, or `scipy.stats.var` would both work.\n extend : boolean\n If True, use the statistics returned by `stat_funcs` in addition to, rather than in place\n of, the default statistics. This is only meaningful when `stat_funcs` is not None.\n credible_interval : float, optional\n Credible interval to plot. Defaults to 0.94. This is only meaningful when `stat_funcs` is\n None.\n order : {\"C\", \"F\"}\n If fmt is \"wide\", use either C or F unpacking order. Defaults to C.\n\n Returns\n -------\n pandas.DataFrame\n With summary statistics for each variable. Defaults statistics are: `mean`, `sd`,\n `hpd_3%`, `hpd_97%`, `mc_error`, `ess` and `r_hat`. `ess` and `r_hat` are only computed\n for traces with 2 or more chains.\n\n Examples\n --------\n .. code:: ipython\n\n >>> az.summary(trace, ['mu'])\n mean sd mc_error hpd_3 hpd_97 ess r_hat\n mu[0] 0.10 0.06 0.00 -0.02 0.23 487.0 1.00\n mu[1] -0.04 0.06 0.00 -0.17 0.08 379.0 1.00\n\n Other statistics can be calculated by passing a list of functions.\n\n .. code:: ipython\n\n >>> import pandas as pd\n >>> def trace_sd(x):\n ... return pd.Series(np.std(x, 0), name='sd')\n ...\n >>> def trace_quantiles(x):\n ... return pd.DataFrame(pd.quantiles(x, [5, 50, 95]))\n ...\n >>> az.summary(trace, ['mu'], stat_funcs=[trace_sd, trace_quantiles])\n sd 5 50 95\n mu[0] 0.06 0.00 0.10 0.21\n mu[1] 0.07 -0.16 -0.04 0.06\n \"\"\"\n posterior = convert_to_dataset(data, group=\"posterior\")\n var_names = _var_names(var_names, posterior)\n posterior = posterior if var_names is None else posterior[var_names]\n\n fmt_group = (\"wide\", \"long\", \"xarray\")\n if not isinstance(fmt, str) or (fmt.lower() not in fmt_group):\n raise TypeError(\"Invalid format: '{}'! Formatting options are: {}\".format(fmt, fmt_group))\n\n unpack_order_group = (\"C\", \"F\")\n if not isinstance(order, str) or (order.upper() not in unpack_order_group):\n raise TypeError(\n \"Invalid order: '{}'! Unpacking options are: {}\".format(order, unpack_order_group)\n )\n\n alpha = 1 - credible_interval\n\n metrics = []\n metric_names = []\n\n if stat_funcs is not None:\n if isinstance(stat_funcs, dict):\n for stat_func_name, stat_func in stat_funcs.items():\n metrics.append(\n xr.apply_ufunc(\n _make_ufunc(stat_func), posterior, input_core_dims=((\"chain\", \"draw\"),)\n )\n )\n metric_names.append(stat_func_name)\n else:\n for stat_func in stat_funcs:\n metrics.append(\n xr.apply_ufunc(\n _make_ufunc(stat_func), posterior, input_core_dims=((\"chain\", \"draw\"),)\n )\n )\n metric_names.append(stat_func.__name__)\n\n if extend:\n metrics.append(posterior.mean(dim=(\"chain\", \"draw\")))\n metric_names.append(\"mean\")\n\n metrics.append(posterior.std(dim=(\"chain\", \"draw\")))\n metric_names.append(\"sd\")\n\n metrics.append(\n xr.apply_ufunc(_make_ufunc(_mc_error), posterior, input_core_dims=((\"chain\", \"draw\"),))\n )\n metric_names.append(\"mc error\")\n\n metrics.append(\n xr.apply_ufunc(\n _make_ufunc(hpd, index=0, credible_interval=credible_interval),\n posterior,\n input_core_dims=((\"chain\", \"draw\"),),\n )\n )\n metric_names.append(\"hpd {:g}%\".format(100 * alpha / 2))\n\n metrics.append(\n xr.apply_ufunc(\n _make_ufunc(hpd, index=1, credible_interval=credible_interval),\n posterior,\n input_core_dims=((\"chain\", \"draw\"),),\n )\n )\n metric_names.append(\"hpd {:g}%\".format(100 * (1 - alpha / 2)))\n\n if include_circ:\n metrics.append(\n xr.apply_ufunc(\n _make_ufunc(st.circmean, high=np.pi, low=-np.pi),\n posterior,\n input_core_dims=((\"chain\", \"draw\"),),\n )\n )\n metric_names.append(\"circular mean\")\n\n metrics.append(\n xr.apply_ufunc(\n _make_ufunc(st.circstd, high=np.pi, low=-np.pi),\n posterior,\n input_core_dims=((\"chain\", \"draw\"),),\n )\n )\n metric_names.append(\"circular standard deviation\")\n\n metrics.append(\n xr.apply_ufunc(\n _make_ufunc(_mc_error, circular=True),\n posterior,\n input_core_dims=((\"chain\", \"draw\"),),\n )\n )\n metric_names.append(\"circular mc error\")\n\n metrics.append(\n xr.apply_ufunc(\n _make_ufunc(hpd, index=0, credible_interval=credible_interval, circular=True),\n posterior,\n input_core_dims=((\"chain\", \"draw\"),),\n )\n )\n metric_names.append(\"circular hpd {:.2%}\".format(alpha / 2))\n\n metrics.append(\n xr.apply_ufunc(\n _make_ufunc(hpd, index=1, credible_interval=credible_interval, circular=True),\n posterior,\n input_core_dims=((\"chain\", \"draw\"),),\n )\n )\n metric_names.append(\"circular hpd {:.2%}\".format(1 - alpha / 2))\n\n if len(posterior.chain) > 1:\n metrics.append(effective_sample_size(posterior, var_names=var_names))\n metric_names.append(\"ess\")\n\n metrics.append(rhat(posterior, var_names=var_names))\n metric_names.append(\"r_hat\")\n\n joined = xr.concat(metrics, dim=\"metric\").assign_coords(metric=metric_names)\n\n if fmt.lower() == \"wide\":\n dfs = []\n for var_name, values in joined.data_vars.items():\n if len(values.shape[1:]):\n metric = list(values.metric.values)\n data_dict = OrderedDict()\n for idx in np.ndindex(values.shape[1:] if order == \"C\" else values.shape[1:][::-1]):\n if order == \"F\":\n idx = tuple(idx[::-1])\n ser = pd.Series(values[(Ellipsis, *idx)].values, index=metric)\n key = \"{}[{}]\".format(var_name, \",\".join(map(str, idx)))\n data_dict[key] = ser\n df = pd.DataFrame.from_dict(data_dict, orient=\"index\")\n df = df.loc[list(data_dict.keys())]\n else:\n df = values.to_dataframe()\n df.index = list(df.index)\n df = df.T\n dfs.append(df)\n summary_df = pd.concat(dfs, sort=False)\n elif fmt.lower() == \"long\":\n df = joined.to_dataframe().reset_index().set_index(\"metric\")\n df.index = list(df.index)\n summary_df = df\n else:\n summary_df = joined\n return summary_df.round(round_to)"} +{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _make_ufunc(func, index=Ellipsis, **kwargs): # noqa: D202\n\n def _ufunc(ary):\n target = np.empty(ary.shape[:-2])\n for idx in np.ndindex(target.shape):\n target[idx] = np.asarray(func(ary[idx].ravel(), **kwargs))[index]\n return target\n\n return _ufunc", "response": "Make ufunc from function."} +{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _mc_error(x, batches=5, circular=False):\n if x.ndim > 1:\n\n dims = np.shape(x)\n trace = np.transpose([t.ravel() for t in x])\n\n return np.reshape([_mc_error(t, batches) for t in trace], dims[1:])\n\n else:\n if batches == 1:\n if circular:\n std = st.circstd(x, high=np.pi, low=-np.pi)\n else:\n std = np.std(x)\n return std / np.sqrt(len(x))\n\n batched_traces = np.resize(x, (batches, int(len(x) / batches)))\n\n if circular:\n means = st.circmean(batched_traces, high=np.pi, low=-np.pi, axis=1)\n std = st.circstd(means, high=np.pi, low=-np.pi)\n else:\n means = np.mean(batched_traces, 1)\n std = np.std(means)\n\n return std / np.sqrt(batches)", "response": "Calculate the simulation standard error accounting for non - independent samples."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncalculating the widely available information criterion.", "response": "def waic(data, pointwise=False, scale=\"deviance\"):\n \"\"\"Calculate the widely available information criterion.\n\n Also calculates the WAIC's standard error and the effective number of\n parameters of the samples in trace from model. Read more theory here - in\n a paper by some of the leading authorities on model selection\n dx.doi.org/10.1111/1467-9868.00353\n\n Parameters\n ----------\n data : obj\n Any object that can be converted to an az.InferenceData object\n Refer to documentation of az.convert_to_dataset for details\n pointwise: bool\n if True the pointwise predictive accuracy will be returned.\n Default False\n scale : str\n Output scale for loo. Available options are:\n\n - `deviance` : (default) -2 * (log-score)\n - `log` : 1 * log-score\n - `negative_log` : -1 * (log-score)\n\n Returns\n -------\n DataFrame with the following columns:\n waic: widely available information criterion\n waic_se: standard error of waic\n p_waic: effective number parameters\n var_warn: 1 if posterior variance of the log predictive\n densities exceeds 0.4\n waic_i: and array of the pointwise predictive accuracy, only if pointwise True\n waic_scale: scale of the waic results\n \"\"\"\n inference_data = convert_to_inference_data(data)\n for group in (\"sample_stats\",):\n if not hasattr(inference_data, group):\n raise TypeError(\n \"Must be able to extract a {group} group from data!\".format(group=group)\n )\n if \"log_likelihood\" not in inference_data.sample_stats:\n raise TypeError(\"Data must include log_likelihood in sample_stats\")\n log_likelihood = inference_data.sample_stats.log_likelihood\n\n if scale.lower() == \"deviance\":\n scale_value = -2\n elif scale.lower() == \"log\":\n scale_value = 1\n elif scale.lower() == \"negative_log\":\n scale_value = -1\n else:\n raise TypeError('Valid scale values are \"deviance\", \"log\", \"negative_log\"')\n\n n_samples = log_likelihood.chain.size * log_likelihood.draw.size\n new_shape = (n_samples,) + log_likelihood.shape[2:]\n log_likelihood = log_likelihood.values.reshape(*new_shape)\n\n lppd_i = _logsumexp(log_likelihood, axis=0, b_inv=log_likelihood.shape[0])\n\n vars_lpd = np.var(log_likelihood, axis=0)\n warn_mg = 0\n if np.any(vars_lpd > 0.4):\n warnings.warn(\n \"\"\"For one or more samples the posterior variance of the log predictive\n densities exceeds 0.4. This could be indication of WAIC starting to fail see\n http://arxiv.org/abs/1507.04544 for details\n \"\"\"\n )\n warn_mg = 1\n\n waic_i = scale_value * (lppd_i - vars_lpd)\n waic_se = (len(waic_i) * np.var(waic_i)) ** 0.5\n waic_sum = np.sum(waic_i)\n p_waic = np.sum(vars_lpd)\n\n if pointwise:\n if np.equal(waic_sum, waic_i).all(): # pylint: disable=no-member\n warnings.warn(\n \"\"\"The point-wise WAIC is the same with the sum WAIC, please double check\n the Observed RV in your model to make sure it returns element-wise logp.\n \"\"\"\n )\n return pd.Series(\n data=[waic_sum, waic_se, p_waic, warn_mg, waic_i, scale],\n index=[\"waic\", \"waic_se\", \"p_waic\", \"warning\", \"waic_i\", \"waic_scale\"],\n )\n else:\n return pd.Series(\n data=[waic_sum, waic_se, p_waic, warn_mg, scale],\n index=[\"waic\", \"waic_se\", \"p_waic\", \"warning\", \"waic_scale\"],\n )"} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nplot posterior of traces as violin plot.", "response": "def plot_violin(\n data,\n var_names=None,\n quartiles=True,\n credible_interval=0.94,\n shade=0.35,\n bw=4.5,\n sharey=True,\n figsize=None,\n textsize=None,\n ax=None,\n kwargs_shade=None,\n):\n \"\"\"Plot posterior of traces as violin plot.\n\n Notes\n -----\n If multiple chains are provided for a variable they will be combined\n\n Parameters\n ----------\n data : obj\n Any object that can be converted to an az.InferenceData object\n Refer to documentation of az.convert_to_dataset for details\n var_names: list, optional\n List of variables to plot (defaults to None, which results in all variables plotted)\n quartiles : bool, optional\n Flag for plotting the interquartile range, in addition to the credible_interval*100%\n intervals. Defaults to True\n credible_interval : float, optional\n Credible intervals. Defaults to 0.94.\n shade : float\n Alpha blending value for the shaded area under the curve, between 0\n (no shade) and 1 (opaque). Defaults to 0\n bw : float\n Bandwidth scaling factor. Should be larger than 0. The higher this number the smoother the\n KDE will be. Defaults to 4.5 which is essentially the same as the Scott's rule of thumb\n (the default rule used by SciPy).\n figsize : tuple\n Figure size. If None it will be defined automatically.\n textsize: int\n Text size of the point_estimates, axis ticks, and HPD. If None it will be autoscaled\n based on figsize.\n sharey : bool\n Defaults to True, violinplots share a common y-axis scale.\n ax : matplotlib axes\n kwargs_shade : dicts, optional\n Additional keywords passed to `fill_between`, or `barh` to control the shade\n\n Returns\n -------\n ax : matplotlib axes\n \"\"\"\n data = convert_to_dataset(data, group=\"posterior\")\n var_names = _var_names(var_names, data)\n\n plotters = list(xarray_var_iter(data, var_names=var_names, combined=True))\n\n if kwargs_shade is None:\n kwargs_shade = {}\n\n (figsize, ax_labelsize, _, xt_labelsize, linewidth, _) = _scale_fig_size(\n figsize, textsize, 1, len(plotters)\n )\n ax_labelsize *= 2\n\n if ax is None:\n fig, ax = plt.subplots(\n 1, len(plotters), figsize=figsize, sharey=sharey, constrained_layout=True\n )\n\n else:\n fig = ax.figure\n\n ax = np.atleast_1d(ax)\n\n for axind, (var_name, selection, x) in enumerate(plotters):\n val = x.flatten()\n if val[0].dtype.kind == \"i\":\n cat_hist(val, shade, ax[axind], **kwargs_shade)\n else:\n _violinplot(val, shade, bw, ax[axind], **kwargs_shade)\n\n per = np.percentile(val, [25, 75, 50])\n hpd_intervals = hpd(val, credible_interval)\n\n if quartiles:\n ax[axind].plot([0, 0], per[:2], lw=linewidth * 3, color=\"k\", solid_capstyle=\"round\")\n ax[axind].plot([0, 0], hpd_intervals, lw=linewidth, color=\"k\", solid_capstyle=\"round\")\n ax[axind].plot(0, per[-1], \"wo\", ms=linewidth * 1.5)\n\n ax[axind].set_xlabel(make_label(var_name, selection), fontsize=ax_labelsize)\n ax[axind].set_xticks([])\n ax[axind].tick_params(labelsize=xt_labelsize)\n ax[axind].grid(None, axis=\"x\")\n\n if sharey:\n fig.subplots_adjust(wspace=0)\n else:\n fig.tight_layout()\n\n return ax"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef cat_hist(val, shade, ax, **kwargs_shade):\n bins = get_bins(val)\n binned_d, _ = np.histogram(val, bins=bins, normed=True)\n\n bin_edges = np.linspace(np.min(val), np.max(val), len(bins))\n centers = 0.5 * (bin_edges + np.roll(bin_edges, 1))[:-1]\n heights = np.diff(bin_edges)\n\n lefts = -0.5 * binned_d\n ax.barh(centers, binned_d, height=heights, left=lefts, alpha=shade, **kwargs_shade)", "response": "Auxiliary function to plot discrete - violinplots."} +{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nplotting a scatter or hexbin of two variables.", "response": "def plot_joint(\n data,\n var_names=None,\n coords=None,\n figsize=None,\n textsize=None,\n kind=\"scatter\",\n gridsize=\"auto\",\n contour=True,\n fill_last=True,\n joint_kwargs=None,\n marginal_kwargs=None,\n):\n \"\"\"\n Plot a scatter or hexbin of two variables with their respective marginals distributions.\n\n Parameters\n ----------\n data : obj\n Any object that can be converted to an az.InferenceData object\n Refer to documentation of az.convert_to_dataset for details\n var_names : Iter of 2 e.g. (var_1, var_2)\n Variables to be plotted, two variables are required.\n coords : mapping, optional\n Coordinates of var_names to be plotted. Passed to `Dataset.sel`\n figsize : tuple\n Figure size. If None it will be defined automatically.\n textsize: float\n Text size scaling factor for labels, titles and lines. If None it will be autoscaled based\n on figsize.\n kind : str\n Type of plot to display (scatter, kde or hexbin)\n gridsize : int or (int, int), optional.\n The number of hexagons in the x-direction. Ignored when hexbin is False. See `plt.hexbin`\n for details\n contour : bool\n If True plot the 2D KDE using contours, otherwise plot a smooth 2D KDE. Defaults to True.\n fill_last : bool\n If True fill the last contour of the 2D KDE plot. Defaults to True.\n joint_kwargs : dicts, optional\n Additional keywords modifying the join distribution (central subplot)\n marginal_kwargs : dicts, optional\n Additional keywords modifying the marginals distributions (top and right subplot)\n\n Returns\n -------\n axjoin : matplotlib axes, join (central) distribution\n ax_hist_x : matplotlib axes, x (top) distribution\n ax_hist_y : matplotlib axes, y (right) distribution\n\n Examples\n --------\n Scatter Joint plot\n\n .. plot::\n :context: close-figs\n\n >>> import arviz as az\n >>> data = az.load_arviz_data('non_centered_eight')\n >>> az.plot_joint(data,\n >>> var_names=['theta'],\n >>> coords={'school': ['Choate', 'Phillips Andover']},\n >>> kind='scatter',\n >>> figsize=(6, 6))\n\n Hexbin Joint plot\n\n .. plot::\n :context: close-figs\n\n >>> az.plot_joint(data,\n >>> var_names=['theta'],\n >>> coords={'school': ['Choate', 'Phillips Andover']},\n >>> kind='hexbin',\n >>> figsize=(6, 6))\n\n KDE Joint plot\n\n .. plot::\n :context: close-figs\n\n >>> az.plot_joint(data,\n >>> var_names=['theta'],\n >>> coords={'school': ['Choate', 'Phillips Andover']},\n >>> kind='kde',\n >>> figsize=(6, 6))\n\n \"\"\"\n valid_kinds = [\"scatter\", \"kde\", \"hexbin\"]\n if kind not in valid_kinds:\n raise ValueError(\n (\"Plot type {} not recognized.\" \"Plot type must be in {}\").format(kind, valid_kinds)\n )\n\n data = convert_to_dataset(data, group=\"posterior\")\n\n if coords is None:\n coords = {}\n\n var_names = _var_names(var_names, data)\n\n plotters = list(xarray_var_iter(get_coords(data, coords), var_names=var_names, combined=True))\n\n if len(plotters) != 2:\n raise Exception(\n \"Number of variables to be plotted must 2 (you supplied {})\".format(len(plotters))\n )\n\n figsize, ax_labelsize, _, xt_labelsize, linewidth, _ = _scale_fig_size(figsize, textsize)\n\n if joint_kwargs is None:\n joint_kwargs = {}\n\n if marginal_kwargs is None:\n marginal_kwargs = {}\n marginal_kwargs.setdefault(\"plot_kwargs\", {})\n marginal_kwargs[\"plot_kwargs\"][\"linewidth\"] = linewidth\n\n # Instantiate figure and grid\n fig, _ = plt.subplots(0, 0, figsize=figsize, constrained_layout=True)\n grid = plt.GridSpec(4, 4, hspace=0.1, wspace=0.1, figure=fig)\n\n # Set up main plot\n axjoin = fig.add_subplot(grid[1:, :-1])\n\n # Set up top KDE\n ax_hist_x = fig.add_subplot(grid[0, :-1], sharex=axjoin)\n ax_hist_x.tick_params(labelleft=False, labelbottom=False)\n\n # Set up right KDE\n ax_hist_y = fig.add_subplot(grid[1:, -1], sharey=axjoin)\n ax_hist_y.tick_params(labelleft=False, labelbottom=False)\n\n # Set labels for axes\n x_var_name = make_label(plotters[0][0], plotters[0][1])\n y_var_name = make_label(plotters[1][0], plotters[1][1])\n\n axjoin.set_xlabel(x_var_name, fontsize=ax_labelsize)\n axjoin.set_ylabel(y_var_name, fontsize=ax_labelsize)\n axjoin.tick_params(labelsize=xt_labelsize)\n\n # Flatten data\n x = plotters[0][2].flatten()\n y = plotters[1][2].flatten()\n\n if kind == \"scatter\":\n axjoin.scatter(x, y, **joint_kwargs)\n elif kind == \"kde\":\n plot_kde(x, y, contour=contour, fill_last=fill_last, ax=axjoin, **joint_kwargs)\n else:\n if gridsize == \"auto\":\n gridsize = int(len(x) ** 0.35)\n axjoin.hexbin(x, y, mincnt=1, gridsize=gridsize, **joint_kwargs)\n axjoin.grid(False)\n\n for val, ax, rotate in ((x, ax_hist_x, False), (y, ax_hist_y, True)):\n plot_dist(val, textsize=xt_labelsize, rotated=rotate, ax=ax, **marginal_kwargs)\n\n ax_hist_x.set_xlim(axjoin.get_xlim())\n ax_hist_y.set_ylim(axjoin.get_ylim())\n\n return axjoin, ax_hist_x, ax_hist_y"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef plot_hpd(\n x,\n y,\n credible_interval=0.94,\n color=\"C1\",\n circular=False,\n smooth=True,\n smooth_kwargs=None,\n fill_kwargs=None,\n plot_kwargs=None,\n ax=None,\n):\n \"\"\"\n Plot hpd intervals for regression data.\n\n Parameters\n ----------\n x : array-like\n Values to plot\n y : array-like\n values \u200b\u200bfrom which to compute the hpd\n credible_interval : float, optional\n Credible interval to plot. Defaults to 0.94.\n color : str\n Color used for the limits of the HPD interval and fill. Should be a valid matplotlib color\n circular : bool, optional\n Whether to compute the hpd taking into account `x` is a circular variable\n (in the range [-np.pi, np.pi]) or not. Defaults to False (i.e non-circular variables).\n smooth : boolean\n If True the result will be smoothed by first computing a linear interpolation of the data\n over a regular grid and then applying the Savitzky-Golay filter to the interpolated data.\n Defaults to True.\n smooth_kwargs : dict, optional\n Additional keywords modifying the Savitzky-Golay filter. See Scipy's documentation for\n details\n fill_kwargs : dict\n Keywords passed to `fill_between` (use fill_kwargs={'alpha': 0} to disable fill).\n plot_kwargs : dict\n Keywords passed to HPD limits\n ax : matplotlib axes\n\n Returns\n -------\n ax : matplotlib axes\n \"\"\"\n if plot_kwargs is None:\n plot_kwargs = {}\n plot_kwargs.setdefault(\"color\", color)\n plot_kwargs.setdefault(\"alpha\", 0)\n\n if fill_kwargs is None:\n fill_kwargs = {}\n fill_kwargs.setdefault(\"color\", color)\n fill_kwargs.setdefault(\"alpha\", 0.5)\n\n if ax is None:\n ax = gca()\n\n hpd_ = hpd(y, credible_interval=credible_interval, circular=circular)\n\n if smooth:\n if smooth_kwargs is None:\n smooth_kwargs = {}\n smooth_kwargs.setdefault(\"window_length\", 55)\n smooth_kwargs.setdefault(\"polyorder\", 2)\n x_data = np.linspace(x.min(), x.max(), 200)\n hpd_interp = griddata(x, hpd_, x_data)\n y_data = savgol_filter(hpd_interp, axis=0, **smooth_kwargs)\n else:\n idx = np.argsort(x)\n x_data = x[idx]\n y_data = hpd_[idx]\n\n ax.plot(x_data, y_data, **plot_kwargs)\n ax.fill_between(x_data, y_data[:, 0], y_data[:, 1], **fill_kwargs)\n\n return ax", "response": "Plots the HPD intervals for regression data."} +{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncomputes the effective sample size for a 2D array.", "response": "def _get_ess(sample_array):\n \"\"\"Compute the effective sample size for a 2D array.\"\"\"\n shape = sample_array.shape\n if len(shape) != 2:\n raise TypeError(\"Effective sample size calculation requires 2 dimensional arrays.\")\n n_chain, n_draws = shape\n if n_chain <= 1:\n raise TypeError(\"Effective sample size calculation requires multiple chains.\")\n\n acov = np.asarray([_autocov(sample_array[chain]) for chain in range(n_chain)])\n\n chain_mean = sample_array.mean(axis=1)\n chain_var = acov[:, 0] * n_draws / (n_draws - 1.0)\n acov_t = acov[:, 1] * n_draws / (n_draws - 1.0)\n mean_var = np.mean(chain_var)\n var_plus = mean_var * (n_draws - 1.0) / n_draws\n var_plus += np.var(chain_mean, ddof=1)\n\n rho_hat_t = np.zeros(n_draws)\n rho_hat_even = 1.0\n rho_hat_t[0] = rho_hat_even\n rho_hat_odd = 1.0 - (mean_var - np.mean(acov_t)) / var_plus\n rho_hat_t[1] = rho_hat_odd\n\n # Geyer's initial positive sequence\n max_t = 1\n t = 1\n while t < (n_draws - 2) and (rho_hat_even + rho_hat_odd) >= 0.0:\n rho_hat_even = 1.0 - (mean_var - np.mean(acov[:, t + 1])) / var_plus\n rho_hat_odd = 1.0 - (mean_var - np.mean(acov[:, t + 2])) / var_plus\n if (rho_hat_even + rho_hat_odd) >= 0:\n rho_hat_t[t + 1] = rho_hat_even\n rho_hat_t[t + 2] = rho_hat_odd\n max_t = t + 2\n t += 2\n\n # Geyer's initial monotone sequence\n t = 3\n while t <= max_t - 2:\n if (rho_hat_t[t + 1] + rho_hat_t[t + 2]) > (rho_hat_t[t - 1] + rho_hat_t[t]):\n rho_hat_t[t + 1] = (rho_hat_t[t - 1] + rho_hat_t[t]) / 2.0\n rho_hat_t[t + 2] = rho_hat_t[t + 1]\n t += 2\n ess = (\n int((n_chain * n_draws) / (-1.0 + 2.0 * np.sum(rho_hat_t)))\n if not np.any(np.isnan(rho_hat_t))\n else np.nan\n )\n return ess"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _rhat_ufunc(ary):\n target = np.empty(ary.shape[:-2])\n for idx in np.ndindex(target.shape):\n target[idx] = _get_split_rhat(ary[idx])\n return target", "response": "Ufunc for computing effective sample size."} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncomputes the split - rhat for a 2d array.", "response": "def _get_split_rhat(values, round_to=2):\n \"\"\"Compute the split-rhat for a 2d array.\"\"\"\n shape = values.shape\n if len(shape) != 2:\n raise TypeError(\"Effective sample size calculation requires 2 dimensional arrays.\")\n _, num_samples = shape\n num_split = num_samples // 2\n # Calculate split chain mean\n split_chain_mean1 = np.mean(values[:, :num_split], axis=1)\n split_chain_mean2 = np.mean(values[:, num_split:], axis=1)\n split_chain_mean = np.concatenate((split_chain_mean1, split_chain_mean2))\n # Calculate split chain variance\n split_chain_var1 = np.var(values[:, :num_split], axis=1, ddof=1)\n split_chain_var2 = np.var(values[:, num_split:], axis=1, ddof=1)\n split_chain_var = np.concatenate((split_chain_var1, split_chain_var2))\n # Calculate between-chain variance\n between_chain_variance = num_samples / 2 * np.var(split_chain_mean, ddof=1)\n # Calculate within-chain variance\n within_chain_variance = np.mean(split_chain_var)\n # Estimate of marginal posterior variance\n split_rhat = np.sqrt(\n (between_chain_variance / within_chain_variance + num_samples / 2 - 1) / (num_samples / 2)\n )\n\n return round(split_rhat, round_to)"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef geweke(values, first=0.1, last=0.5, intervals=20):\n # Filter out invalid intervals\n for interval in (first, last):\n if interval <= 0 or interval >= 1:\n raise ValueError(\"Invalid intervals for Geweke convergence analysis\", (first, last))\n if first + last >= 1:\n raise ValueError(\"Invalid intervals for Geweke convergence analysis\", (first, last))\n\n # Initialize list of z-scores\n zscores = []\n\n # Last index value\n end = len(values) - 1\n\n # Start intervals going up to the % of the chain\n last_start_idx = (1 - last) * end\n\n # Calculate starting indices\n start_indices = np.linspace(0, last_start_idx, num=intervals, endpoint=True, dtype=int)\n\n # Loop over start indices\n for start in start_indices:\n # Calculate slices\n first_slice = values[start : start + int(first * (end - start))]\n last_slice = values[int(end - last * (end - start)) :]\n\n z_score = first_slice.mean() - last_slice.mean()\n z_score /= np.sqrt(first_slice.var() + last_slice.var())\n\n zscores.append([start, z_score])\n\n return np.array(zscores)", "response": "r Compute the z - scores for convergence diagnostics."} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef ks_summary(pareto_tail_indices):\n kcounts, _ = np.histogram(pareto_tail_indices, bins=[-np.Inf, 0.5, 0.7, 1, np.Inf])\n kprop = kcounts / len(pareto_tail_indices) * 100\n df_k = pd.DataFrame(\n dict(_=[\"(good)\", \"(ok)\", \"(bad)\", \"(very bad)\"], Count=kcounts, Pct=kprop)\n ).rename(index={0: \"(-Inf, 0.5]\", 1: \" (0.5, 0.7]\", 2: \" (0.7, 1]\", 3: \" (1, Inf)\"})\n\n if np.sum(kcounts[1:]) == 0:\n warnings.warn(\"All Pareto k estimates are good (k < 0.5)\")\n elif np.sum(kcounts[2:]) == 0:\n warnings.warn(\"All Pareto k estimates are ok (k < 0.7)\")\n\n return df_k", "response": "Display a summary of the Pareto tail indices."} +{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nplots the trace of the variables in the base base.", "response": "def plot_trace(\n data,\n var_names=None,\n coords=None,\n divergences=\"bottom\",\n figsize=None,\n textsize=None,\n lines=None,\n combined=False,\n plot_kwargs=None,\n fill_kwargs=None,\n rug_kwargs=None,\n hist_kwargs=None,\n trace_kwargs=None,\n):\n \"\"\"Plot distribution (histogram or kernel density estimates) and sampled values.\n\n If `divergences` data is available in `sample_stats`, will plot the location of divergences as\n dashed vertical lines.\n\n Parameters\n ----------\n data : obj\n Any object that can be converted to an az.InferenceData object\n Refer to documentation of az.convert_to_dataset for details\n var_names : list of variable names\n Variables to be plotted, two variables are required.\n coords : mapping, optional\n Coordinates of var_names to be plotted. Passed to `Dataset.sel`\n divergences : {\"bottom\", \"top\", None, False}\n Plot location of divergences on the traceplots. Options are \"bottom\", \"top\", or False-y.\n figsize : figure size tuple\n If None, size is (12, variables * 2)\n textsize: float\n Text size scaling factor for labels, titles and lines. If None it will be autoscaled based\n on figsize.\n lines : tuple\n Tuple of (var_name, {'coord': selection}, [line, positions]) to be overplotted as\n vertical lines on the density and horizontal lines on the trace.\n combined : bool\n Flag for combining multiple chains into a single line. If False (default), chains will be\n plotted separately.\n plot_kwargs : dict\n Extra keyword arguments passed to `arviz.plot_dist`. Only affects continuous variables.\n fill_kwargs : dict\n Extra keyword arguments passed to `arviz.plot_dist`. Only affects continuous variables.\n rug_kwargs : dict\n Extra keyword arguments passed to `arviz.plot_dist`. Only affects continuous variables.\n hist_kwargs : dict\n Extra keyword arguments passed to `arviz.plot_dist`. Only affects discrete variables.\n trace_kwargs : dict\n Extra keyword arguments passed to `plt.plot`\n Returns\n -------\n axes : matplotlib axes\n\n\n Examples\n --------\n Plot a subset variables\n\n .. plot::\n :context: close-figs\n\n >>> import arviz as az\n >>> data = az.load_arviz_data('non_centered_eight')\n >>> coords = {'theta_t_dim_0': [0, 1], 'school':['Lawrenceville']}\n >>> az.plot_trace(data, var_names=('theta_t', 'theta'), coords=coords)\n\n Combine all chains into one distribution and trace\n\n .. plot::\n :context: close-figs\n\n >>> coords = {'theta_t_dim_0': [0, 1], 'school':['Lawrenceville']}\n >>> az.plot_trace(data, var_names=('theta_t', 'theta'), coords=coords, combined=True)\n\n\n Plot reference lines against distribution and trace\n\n .. plot::\n :context: close-figs\n\n >>> lines = (('theta_t',{'theta_t_dim_0':0}, [-1]),)\n >>> coords = {'theta_t_dim_0': [0, 1], 'school':['Lawrenceville']}\n >>> az.plot_trace(data, var_names=('theta_t', 'theta'), coords=coords, lines=lines)\n \"\"\"\n if divergences:\n try:\n divergence_data = convert_to_dataset(data, group=\"sample_stats\").diverging\n except (ValueError, AttributeError): # No sample_stats, or no `.diverging`\n divergences = False\n\n data = convert_to_dataset(data, group=\"posterior\")\n var_names = _var_names(var_names, data)\n\n if coords is None:\n coords = {}\n\n if lines is None:\n lines = ()\n\n plotters = list(xarray_var_iter(get_coords(data, coords), var_names=var_names, combined=True))\n\n if figsize is None:\n figsize = (12, len(plotters) * 2)\n\n if trace_kwargs is None:\n trace_kwargs = {}\n\n trace_kwargs.setdefault(\"alpha\", 0.35)\n\n if hist_kwargs is None:\n hist_kwargs = {}\n if plot_kwargs is None:\n plot_kwargs = {}\n if fill_kwargs is None:\n fill_kwargs = {}\n if rug_kwargs is None:\n rug_kwargs = {}\n\n hist_kwargs.setdefault(\"alpha\", 0.35)\n\n figsize, _, titlesize, xt_labelsize, linewidth, _ = _scale_fig_size(\n figsize, textsize, rows=len(plotters), cols=2\n )\n trace_kwargs.setdefault(\"linewidth\", linewidth)\n plot_kwargs.setdefault(\"linewidth\", linewidth)\n\n _, axes = plt.subplots(\n len(plotters), 2, squeeze=False, figsize=figsize, constrained_layout=True\n )\n\n colors = {}\n for idx, (var_name, selection, value) in enumerate(plotters):\n colors[idx] = []\n if combined:\n value = value.flatten()\n value = np.atleast_2d(value)\n\n for row in value:\n axes[idx, 1].plot(np.arange(len(row)), row, **trace_kwargs)\n\n colors[idx].append(axes[idx, 1].get_lines()[-1].get_color())\n plot_kwargs[\"color\"] = colors[idx][-1]\n plot_dist(\n row,\n textsize=xt_labelsize,\n ax=axes[idx, 0],\n hist_kwargs=hist_kwargs,\n plot_kwargs=plot_kwargs,\n fill_kwargs=fill_kwargs,\n rug_kwargs=rug_kwargs,\n )\n\n if value[0].dtype.kind == \"i\":\n xticks = get_bins(value)\n axes[idx, 0].set_xticks(xticks[:-1])\n axes[idx, 0].set_yticks([])\n for col in (0, 1):\n axes[idx, col].set_title(make_label(var_name, selection), fontsize=titlesize, wrap=True)\n axes[idx, col].tick_params(labelsize=xt_labelsize)\n\n xlims = [ax.get_xlim() for ax in axes[idx, :]]\n ylims = [ax.get_ylim() for ax in axes[idx, :]]\n\n if divergences:\n div_selection = {k: v for k, v in selection.items() if k in divergence_data.dims}\n divs = divergence_data.sel(**div_selection).values\n if combined:\n divs = divs.flatten()\n divs = np.atleast_2d(divs)\n\n for chain, chain_divs in enumerate(divs):\n div_idxs = np.arange(len(chain_divs))[chain_divs]\n if div_idxs.size > 0:\n if divergences == \"top\":\n ylocs = [ylim[1] for ylim in ylims]\n else:\n ylocs = [ylim[0] for ylim in ylims]\n values = value[chain, div_idxs]\n axes[idx, 1].plot(\n div_idxs,\n np.zeros_like(div_idxs) + ylocs[1],\n marker=\"|\",\n color=\"black\",\n markeredgewidth=1.5,\n markersize=30,\n linestyle=\"None\",\n alpha=hist_kwargs[\"alpha\"],\n zorder=-5,\n )\n axes[idx, 1].set_ylim(*ylims[1])\n axes[idx, 0].plot(\n values,\n np.zeros_like(values) + ylocs[0],\n marker=\"|\",\n color=\"black\",\n markeredgewidth=1.5,\n markersize=30,\n linestyle=\"None\",\n alpha=trace_kwargs[\"alpha\"],\n zorder=-5,\n )\n axes[idx, 0].set_ylim(*ylims[0])\n\n for _, _, vlines in (j for j in lines if j[0] == var_name and j[1] == selection):\n if isinstance(vlines, (float, int)):\n line_values = [vlines]\n else:\n line_values = np.atleast_1d(vlines).ravel()\n axes[idx, 0].vlines(\n line_values, *ylims[0], colors=colors[idx][0], linewidth=1.5, alpha=0.75\n )\n axes[idx, 1].hlines(\n line_values,\n *xlims[1],\n colors=colors[idx][0],\n linewidth=1.5,\n alpha=trace_kwargs[\"alpha\"]\n )\n axes[idx, 0].set_ylim(bottom=0, top=ylims[0][1])\n axes[idx, 1].set_xlim(left=0, right=data.draw.max())\n axes[idx, 1].set_ylim(*ylims[1])\n return axes"} +{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _histplot_op(ax, data, **kwargs):\n bins = get_bins(data)\n ax.hist(data, bins=bins, align=\"left\", density=True, **kwargs)\n return ax", "response": "Add a histogram for the data to the axes."} +{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nconcatenates the given list of InferenceData objects on a group level.", "response": "def concat(*args, copy=True, inplace=False):\n \"\"\"Concatenate InferenceData objects on a group level.\n\n Supports only concatenating with independent unique groups.\n\n Parameters\n ----------\n *args : InferenceData\n Variable length InferenceData list or\n Sequence of InferenceData.\n copy : bool\n If True, groups are copied to the new InferenceData object.\n inplace : bool\n If True, merge args to first object.\n\n Returns\n -------\n InferenceData\n A new InferenceData object by default.\n When `inplace==True` merge args to first arg and return `None`\n \"\"\"\n if len(args) == 0:\n return InferenceData()\n if len(args) == 1 and isinstance(args[0], Sequence):\n args = args[0]\n elif len(args) == 1:\n if isinstance(args[0], InferenceData):\n if inplace:\n return None\n else:\n if copy:\n return deepcopy(args[0])\n else:\n return args[0]\n\n # assert that all args are InferenceData\n for i, arg in enumerate(args):\n if not isinstance(arg, InferenceData):\n raise TypeError(\n \"Concatenating is supported only\"\n \"between InferenceData objects. Input arg {} is {}\".format(i, type(arg))\n )\n # assert that groups are independent\n first_arg = args[0]\n first_arg_groups = ccopy(first_arg._groups)\n args_groups = dict()\n for arg in args[1:]:\n for group in arg._groups:\n if group in args_groups or group in first_arg_groups:\n raise NotImplementedError(\"Concatenating with overlapping groups is not supported.\")\n group_data = getattr(arg, group)\n args_groups[group] = deepcopy(group_data) if copy else group_data\n\n # add first_arg to args_groups if inplace is False\n if not inplace:\n for group in first_arg_groups:\n group_data = getattr(first_arg, group)\n args_groups[group] = deepcopy(group_data) if copy else group_data\n\n basic_order = [\n \"posterior\",\n \"posterior_predictive\",\n \"sample_stats\",\n \"prior\",\n \"prior_predictive\",\n \"sample_stats_prior\",\n \"observed_data\",\n ]\n other_groups = [group for group in args_groups if group not in basic_order]\n\n if not inplace:\n # Keep order for python 3.5\n inference_data_dict = OrderedDict()\n for group in basic_order + other_groups:\n if group not in args_groups:\n continue\n if inplace:\n first_arg._groups.append(group)\n setattr(first_arg, group, args_groups[group])\n else:\n inference_data_dict[group] = args_groups[group]\n if inplace:\n other_groups = [\n group for group in first_arg_groups if group not in basic_order\n ] + other_groups\n sorted_groups = [\n group for group in basic_order + other_groups if group in first_arg._groups\n ]\n setattr(first_arg, \"_groups\", sorted_groups)\n return None\n return InferenceData(**inference_data_dict)"} +{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef from_netcdf(filename):\n groups = {}\n with nc.Dataset(filename, mode=\"r\") as data:\n data_groups = list(data.groups)\n\n for group in data_groups:\n with xr.open_dataset(filename, group=group) as data:\n groups[group] = data\n return InferenceData(**groups)", "response": "Initialize an InferenceData object from a netcdf file."} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef to_netcdf(self, filename, compress=True):\n mode = \"w\" # overwrite first, then append\n if self._groups: # check's whether a group is present or not.\n for group in self._groups:\n data = getattr(self, group)\n kwargs = {}\n if compress:\n kwargs[\"encoding\"] = {var_name: {\"zlib\": True} for var_name in data.variables}\n data.to_netcdf(filename, mode=mode, group=group, **kwargs)\n data.close()\n mode = \"a\"\n else: # creates a netcdf file for an empty InferenceData object.\n empty_netcdf_file = nc.Dataset(filename, mode=\"w\", format=\"NETCDF4\")\n empty_netcdf_file.close()\n return filename", "response": "Write InferenceData to file using netcdf4."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nconvert dictionary data into an InferenceData object.", "response": "def from_dict(\n posterior=None,\n *,\n posterior_predictive=None,\n sample_stats=None,\n prior=None,\n prior_predictive=None,\n sample_stats_prior=None,\n observed_data=None,\n coords=None,\n dims=None\n):\n \"\"\"Convert Dictionary data into an InferenceData object.\n\n Parameters\n ----------\n posterior : dict\n posterior_predictive : dict\n sample_stats : dict\n \"log_likelihood\" variable for stats needs to be here.\n prior : dict\n prior_predictive : dict\n observed_data : dict\n coords : dict[str, iterable]\n A dictionary containing the values that are used as index. The key\n is the name of the dimension, the values are the index values.\n dims : dict[str, List(str)]\n A mapping from variables to a list of coordinate names for the variable.\n\n Returns\n -------\n InferenceData object\n \"\"\"\n return DictConverter(\n posterior=posterior,\n posterior_predictive=posterior_predictive,\n sample_stats=sample_stats,\n prior=prior,\n prior_predictive=prior_predictive,\n sample_stats_prior=sample_stats_prior,\n observed_data=observed_data,\n coords=coords,\n dims=dims,\n ).to_inference_data()"} +{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef posterior_to_xarray(self):\n data = self.posterior\n if not isinstance(data, dict):\n raise TypeError(\"DictConverter.posterior is not a dictionary\")\n\n if \"log_likelihood\" in data:\n warnings.warn(\n \"log_likelihood found in posterior.\"\n \" For stats functions log_likelihood needs to be in sample_stats.\",\n SyntaxWarning,\n )\n\n return dict_to_dataset(data, library=None, coords=self.coords, dims=self.dims)", "response": "Convert posterior samples to xarray."} +{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nconverting sample_stats samples to xarray.", "response": "def sample_stats_to_xarray(self):\n \"\"\"Convert sample_stats samples to xarray.\"\"\"\n data = self.sample_stats\n if not isinstance(data, dict):\n raise TypeError(\"DictConverter.sample_stats is not a dictionary\")\n\n return dict_to_dataset(data, library=None, coords=self.coords, dims=self.dims)"} +{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nconverting posterior_predictive samples to xarray.", "response": "def posterior_predictive_to_xarray(self):\n \"\"\"Convert posterior_predictive samples to xarray.\"\"\"\n data = self.posterior_predictive\n if not isinstance(data, dict):\n raise TypeError(\"DictConverter.posterior_predictive is not a dictionary\")\n\n return dict_to_dataset(data, library=None, coords=self.coords, dims=self.dims)"}